diff options
| author | CoprDistGit <infra@openeuler.org> | 2025-01-25 05:07:37 +0000 |
|---|---|---|
| committer | CoprDistGit <infra@openeuler.org> | 2025-01-25 05:07:37 +0000 |
| commit | 579fd98117ef53cef3c2e6d9df36436f7503f896 (patch) | |
| tree | c908aaed1b7f92c2e5aaa55ac6981ebe00dfe9bf | |
| parent | ecab5ab593800179763b4e143ef193c651e6bcf6 (diff) | |
automatic import of iSuladopeneuler24.03_LTS_SP1
176 files changed, 51628 insertions, 6 deletions
diff --git a/0001-code-improve-for-sandbox.cc.patch b/0001-code-improve-for-sandbox.cc.patch new file mode 100644 index 0000000..f804203 --- /dev/null +++ b/0001-code-improve-for-sandbox.cc.patch @@ -0,0 +1,27 @@ +From 9497e03709a035805effd96eaa21f6c221a79e94 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Fri, 19 Jan 2024 17:12:30 +0800 +Subject: [PATCH 01/43] code improve for sandbox.cc + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/sandbox/sandbox.cc | 3 +-- + 1 file changed, 1 insertion(+), 2 deletions(-) + +diff --git a/src/daemon/sandbox/sandbox.cc b/src/daemon/sandbox/sandbox.cc +index 359cfbad..7b6496ed 100644 +--- a/src/daemon/sandbox/sandbox.cc ++++ b/src/daemon/sandbox/sandbox.cc +@@ -1086,8 +1086,7 @@ void Sandbox::FillSandboxMetadata(sandbox_metadata* metadata, Errors &error) + metadata->task_address = util_strdup_s(m_taskAddress.c_str()); + metadata->net_ns_path = util_strdup_s(m_netNsPath.c_str()); + +- google::protobuf::util::MessageToJsonString(*m_sandboxConfig.get(), &jsonStr); +- if (jsonStr.length() == 0) { ++ if (!google::protobuf::util::MessageToJsonString(*m_sandboxConfig.get(), &jsonStr).ok()) { + error.Errorf("Failed to get sandbox config json for sandbox: '%s'", m_id.c_str()); + ERROR("Failed to get sandbox config json for sandbox: '%s'", m_id.c_str()); + } +-- +2.34.1 + diff --git a/0002-fix-compile-error-with-protobuf-25.1-and-grpc-1.60.x.patch b/0002-fix-compile-error-with-protobuf-25.1-and-grpc-1.60.x.patch new file mode 100644 index 0000000..f43ce86 --- /dev/null +++ b/0002-fix-compile-error-with-protobuf-25.1-and-grpc-1.60.x.patch @@ -0,0 +1,255 @@ +From 71f8d4accbec5153b362281bbaf9a516ccd083f5 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Mon, 22 Jan 2024 15:55:16 +0800 +Subject: [PATCH 02/43] fix compile error with protobuf 25.1 and grpc 1.60.x + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + cmake/checker.cmake | 76 +++++++++++++++++++ + cmake/protoc.cmake | 3 - + src/CMakeLists.txt | 12 +-- + .../sandbox/controller/manager/CMakeLists.txt | 4 +- + .../sandboxer/async_wait_call/CMakeLists.txt | 4 +- + .../sandboxer/sandboxer_client/CMakeLists.txt | 4 +- + .../sandboxer_controller/CMakeLists.txt | 4 +- + test/sandbox/controller/shim/CMakeLists.txt | 4 +- + test/sandbox/sandbox/CMakeLists.txt | 4 +- + test/sandbox/sandbox_manager/CMakeLists.txt | 4 +- + 10 files changed, 87 insertions(+), 32 deletions(-) + +diff --git a/cmake/checker.cmake b/cmake/checker.cmake +index e19618e4..b0c395ef 100644 +--- a/cmake/checker.cmake ++++ b/cmake/checker.cmake +@@ -154,7 +154,83 @@ if (GRPC_CONNECTOR) + find_library(GPR_LIBRARY gpr) + _CHECK(GPR_LIBRARY "GPR_LIBRARY-NOTFOUND" "libgpr.so") + # no check ++ ++ # The use of absl libraries depends on the version of protobuf and grpc. ++ # Versions of protobuf before v22.0 do not require absl libraries at all. ++ # However, versions after v22.0 require the support of absl libraries. ++ # As a result, we skip the check for absl libraries in order to accommodate different protobuf and grpc versions. ++ set(ISULAD_ABSL_USED_TARGETS) + find_library(ABSL_SYNC_LIB absl_synchronization) ++ if (ABSL_SYNC_LIB) ++ set(ISULAD_ABSL_USED_TARGETS ++ ${ISULAD_ABSL_USED_TARGETS} ++ ${ABSL_SYNC_LIB} ++ ) ++ endif() ++ ++ find_library(ABSL_CORD_LIB absl_cord) ++ if (ABSL_CORD_LIB) ++ set(ISULAD_ABSL_USED_TARGETS ++ ${ISULAD_ABSL_USED_TARGETS} ++ ${ABSL_CORD_LIB} ++ ) ++ endif() ++ ++ find_library(ABSL_CORDZ_FUNCTIONS_LIB absl_cordz_functions) ++ if (ABSL_CORDZ_FUNCTIONS_LIB) ++ set(ISULAD_ABSL_USED_TARGETS ++ ${ISULAD_ABSL_USED_TARGETS} ++ ${ABSL_CORDZ_FUNCTIONS_LIB} ++ ) ++ endif() ++ ++ find_library(ABSL_CORDZ_INFO_LIB absl_cordz_info) ++ if (ABSL_CORDZ_INFO_LIB) ++ set(ISULAD_ABSL_USED_TARGETS ++ ${ISULAD_ABSL_USED_TARGETS} ++ ${ABSL_CORDZ_INFO_LIB} ++ ) ++ endif() ++ ++ find_library(ABSL_HASH_LIB absl_hash) ++ if (ABSL_HASH_LIB) ++ set(ISULAD_ABSL_USED_TARGETS ++ ${ISULAD_ABSL_USED_TARGETS} ++ ${ABSL_HASH_LIB} ++ ) ++ endif() ++ ++ find_library(ABSL_LOG_INTERNAL_CHECK_OP_LIB absl_log_internal_check_op) ++ if (ABSL_LOG_INTERNAL_CHECK_OP_LIB) ++ set(ISULAD_ABSL_USED_TARGETS ++ ${ISULAD_ABSL_USED_TARGETS} ++ ${ABSL_LOG_INTERNAL_CHECK_OP_LIB} ++ ) ++ endif() ++ ++ find_library(ABSL_LOG_INTERNAL_MESSAGE_LIB absl_log_internal_message) ++ if (ABSL_LOG_INTERNAL_MESSAGE_LIB) ++ set(ISULAD_ABSL_USED_TARGETS ++ ${ISULAD_ABSL_USED_TARGETS} ++ ${ABSL_LOG_INTERNAL_MESSAGE_LIB} ++ ) ++ endif() ++ ++ find_library(ABSL_LOG_INTERNAL_NULLGUARD_LIB absl_log_internal_nullguard) ++ if (ABSL_LOG_INTERNAL_NULLGUARD_LIB) ++ set(ISULAD_ABSL_USED_TARGETS ++ ${ISULAD_ABSL_USED_TARGETS} ++ ${ABSL_LOG_INTERNAL_NULLGUARD_LIB} ++ ) ++ endif() ++ ++ find_library(ABSL_STATUS_LIB absl_status) ++ if (ABSL_STATUS_LIB) ++ set(ISULAD_ABSL_USED_TARGETS ++ ${ISULAD_ABSL_USED_TARGETS} ++ ${ABSL_STATUS_LIB} ++ ) ++ endif() + + # check websocket + find_path(WEBSOCKET_INCLUDE_DIR libwebsockets.h) +diff --git a/cmake/protoc.cmake b/cmake/protoc.cmake +index 80c08687..6e2d1b84 100644 +--- a/cmake/protoc.cmake ++++ b/cmake/protoc.cmake +@@ -69,9 +69,6 @@ endif() + + if (ENABLE_CRI_API_V1 AND ENABLE_SANDBOXER) + execute_process(COMMAND mkdir -p ${SANDBOX_PROTOS_OUT_PATH}) +- PROTOC_CPP_GEN(sandbox ${SANDBOX_PROTOS_OUT_PATH} ${PROTOS_PATH}/sandbox/google/protobuf/any.proto) +- PROTOC_CPP_GEN(sandbox ${SANDBOX_PROTOS_OUT_PATH} ${PROTOS_PATH}/sandbox/google/protobuf/empty.proto) +- PROTOC_CPP_GEN(sandbox ${SANDBOX_PROTOS_OUT_PATH} ${PROTOS_PATH}/sandbox/google/protobuf/timestamp.proto) + PROTOC_CPP_GEN(sandbox ${SANDBOX_PROTOS_OUT_PATH} ${PROTOS_PATH}/sandbox/sandbox/types/sandbox.proto) + PROTOC_CPP_GEN(sandbox ${SANDBOX_PROTOS_OUT_PATH} ${PROTOS_PATH}/sandbox/sandbox/types/mount.proto) + PROTOC_CPP_GEN(sandbox ${SANDBOX_PROTOS_OUT_PATH} ${PROTOS_PATH}/sandbox/sandbox/types/platform.proto) +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index 860447de..d1bc65f9 100644 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -85,10 +85,8 @@ target_link_libraries(libisula_client libisulad_tools) + if (GRPC_CONNECTOR) + target_link_libraries(libisula_client -Wl,--as-needed -lstdc++) + target_link_libraries(libisula_client -Wl,--as-needed ${PROTOBUF_LIBRARY}) +- target_link_libraries(libisula_client -Wl,--no-as-needed ${GRPC_PP_REFLECTION_LIBRARY} ${GRPC_PP_LIBRARY} ${GRPC_LIBRARY} ${GPR_LIBRARY}) +- if(ABSL_SYNC_LIB) +- target_link_libraries(libisula_client -Wl,--no-as-needed ${ABSL_SYNC_LIB}) +- endif() ++ target_link_libraries(libisula_client -Wl,--as-needed ${GRPC_PP_REFLECTION_LIBRARY} ${GRPC_PP_LIBRARY} ${GRPC_LIBRARY} ${GPR_LIBRARY}) ++ target_link_libraries(libisula_client -Wl,--as-needed ${ISULAD_ABSL_USED_TARGETS}) + else() + target_link_libraries(libisula_client -ldl libhttpclient) + set_target_properties(libisula_client PROPERTIES LINKER_LANGUAGE "C") +@@ -182,13 +180,11 @@ endif() + if (GRPC_CONNECTOR) + target_link_libraries(isulad -Wl,--as-needed -lstdc++) + target_link_libraries(isulad -Wl,--as-needed ${WEBSOCKET_LIBRARY} ${PROTOBUF_LIBRARY}) +- target_link_libraries(isulad -Wl,--no-as-needed ${GRPC_PP_REFLECTION_LIBRARY} ${GRPC_PP_LIBRARY} ${GRPC_LIBRARY} ${GPR_LIBRARY}) ++ target_link_libraries(isulad -Wl,--as-needed ${GRPC_PP_REFLECTION_LIBRARY} ${GRPC_PP_LIBRARY} ${GRPC_LIBRARY} ${GPR_LIBRARY}) + if (ENABLE_METRICS) + target_link_libraries(isulad ${EVHTP_LIBRARY} ${EVENT_LIBRARY}) + endif() +- if(ABSL_SYNC_LIB) +- target_link_libraries(isulad -Wl,--no-as-needed ${ABSL_SYNC_LIB}) +- endif() ++ target_link_libraries(isulad -Wl,--as-needed ${ISULAD_ABSL_USED_TARGETS}) + else() + message("Restful iSulad") + target_link_libraries(isulad ${EVHTP_LIBRARY} ${EVENT_LIBRARY}) +diff --git a/test/sandbox/controller/manager/CMakeLists.txt b/test/sandbox/controller/manager/CMakeLists.txt +index 3724538e..6e8c9052 100644 +--- a/test/sandbox/controller/manager/CMakeLists.txt ++++ b/test/sandbox/controller/manager/CMakeLists.txt +@@ -40,8 +40,6 @@ target_include_directories(${EXE} PUBLIC + ) + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lgrpc -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) +-if(ABSL_SYNC_LIB) +- target_link_libraries(${EXE} -Wl,--no-as-needed ${ABSL_SYNC_LIB}) +-endif() ++target_link_libraries(${EXE} -Wl,--as-needed ${ISULAD_ABSL_USED_TARGETS}) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) + set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/sandbox/controller/sandboxer/async_wait_call/CMakeLists.txt b/test/sandbox/controller/sandboxer/async_wait_call/CMakeLists.txt +index c8eb803e..0631988a 100644 +--- a/test/sandbox/controller/sandboxer/async_wait_call/CMakeLists.txt ++++ b/test/sandbox/controller/sandboxer/async_wait_call/CMakeLists.txt +@@ -32,8 +32,6 @@ target_include_directories(${EXE} PUBLIC + ) + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lgrpc -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) +-if(ABSL_SYNC_LIB) +- target_link_libraries(${EXE} -Wl,--no-as-needed ${ABSL_SYNC_LIB}) +-endif() ++target_link_libraries(${EXE} -Wl,--as-needed ${ISULAD_ABSL_USED_TARGETS}) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) + set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/sandbox/controller/sandboxer/sandboxer_client/CMakeLists.txt b/test/sandbox/controller/sandboxer/sandboxer_client/CMakeLists.txt +index 91f26883..881797c6 100644 +--- a/test/sandbox/controller/sandboxer/sandboxer_client/CMakeLists.txt ++++ b/test/sandbox/controller/sandboxer/sandboxer_client/CMakeLists.txt +@@ -36,8 +36,6 @@ target_include_directories(${EXE} PUBLIC + ) + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lgrpc -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) +-if(ABSL_SYNC_LIB) +- target_link_libraries(${EXE} -Wl,--no-as-needed ${ABSL_SYNC_LIB}) +-endif() ++target_link_libraries(${EXE} -Wl,--as-needed ${ISULAD_ABSL_USED_TARGETS}) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) + set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/sandbox/controller/sandboxer/sandboxer_controller/CMakeLists.txt b/test/sandbox/controller/sandboxer/sandboxer_controller/CMakeLists.txt +index d38392e0..963ce9a5 100644 +--- a/test/sandbox/controller/sandboxer/sandboxer_controller/CMakeLists.txt ++++ b/test/sandbox/controller/sandboxer/sandboxer_controller/CMakeLists.txt +@@ -33,8 +33,6 @@ target_include_directories(${EXE} PUBLIC + ) + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lgrpc -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) +-if(ABSL_SYNC_LIB) +- target_link_libraries(${EXE} -Wl,--no-as-needed ${ABSL_SYNC_LIB}) +-endif() ++target_link_libraries(${EXE} -Wl,--as-needed ${ISULAD_ABSL_USED_TARGETS}) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) + set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/sandbox/controller/shim/CMakeLists.txt b/test/sandbox/controller/shim/CMakeLists.txt +index af066546..6423bb80 100644 +--- a/test/sandbox/controller/shim/CMakeLists.txt ++++ b/test/sandbox/controller/shim/CMakeLists.txt +@@ -50,8 +50,6 @@ target_include_directories(${EXE} PUBLIC + ) + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) +-if(ABSL_SYNC_LIB) +- target_link_libraries(${EXE} -Wl,--no-as-needed ${ABSL_SYNC_LIB}) +-endif() ++target_link_libraries(${EXE} -Wl,--as-needed ${ISULAD_ABSL_USED_TARGETS}) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) + set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/sandbox/sandbox/CMakeLists.txt b/test/sandbox/sandbox/CMakeLists.txt +index efcc2bdc..138d4d8d 100644 +--- a/test/sandbox/sandbox/CMakeLists.txt ++++ b/test/sandbox/sandbox/CMakeLists.txt +@@ -48,8 +48,6 @@ target_include_directories(${EXE} PUBLIC + ) + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lgrpc -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) +-if(ABSL_SYNC_LIB) +- target_link_libraries(${EXE} -Wl,--no-as-needed ${ABSL_SYNC_LIB}) +-endif() ++target_link_libraries(${EXE} -Wl,--as-needed ${ISULAD_ABSL_USED_TARGETS}) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) + set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/sandbox/sandbox_manager/CMakeLists.txt b/test/sandbox/sandbox_manager/CMakeLists.txt +index f43b0f97..5a7cb2ea 100644 +--- a/test/sandbox/sandbox_manager/CMakeLists.txt ++++ b/test/sandbox/sandbox_manager/CMakeLists.txt +@@ -48,8 +48,6 @@ target_include_directories(${EXE} PUBLIC + ) + set_target_properties(${EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,util_list_all_subdir") + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lgrpc -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) +-if(ABSL_SYNC_LIB) +- target_link_libraries(${EXE} -Wl,--no-as-needed ${ABSL_SYNC_LIB}) +-endif() ++target_link_libraries(${EXE} -Wl,--as-needed ${ISULAD_ABSL_USED_TARGETS}) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) + set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +-- +2.34.1 + diff --git a/0003-bugfix-for-mount-point-remains-under-special-circums.patch b/0003-bugfix-for-mount-point-remains-under-special-circums.patch new file mode 100644 index 0000000..4465d41 --- /dev/null +++ b/0003-bugfix-for-mount-point-remains-under-special-circums.patch @@ -0,0 +1,123 @@ +From cd018d3c1ebff2a328912d99fc43c9a7e4f60704 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Thu, 25 Jan 2024 11:24:59 +0800 +Subject: [PATCH 03/43] bugfix for mount point remains under special + circumstances + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/cmd/isulad/main.c | 14 +++++++------- + .../modules/container/leftover_cleanup/cleanup.c | 14 +++++++------- + src/utils/tar/util_archive.c | 14 +++++++------- + 3 files changed, 21 insertions(+), 21 deletions(-) + +diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c +index d9d8afa0..deca72be 100644 +--- a/src/cmd/isulad/main.c ++++ b/src/cmd/isulad/main.c +@@ -1271,23 +1271,23 @@ static int do_ensure_isulad_tmpdir_security(const char *isulad_tmp_dir) + char tmp_dir[PATH_MAX] = { 0 }; + char cleanpath[PATH_MAX] = { 0 }; + +- nret = snprintf(tmp_dir, PATH_MAX, "%s/isulad_tmpdir", isulad_tmp_dir); +- if (nret < 0 || (size_t)nret >= PATH_MAX) { +- ERROR("Failed to snprintf"); ++ if (realpath(isulad_tmp_dir, cleanpath) == NULL) { ++ ERROR("Failed to get real path for %s", tmp_dir); + return -1; + } + +- if (util_clean_path(tmp_dir, cleanpath, sizeof(cleanpath)) == NULL) { +- ERROR("Failed to clean path for %s", tmp_dir); ++ nret = snprintf(tmp_dir, PATH_MAX, "%s/isulad_tmpdir", cleanpath); ++ if (nret < 0 || (size_t)nret >= PATH_MAX) { ++ ERROR("Failed to snprintf"); + return -1; + } + +- if (isulad_tmpdir_security_check(cleanpath) == 0) { ++ if (isulad_tmpdir_security_check(tmp_dir) == 0) { + return 0; + } + + INFO("iSulad tmpdir: %s does not meet security requirements, recreate it", isulad_tmp_dir); +- return recreate_tmpdir(cleanpath); ++ return recreate_tmpdir(tmp_dir); + } + + static int ensure_isulad_tmpdir_security() +diff --git a/src/daemon/modules/container/leftover_cleanup/cleanup.c b/src/daemon/modules/container/leftover_cleanup/cleanup.c +index b78a4d15..08151f42 100644 +--- a/src/daemon/modules/container/leftover_cleanup/cleanup.c ++++ b/src/daemon/modules/container/leftover_cleanup/cleanup.c +@@ -175,22 +175,22 @@ static void cleanup_path(char *dir) + char tmp_dir[PATH_MAX] = { 0 }; + char cleanpath[PATH_MAX] = { 0 }; + +- nret = snprintf(tmp_dir, PATH_MAX, "%s/isulad_tmpdir", dir); +- if (nret < 0 || (size_t)nret >= PATH_MAX) { +- ERROR("Failed to snprintf"); ++ if (realpath(dir, cleanpath) == NULL) { ++ ERROR("get real path for %s failed", tmp_dir); + return; + } + +- if (util_clean_path(tmp_dir, cleanpath, sizeof(cleanpath)) == NULL) { +- ERROR("clean path for %s failed", tmp_dir); ++ nret = snprintf(tmp_dir, PATH_MAX, "%s/isulad_tmpdir", cleanpath); ++ if (nret < 0 || (size_t)nret >= PATH_MAX) { ++ ERROR("Failed to snprintf"); + return; + } + +- if (!util_dir_exists(cleanpath)) { ++ if (!util_dir_exists(tmp_dir)) { + return; + } + +- nret = util_scan_subdirs(cleanpath, walk_isulad_tmpdir_cb, NULL); ++ nret = util_scan_subdirs(tmp_dir, walk_isulad_tmpdir_cb, NULL); + if (nret != 0) { + ERROR("failed to scan isulad tmp subdirs"); + } +diff --git a/src/utils/tar/util_archive.c b/src/utils/tar/util_archive.c +index 0a7309c9..e4c302bc 100644 +--- a/src/utils/tar/util_archive.c ++++ b/src/utils/tar/util_archive.c +@@ -218,18 +218,18 @@ static int make_safedir_is_noexec(const char *flock_path, const char *dstdir, ch + isulad_tmpdir_env = DEFAULT_ISULAD_TMPDIR; + } + +- nret = snprintf(isula_tmpdir, PATH_MAX, "%s/isulad_tmpdir", isulad_tmpdir_env); +- if (nret < 0 || (size_t)nret >= PATH_MAX) { +- ERROR("Failed to snprintf"); ++ if (realpath(isulad_tmpdir_env, cleanpath) == NULL) { ++ ERROR("Failed to get real path for %s", isula_tmpdir); + return -1; + } + +- if (util_clean_path(isula_tmpdir, cleanpath, sizeof(cleanpath)) == NULL) { +- ERROR("clean path for %s failed", isula_tmpdir); ++ nret = snprintf(isula_tmpdir, PATH_MAX, "%s/isulad_tmpdir", cleanpath); ++ if (nret < 0 || (size_t)nret >= PATH_MAX) { ++ ERROR("Failed to snprintf"); + return -1; + } + +- nret = snprintf(tmp_dir, PATH_MAX, "%s/tar-chroot-XXXXXX", cleanpath); ++ nret = snprintf(tmp_dir, PATH_MAX, "%s/tar-chroot-XXXXXX", isula_tmpdir); + if (nret < 0 || (size_t)nret >= PATH_MAX) { + ERROR("Failed to snprintf string"); + return -1; +@@ -247,7 +247,7 @@ static int make_safedir_is_noexec(const char *flock_path, const char *dstdir, ch + } + + // ensure parent dir is exist +- if (util_mkdir_p(cleanpath, ISULAD_TEMP_DIRECTORY_MODE) != 0) { ++ if (util_mkdir_p(isula_tmpdir, ISULAD_TEMP_DIRECTORY_MODE) != 0) { + return -1; + } + +-- +2.34.1 + diff --git a/0004-do-not-cleanup-if-the-directory-does-not-exist.patch b/0004-do-not-cleanup-if-the-directory-does-not-exist.patch new file mode 100644 index 0000000..d643d8e --- /dev/null +++ b/0004-do-not-cleanup-if-the-directory-does-not-exist.patch @@ -0,0 +1,71 @@ +From 7f13d95572040d30b70edbfac3c4b7350ee8855c Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Fri, 26 Jan 2024 12:59:45 +0800 +Subject: [PATCH 04/43] do not cleanup if the directory does not exist + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/cmd/isulad/main.c | 20 ++++++++++++++++++- + .../container/leftover_cleanup/cleanup.c | 13 +++++++++++- + 2 files changed, 31 insertions(+), 2 deletions(-) + +diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c +index deca72be..fd0b6e89 100644 +--- a/src/cmd/isulad/main.c ++++ b/src/cmd/isulad/main.c +@@ -1270,8 +1270,26 @@ static int do_ensure_isulad_tmpdir_security(const char *isulad_tmp_dir) + int nret; + char tmp_dir[PATH_MAX] = { 0 }; + char cleanpath[PATH_MAX] = { 0 }; ++ char isulad_tmp_cleanpath[PATH_MAX] = { 0 }; + +- if (realpath(isulad_tmp_dir, cleanpath) == NULL) { ++ if (util_clean_path(isulad_tmp_dir, isulad_tmp_cleanpath, sizeof(isulad_tmp_cleanpath)) == NULL) { ++ ERROR("Failed to clean path for %s", isulad_tmp_dir); ++ return -1; ++ } ++ ++ // Determine whether isulad_tmp_dir exists. If it does not exist, create it ++ // to prevent realpath from reporting errors because the folder does not exist. ++ if (!util_dir_exists(isulad_tmp_cleanpath)) { ++ nret = snprintf(tmp_dir, PATH_MAX, "%s/isulad_tmpdir", isulad_tmp_cleanpath); ++ if (nret < 0 || (size_t)nret >= PATH_MAX) { ++ ERROR("Failed to snprintf"); ++ return -1; ++ } ++ INFO("iSulad tmpdir: %s does not exist, create it", isulad_tmp_dir); ++ return recreate_tmpdir(tmp_dir); ++ } ++ ++ if (realpath(isulad_tmp_cleanpath, cleanpath) == NULL) { + ERROR("Failed to get real path for %s", tmp_dir); + return -1; + } +diff --git a/src/daemon/modules/container/leftover_cleanup/cleanup.c b/src/daemon/modules/container/leftover_cleanup/cleanup.c +index 08151f42..16dba630 100644 +--- a/src/daemon/modules/container/leftover_cleanup/cleanup.c ++++ b/src/daemon/modules/container/leftover_cleanup/cleanup.c +@@ -174,8 +174,19 @@ static void cleanup_path(char *dir) + int nret; + char tmp_dir[PATH_MAX] = { 0 }; + char cleanpath[PATH_MAX] = { 0 }; ++ char dir_cleanpath[PATH_MAX] = { 0 }; + +- if (realpath(dir, cleanpath) == NULL) { ++ if (util_clean_path(dir, dir_cleanpath, sizeof(dir_cleanpath)) == NULL) { ++ ERROR("clean path for %s failed", dir); ++ return; ++ } ++ ++ // If dir does not exist, skip cleanup ++ if (!util_dir_exists(dir_cleanpath)) { ++ return; ++ } ++ ++ if (realpath(dir_cleanpath, cleanpath) == NULL) { + ERROR("get real path for %s failed", tmp_dir); + return; + } +-- +2.34.1 + diff --git a/0005-module-only-deletes-the-temporary-files-it-creates.patch b/0005-module-only-deletes-the-temporary-files-it-creates.patch new file mode 100644 index 0000000..50e6515 --- /dev/null +++ b/0005-module-only-deletes-the-temporary-files-it-creates.patch @@ -0,0 +1,151 @@ +From 69dcd191afbdea5a178fb96a21e28537c2fc6a75 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Sat, 27 Jan 2024 11:16:37 +0800 +Subject: [PATCH 05/43] module only deletes the temporary files it creates + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/cmd/isulad/main.c | 2 +- + src/common/constants.h | 2 +- + src/daemon/modules/image/oci/oci_image.c | 44 +++++++++++++++++-- + src/daemon/modules/image/oci/oci_image.h | 4 ++ + src/daemon/modules/image/oci/oci_load.c | 2 +- + .../modules/image/oci/registry/registry.c | 2 +- + 6 files changed, 48 insertions(+), 8 deletions(-) + +diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c +index fd0b6e89..7c0c072e 100644 +--- a/src/cmd/isulad/main.c ++++ b/src/cmd/isulad/main.c +@@ -1252,7 +1252,7 @@ static int isulad_tmpdir_security_check(const char *tmp_dir) + + static int recreate_tmpdir(const char *tmp_dir) + { +- if (util_recursive_rmdir(tmp_dir, 0) != 0) { ++ if (util_path_remove(tmp_dir) != 0) { + ERROR("Failed to remove directory %s", tmp_dir); + return -1; + } +diff --git a/src/common/constants.h b/src/common/constants.h +index 27d4956e..8a6f86d8 100644 +--- a/src/common/constants.h ++++ b/src/common/constants.h +@@ -50,7 +50,7 @@ extern "C" { + + #define TEMP_DIRECTORY_MODE 0700 + +-#define ISULAD_TEMP_DIRECTORY_MODE 0600 ++#define ISULAD_TEMP_DIRECTORY_MODE 0700 + + #define CONSOLE_FIFO_DIRECTORY_MODE 0770 + +diff --git a/src/daemon/modules/image/oci/oci_image.c b/src/daemon/modules/image/oci/oci_image.c +index 9cf2cd4f..ce1c8a6b 100644 +--- a/src/daemon/modules/image/oci/oci_image.c ++++ b/src/daemon/modules/image/oci/oci_image.c +@@ -283,10 +283,42 @@ out: + return ret; + } + ++// remove dir that image module created ++// return false when failed to rmdir ++// eg: oci-image-load-XXXXXX && registry-XXXXXX ++static bool remove_image_tmpdir_cb(const char *path_name, const struct dirent *sub_dir, void *context) ++{ ++ int nret = 0; ++ char tmpdir[PATH_MAX] = { 0 }; ++ ++ if (sub_dir == NULL) { ++ return true; ++ } ++ ++ if (!util_has_prefix(sub_dir->d_name, LOAD_TMPDIR_PREFIX) && !util_has_prefix(sub_dir->d_name, REGISTRY_TMPDIR_PREFIX)) { ++ // only remove directory that image module created ++ return true; ++ } ++ ++ nret = snprintf(tmpdir, PATH_MAX, "%s/%s", path_name, sub_dir->d_name); ++ if (nret < 0 || (size_t)nret >= PATH_MAX) { ++ ERROR("Failed to snprintf for %s", sub_dir->d_name); ++ return false; ++ } ++ ++ if (util_recursive_rmdir(tmpdir, 0) != 0) { ++ ERROR("Failed to remove path %s", tmpdir); ++ return false; ++ } ++ ++ return true; ++} ++ + static int recreate_image_tmpdir() + { + char *image_tmp_path = NULL; + int ret = 0; ++ int nret = 0; + + image_tmp_path = oci_get_isulad_tmpdir(g_oci_image_module_data.root_dir); + if (image_tmp_path == NULL) { +@@ -295,10 +327,14 @@ static int recreate_image_tmpdir() + goto out; + } + +- if (util_recursive_rmdir(image_tmp_path, 0)) { +- ERROR("failed to remove directory %s", image_tmp_path); +- ret = -1; +- goto out; ++ // If image_tmp_path exist, cleanup it ++ if (util_dir_exists(image_tmp_path)) { ++ nret = util_scan_subdirs(image_tmp_path, remove_image_tmpdir_cb, NULL); ++ if (nret != 0) { ++ ERROR("Failed to scan isulad tmp subdirs"); ++ ret = -1; ++ goto out; ++ } + } + + if (util_mkdir_p(image_tmp_path, TEMP_DIRECTORY_MODE)) { +diff --git a/src/daemon/modules/image/oci/oci_image.h b/src/daemon/modules/image/oci/oci_image.h +index c7304897..482091d6 100644 +--- a/src/daemon/modules/image/oci/oci_image.h ++++ b/src/daemon/modules/image/oci/oci_image.h +@@ -38,6 +38,10 @@ struct oci_image_module_data { + char **insecure_registries; + size_t insecure_registries_len; + }; ++ ++#define LOAD_TMPDIR_PREFIX "oci-image-load-" ++#define REGISTRY_TMPDIR_PREFIX "registry-" ++ + struct oci_image_module_data *get_oci_image_data(void); + + int oci_init(const isulad_daemon_configs *args); +diff --git a/src/daemon/modules/image/oci/oci_load.c b/src/daemon/modules/image/oci/oci_load.c +index 31ae3849..534e2647 100644 +--- a/src/daemon/modules/image/oci/oci_load.c ++++ b/src/daemon/modules/image/oci/oci_load.c +@@ -1048,7 +1048,7 @@ static char *oci_load_path_create() + goto out; + } + +- nret = snprintf(tmp_dir, PATH_MAX, "%s/oci-image-load-XXXXXX", image_tmp_path); ++ nret = snprintf(tmp_dir, PATH_MAX, "%s/%sXXXXXX", image_tmp_path, LOAD_TMPDIR_PREFIX); + if (nret < 0 || (size_t)nret >= sizeof(tmp_dir)) { + ERROR("Path is too long"); + ret = -1; +diff --git a/src/daemon/modules/image/oci/registry/registry.c b/src/daemon/modules/image/oci/registry/registry.c +index 751a8727..aed3057a 100644 +--- a/src/daemon/modules/image/oci/registry/registry.c ++++ b/src/daemon/modules/image/oci/registry/registry.c +@@ -1908,7 +1908,7 @@ static int prepare_pull_desc(pull_descriptor *desc, registry_pull_options *optio + goto out; + } + +- sret = snprintf(blobpath, PATH_MAX, "%s/registry-XXXXXX", image_tmp_path); ++ sret = snprintf(blobpath, PATH_MAX, "%s/%sXXXXXX", image_tmp_path, REGISTRY_TMPDIR_PREFIX); + if (sret < 0 || (size_t)sret >= PATH_MAX) { + ERROR("image tmp work path too long"); + ret = -1; +-- +2.34.1 + diff --git a/0006-skip-devmapper-ut.patch b/0006-skip-devmapper-ut.patch new file mode 100644 index 0000000..912b3c7 --- /dev/null +++ b/0006-skip-devmapper-ut.patch @@ -0,0 +1,26 @@ +From b290e7fb553c5cc6746c9dcfe4896098f74bc7d7 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Tue, 30 Jan 2024 12:35:58 +0800 +Subject: [PATCH 06/43] skip devmapper ut + +Signed-off-by: jikai <jikai11@huawei.com> +--- + CI/make-and-install.sh | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh +index c1d26ff1..9bb984cd 100755 +--- a/CI/make-and-install.sh ++++ b/CI/make-and-install.sh +@@ -75,7 +75,7 @@ mkdir build && cd build + cmake -DCMAKE_BUILD_TYPE=Debug -DENABLE_UT=ON -DENABLE_CRI_API_V1=ON -DENABLE_SHIM_V2=ON -DENABLE_METRICS=ON .. + make -j $(nproc) + make install +-ctest -T memcheck --output-on-failure ++ctest -E "driver_devmapper_ut" -T memcheck --output-on-failure + echo_success "===================RUN DT-LLT TESTCASES END=========================" + + # build fuzz +-- +2.34.1 + diff --git a/0007-update-annotations-and-add-ci-cases.patch b/0007-update-annotations-and-add-ci-cases.patch new file mode 100644 index 0000000..07b857e --- /dev/null +++ b/0007-update-annotations-and-add-ci-cases.patch @@ -0,0 +1,174 @@ +From ed4b71b2027a6e9fdf15931fe93aa9e0bb3dc79d Mon Sep 17 00:00:00 2001 +From: leizhongkai <leizhongkai@huawei.com> +Date: Wed, 31 Jan 2024 18:17:52 +0800 +Subject: [PATCH 07/43] update annotations and add ci cases + +Signed-off-by: leizhongkai <leizhongkai@huawei.com> +--- + .../container_cases/dev_cgroup_rule.sh | 24 +++++++++++ + src/daemon/modules/api/specs_api.h | 2 + + .../modules/service/service_container.c | 18 +++++++- + src/daemon/modules/spec/specs.c | 41 ++++++++++++++++++- + 4 files changed, 82 insertions(+), 3 deletions(-) + +diff --git a/CI/test_cases/container_cases/dev_cgroup_rule.sh b/CI/test_cases/container_cases/dev_cgroup_rule.sh +index 839a546c..5616d37a 100755 +--- a/CI/test_cases/container_cases/dev_cgroup_rule.sh ++++ b/CI/test_cases/container_cases/dev_cgroup_rule.sh +@@ -29,6 +29,9 @@ function test_cpu_dev_cgoup_rule_spec() + local image="busybox" + local test="container device cgroup rule test with (${runtime}) => (${FUNCNAME[@]})" + local test_dev="/dev/testA" ++ local default_config="/etc/default/isulad/config.json" ++ local default_config_bak="/etc/default/isulad/config.json.bak" ++ local test_cgroup_parent="/testABC" + + msg_info "${test} starting..." + +@@ -54,6 +57,27 @@ function test_cpu_dev_cgoup_rule_spec() + [[ $? -ne 0 ]] && [[ $cnt -le $priv_minor_88_cnt ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check device minor failed" && ((ret++)) + isula rm -f $priv_cid + ++ def_cid=$(isula run -tid --runtime $runtime -m 10m $image /bin/sh) ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - run container failed" && ((ret++)) ++ cp $default_config $default_config_bak ++ sed -i '/"linux": {/a \ \t\t"devices": [\n\t\t{\n\t\t\t"type": "c",\n\t\t\t"path": "\/dev\/testABC",\n\t\t\t"major": 88,\n\t\t\t"minor": 88\n\t\t}\n\t\t],' $default_config ++ stop_isulad_without_valgrind ++ start_isulad_with_valgrind --cgroup-parent $test_cgroup_parent ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start isulad failed" && ((ret++)) ++ isula restart -t 0 $def_cid ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - restart container failed" && ((ret++)) ++ cat /sys/fs/cgroup/memory/$test_cgroup_parent/$def_cid/memory.limit_in_bytes | grep ^10485760$ ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - --cgroup-parent cannot work" && ((ret++)) ++ cnt=$(cat ${RUNTIME_ROOT_PATH}/${runtime}/$def_cid/config.json | grep "major\": 88" | wc -l) ++ [[ $? -ne 0 ]]&& [[ $cnt -ne 2 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check device major failed" && ((ret++)) ++ cnt=$(cat ${RUNTIME_ROOT_PATH}/${runtime}/$def_cid/config.json | grep "minor\": 88" | wc -l) ++ [[ $? -ne 0 ]] && [[ $cnt -ne 2 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check device minor failed" && ((ret++)) ++ isula rm -f $def_cid ++ cp $default_config_bak $default_config ++ stop_isulad_without_valgrind ++ start_isulad_with_valgrind ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start isulad failed" && ((ret++)) ++ + cid=$(isula run -tid --device "$test_dev:$test_dev" --runtime $runtime $image /bin/sh) + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - run container failed" && ((ret++)) + cnt=$(cat ${RUNTIME_ROOT_PATH}/${runtime}/$cid/config.json | grep "major\": 88" | wc -l) +diff --git a/src/daemon/modules/api/specs_api.h b/src/daemon/modules/api/specs_api.h +index 7c904614..f5f6ad8b 100644 +--- a/src/daemon/modules/api/specs_api.h ++++ b/src/daemon/modules/api/specs_api.h +@@ -41,6 +41,8 @@ int merge_share_namespace(oci_runtime_spec *oci_spec, const host_config *host_sp + const container_config_v2_common_config *v2_spec, + const container_network_settings *network_settings); + ++int update_spec_annotations(oci_runtime_spec *oci_spec, container_config *container_spec, host_config *host_spec); ++ + oci_runtime_spec *load_oci_config(const char *rootpath, const char *name); + + oci_runtime_spec *default_spec(bool system_container); +diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c +index 97f73768..239783b8 100644 +--- a/src/daemon/modules/service/service_container.c ++++ b/src/daemon/modules/service/service_container.c +@@ -691,11 +691,18 @@ out: + epoll_loop_close(&descr); + } + +-static int do_oci_spec_update(const char *id, oci_runtime_spec *oci_spec, host_config *hostconfig) ++static int do_oci_spec_update(const char *id, oci_runtime_spec *oci_spec, container_config *container_spec, host_config *hostconfig) + { + __isula_auto_free char *cgroup_parent = NULL; + int ret; + ++ // First renew annotations for oci spec, cgroup path, rootfs.mount, native.mask ++ // for iSulad daemon might get updated ++ ret = update_spec_annotations(oci_spec, container_spec, hostconfig); ++ if (ret < 0) { ++ return -1; ++ } ++ + // If isulad daemon cgroup parent updated, we should update this config into oci spec + cgroup_parent = merge_container_cgroups_path(id, hostconfig); + if (cgroup_parent == NULL) { +@@ -802,13 +809,20 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo + } + + // Update possible changes +- nret = do_oci_spec_update(id, oci_spec, cont->hostconfig); ++ nret = do_oci_spec_update(id, oci_spec, cont->common_config->config, cont->hostconfig); + if (nret != 0) { + ERROR("Failed to update possible changes for oci spec"); + ret = -1; + goto close_exit_fd; + } + ++ nret = container_to_disk(cont); ++ if (nret != 0) { ++ ERROR("Failed to save container info to disk"); ++ ret = -1; ++ goto close_exit_fd; ++ } ++ + nret = setup_ipc_dirs(cont->hostconfig, cont->common_config); + if (nret != 0) { + ERROR("Failed to setup ipc dirs"); +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index cc49d85f..62e340b1 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -385,6 +385,44 @@ out: + return ret; + } + ++int update_spec_annotations(oci_runtime_spec *oci_spec, container_config *container_spec, host_config *host_spec) ++{ ++ int ret = 0; ++ if (oci_spec == NULL || container_spec == NULL || host_spec == NULL) { ++ return -1; ++ } ++ ++ ret = make_sure_container_spec_annotations(container_spec); ++ if (ret < 0) { ++ return -1; ++ } ++ ++ ret = make_annotations_cgroup_dir(container_spec, host_spec); ++ if (ret != 0) { ++ return -1; ++ } ++ ++ /* add rootfs.mount */ ++ ret = add_rootfs_mount(container_spec); ++ if (ret != 0) { ++ ERROR("Failed to add rootfs mount"); ++ return -1; ++ } ++ ++ /* add native.umask */ ++ ret = add_native_umask(container_spec); ++ if (ret != 0) { ++ ERROR("Failed to add native umask"); ++ return -1; ++ } ++ ++ if (merge_annotations(oci_spec, container_spec)) { ++ return -1; ++ } ++ ++ return 0; ++} ++ + static int make_sure_oci_spec_root(oci_runtime_spec *oci_spec) + { + if (oci_spec->root == NULL) { +@@ -2501,4 +2539,5 @@ int spec_module_init(void) + return -1; + } + return 0; +-} +\ No newline at end of file ++} ++ +-- +2.34.1 + diff --git a/0008-bug-fix-for-device-cgroup-ulimt-oci-update.patch b/0008-bug-fix-for-device-cgroup-ulimt-oci-update.patch new file mode 100644 index 0000000..d433c46 --- /dev/null +++ b/0008-bug-fix-for-device-cgroup-ulimt-oci-update.patch @@ -0,0 +1,184 @@ +From fe3413bb8ebae90f29ce3cc02373f3fc2b5d2fd2 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Mon, 22 Jan 2024 20:19:29 +0800 +Subject: [PATCH 08/43] bug fix for device/cgroup/ulimt oci update + +Signed-off-by: jikai <jikai11@huawei.com> +--- + .../executor/container_cb/execution_create.c | 7 ++- + src/daemon/modules/api/specs_api.h | 4 ++ + .../modules/service/service_container.c | 18 +++--- + src/daemon/modules/spec/specs.c | 60 +++++++++++++++---- + 4 files changed, 63 insertions(+), 26 deletions(-) + +diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c +index ca2a9163..e00afb68 100644 +--- a/src/daemon/executor/container_cb/execution_create.c ++++ b/src/daemon/executor/container_cb/execution_create.c +@@ -533,12 +533,15 @@ static int merge_config_for_syscontainer(const container_create_request *request + value = request->rootfs; + } + +- if (append_json_map_string_string(oci_spec->annotations, "rootfs.mount", value)) { ++ // should also update to container spec ++ if (append_json_map_string_string(container_spec->annotations, "rootfs.mount", value) ++ || append_json_map_string_string(oci_spec->annotations, "rootfs.mount", value)) { + ERROR("Realloc annotations failed"); + ret = -1; + goto out; + } +- if (request->rootfs != NULL && append_json_map_string_string(oci_spec->annotations, "external.rootfs", "true")) { ++ if (request->rootfs != NULL && (append_json_map_string_string(container_spec->annotations, "external.rootfs", "true") ++ || append_json_map_string_string(oci_spec->annotations, "external.rootfs", "true"))) { + ERROR("Realloc annotations failed"); + ret = -1; + goto out; +diff --git a/src/daemon/modules/api/specs_api.h b/src/daemon/modules/api/specs_api.h +index f5f6ad8b..f54c0d31 100644 +--- a/src/daemon/modules/api/specs_api.h ++++ b/src/daemon/modules/api/specs_api.h +@@ -47,6 +47,10 @@ oci_runtime_spec *load_oci_config(const char *rootpath, const char *name); + + oci_runtime_spec *default_spec(bool system_container); + ++int update_oci_container_cgroups_path(const char *id, oci_runtime_spec *oci_spec, const host_config *host_spec); ++ ++int update_oci_ulimit(oci_runtime_spec *oci_spec, const host_config *host_spec); ++ + const oci_runtime_spec *get_readonly_default_oci_spec(bool system_container); + + int spec_module_init(void); +diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c +index 239783b8..a3606a82 100644 +--- a/src/daemon/modules/service/service_container.c ++++ b/src/daemon/modules/service/service_container.c +@@ -693,26 +693,21 @@ out: + + static int do_oci_spec_update(const char *id, oci_runtime_spec *oci_spec, container_config *container_spec, host_config *hostconfig) + { +- __isula_auto_free char *cgroup_parent = NULL; + int ret; + +- // First renew annotations for oci spec, cgroup path, rootfs.mount, native.mask +- // for iSulad daemon might get updated ++ // Renew annotations for oci spec, cgroup path only, ++ // since lxc uses the "cgroup.dir" in oci annotations to create cgroup ++ // should ensure that container spec has the same annotations as oci spec + ret = update_spec_annotations(oci_spec, container_spec, hostconfig); + if (ret < 0) { + return -1; + } + + // If isulad daemon cgroup parent updated, we should update this config into oci spec +- cgroup_parent = merge_container_cgroups_path(id, hostconfig); +- if (cgroup_parent == NULL) { ++ ret = update_oci_container_cgroups_path(id, oci_spec, hostconfig); ++ if (ret < 0) { + return -1; + } +- if (oci_spec->linux->cgroups_path != NULL && strcmp(oci_spec->linux->cgroups_path, cgroup_parent) != 0) { +- free(oci_spec->linux->cgroups_path); +- oci_spec->linux->cgroups_path = cgroup_parent; +- cgroup_parent = NULL; +- } + + // For Linux.Resources, isula update will save changes into oci spec; + // so we just skip it; +@@ -725,7 +720,8 @@ static int do_oci_spec_update(const char *id, oci_runtime_spec *oci_spec, contai + } + + // If isulad daemon ulimit updated, we should update this config into oci spec. +- if (merge_global_ulimit(oci_spec) != 0) { ++ ret = update_oci_ulimit(oci_spec, hostconfig); ++ if (ret < 0) { + return -1; + } + +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index 62e340b1..464b4fb4 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -402,19 +402,8 @@ int update_spec_annotations(oci_runtime_spec *oci_spec, container_config *contai + return -1; + } + +- /* add rootfs.mount */ +- ret = add_rootfs_mount(container_spec); +- if (ret != 0) { +- ERROR("Failed to add rootfs mount"); +- return -1; +- } +- +- /* add native.umask */ +- ret = add_native_umask(container_spec); +- if (ret != 0) { +- ERROR("Failed to add native umask"); +- return -1; +- } ++ // other annotations will either not be updated after containers created ++ // or for rootfs mnt and umask, we do not support the update operation + + if (merge_annotations(oci_spec, container_spec)) { + return -1; +@@ -2302,6 +2291,27 @@ char *merge_container_cgroups_path(const char *id, const host_config *host_spec) + return util_path_join(path, id); + } + ++int update_oci_container_cgroups_path(const char *id, oci_runtime_spec *oci_spec, const host_config *hostconfig) ++{ ++ if (oci_spec == NULL || oci_spec->linux == NULL) { ++ ERROR("Invalid arguments"); ++ return -1; ++ } ++ ++ __isula_auto_free char *cgroup_parent = merge_container_cgroups_path(id, hostconfig); ++ if (cgroup_parent == NULL) { ++ return -1; ++ } ++ ++ if (oci_spec->linux->cgroups_path != NULL && strcmp(oci_spec->linux->cgroups_path, cgroup_parent) != 0) { ++ free(oci_spec->linux->cgroups_path); ++ oci_spec->linux->cgroups_path = cgroup_parent; ++ cgroup_parent = NULL; ++ } ++ ++ return 0; ++} ++ + static int merge_oci_cgroups_path(const char *id, oci_runtime_spec *oci_spec, const host_config *host_spec) + { + if (id == NULL || oci_spec == NULL || host_spec == NULL) { +@@ -2445,6 +2455,30 @@ out: + return ret; + } + ++int update_oci_ulimit(oci_runtime_spec *oci_spec, const host_config *hostconfig) { ++ if (oci_spec == NULL || hostconfig == NULL) { ++ ERROR("Invalid arguments"); ++ return -1; ++ } ++ ++ size_t i = 0; ++ if (oci_spec->process != NULL) { ++ for (i = 0; i < oci_spec->process->rlimits_len; i++) { ++ free_defs_process_rlimits_element(oci_spec->process->rlimits[i]); ++ oci_spec->process->rlimits[i] = NULL; ++ } ++ free(oci_spec->process->rlimits); ++ oci_spec->process->rlimits = NULL; ++ oci_spec->process->rlimits_len = 0; ++ } ++ ++ if (merge_conf_ulimits(oci_spec, hostconfig) != 0 || merge_global_ulimit(oci_spec) != 0) { ++ return -1; ++ } ++ ++ return 0; ++} ++ + /* read oci config */ + oci_runtime_spec *load_oci_config(const char *rootpath, const char *name) + { +-- +2.34.1 + diff --git a/0009-improve-dt-for-oci-spec-update.patch b/0009-improve-dt-for-oci-spec-update.patch new file mode 100644 index 0000000..07c4b82 --- /dev/null +++ b/0009-improve-dt-for-oci-spec-update.patch @@ -0,0 +1,281 @@ +From 82dd5a1db70fdb3f4934a3f9c0ee290ce5bee1b2 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Sat, 27 Jan 2024 15:30:05 +0800 +Subject: [PATCH 09/43] improve dt for oci spec update + +Signed-off-by: jikai <jikai11@huawei.com> +--- + .../container_cases/dev_cgroup_rule.sh | 4 +- + CI/test_cases/container_cases/ulimit.sh | 19 ++ + test/specs/specs/CMakeLists.txt | 1 + + test/specs/specs/specs_ut.cc | 168 ++++++++++++++++++ + 4 files changed, 190 insertions(+), 2 deletions(-) + +diff --git a/CI/test_cases/container_cases/dev_cgroup_rule.sh b/CI/test_cases/container_cases/dev_cgroup_rule.sh +index 5616d37a..33a839c5 100755 +--- a/CI/test_cases/container_cases/dev_cgroup_rule.sh ++++ b/CI/test_cases/container_cases/dev_cgroup_rule.sh +@@ -60,13 +60,13 @@ function test_cpu_dev_cgoup_rule_spec() + def_cid=$(isula run -tid --runtime $runtime -m 10m $image /bin/sh) + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - run container failed" && ((ret++)) + cp $default_config $default_config_bak +- sed -i '/"linux": {/a \ \t\t"devices": [\n\t\t{\n\t\t\t"type": "c",\n\t\t\t"path": "\/dev\/testABC",\n\t\t\t"major": 88,\n\t\t\t"minor": 88\n\t\t}\n\t\t],' $default_config ++ sed -i '/"linux": {/a \ \t\t"devices": [\n\t\t{\n\t\t\t"type": "c",\n\t\t\t"path": "\/dev\/testA",\n\t\t\t"major": 88,\n\t\t\t"minor": 88\n\t\t}\n\t\t],' $default_config + stop_isulad_without_valgrind + start_isulad_with_valgrind --cgroup-parent $test_cgroup_parent + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start isulad failed" && ((ret++)) + isula restart -t 0 $def_cid + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - restart container failed" && ((ret++)) +- cat /sys/fs/cgroup/memory/$test_cgroup_parent/$def_cid/memory.limit_in_bytes | grep ^10485760$ ++ isula exec -it $def_cid sh -c "cat /sys/fs/cgroup/memory/memory.limit_in_bytes | grep ^10485760$" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - --cgroup-parent cannot work" && ((ret++)) + cnt=$(cat ${RUNTIME_ROOT_PATH}/${runtime}/$def_cid/config.json | grep "major\": 88" | wc -l) + [[ $? -ne 0 ]]&& [[ $cnt -ne 2 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check device major failed" && ((ret++)) +diff --git a/CI/test_cases/container_cases/ulimit.sh b/CI/test_cases/container_cases/ulimit.sh +index f823dc1c..41cdcece 100755 +--- a/CI/test_cases/container_cases/ulimit.sh ++++ b/CI/test_cases/container_cases/ulimit.sh +@@ -49,9 +49,28 @@ function test_ulimit() + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - restart failed" && ((ret++)) + cat ${RUNTIME_ROOT_PATH}/${runtime}/$cid/config.json | grep "RLIMIT_" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check rlimit failed after restart" && ((ret++)) ++ ++ check_valgrind_log ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop isulad failed" && ((ret++)) ++ ++ start_isulad_with_valgrind ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start isulad failed" && ((ret++)) ++ ++ # if default ulimit of isulad changed, isula start should do update ulimit of oci spec ++ isula restart -t 0 $cid ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - restart failed" && ((ret++)) ++ cat ${RUNTIME_ROOT_PATH}/${runtime}/$cid/config.json | grep "RLIMIT_" ++ [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check rlimit failed after restart" && ((ret++)) ++ + isula rm -f $cid + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - rm container failed" && ((ret++)) + ++ check_valgrind_log ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop isulad failed" && ((ret++)) ++ ++ start_isulad_with_valgrind --default-ulimit nproc=2048:4096 --default-ulimit nproc=2048:8192 --default-ulimit nofile=1024:4096 ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start isulad failed" && ((ret++)) ++ + isula run --ulimit nproc= $image --runtime $runtime /bin/sh > $ulimitlog 2>&1 + cat $ulimitlog | grep "delimiter '=' can't be the first or the last character" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check failed" && ((ret++)) +diff --git a/test/specs/specs/CMakeLists.txt b/test/specs/specs/CMakeLists.txt +index a9dbc52c..508123fa 100644 +--- a/test/specs/specs/CMakeLists.txt ++++ b/test/specs/specs/CMakeLists.txt +@@ -84,6 +84,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks + ) + ++set_target_properties(${EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,util_common_calloc_s -Wl,--wrap,util_smart_calloc_s -Wl,--wrap,get_readonly_default_oci_spec") + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) + set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/specs/specs/specs_ut.cc b/test/specs/specs/specs_ut.cc +index ad903a3f..47e4ca6e 100644 +--- a/test/specs/specs/specs_ut.cc ++++ b/test/specs/specs/specs_ut.cc +@@ -19,6 +19,7 @@ + #include "mock.h" + #include "isula_libutils/oci_runtime_spec.h" + #include "specs_api.h" ++#include "specs_mount.h" + #include "specs_namespace.h" + #include "specs_security.h" + #include "isula_libutils/host_config.h" +@@ -43,6 +44,46 @@ using ::testing::_; + + using namespace std; + ++static int g_malloc_count = 0; ++static int g_malloc_match = 1; ++ ++extern "C" { ++ DECLARE_WRAPPER_V(util_common_calloc_s, void *, (size_t size)); ++ DEFINE_WRAPPER_V(util_common_calloc_s, void *, (size_t size), (size)); ++ ++ DECLARE_WRAPPER_V(util_smart_calloc_s, void *, (size_t size, size_t len)); ++ DEFINE_WRAPPER_V(util_smart_calloc_s, void *, (size_t size, size_t len), (size, len)); ++ ++ DECLARE_WRAPPER(get_readonly_default_oci_spec, const oci_runtime_spec *, (bool system_container)); ++ DEFINE_WRAPPER(get_readonly_default_oci_spec, const oci_runtime_spec *, (bool system_container), (system_container)); ++} ++ ++void *util_common_calloc_s_fail(size_t size) ++{ ++ g_malloc_count++; ++ ++ if (g_malloc_count == g_malloc_match) { ++ g_malloc_match++; ++ g_malloc_count = 0; ++ return nullptr; ++ } else { ++ return __real_util_common_calloc_s(size); ++ } ++} ++ ++void *util_smart_calloc_s_fail(size_t size, size_t len) ++{ ++ g_malloc_count++; ++ ++ if (g_malloc_count == g_malloc_match) { ++ g_malloc_match++; ++ g_malloc_count = 0; ++ return nullptr; ++ } else { ++ return __real_util_smart_calloc_s(size, len); ++ } ++} ++ + class SpecsUnitTest : public testing::Test { + public: + void SetUp() override +@@ -234,6 +275,32 @@ char *invoke_conf_get_isulad_cgroup_parent() + return util_strdup_s("/var/lib/isulad/engines/lcr"); + } + ++int invoke_conf_get_isulad_default_ulimit_empty(host_config_ulimits_element ***ulimit) ++{ ++ if (ulimit == nullptr) { ++ return -1; ++ } ++ return 0; ++} ++ ++int invoke_conf_get_isulad_default_ulimit(host_config_ulimits_element ***ulimit) ++{ ++ if (ulimit == nullptr) { ++ return -1; ++ } ++ host_config_ulimits_element *ele = static_cast<host_config_ulimits_element*>(util_common_calloc_s(sizeof(host_config_ulimits_element))); ++ if (ele == nullptr) { ++ return -1; ++ } ++ ele->hard = 8192; ++ ele->soft = 2048; ++ ele->name = util_strdup_s("NPROC"); ++ ++ int ret = ulimit_array_append(ulimit, ele, ulimit_array_len(*ulimit)); ++ free_host_config_ulimits_element(ele); ++ return ret; ++} ++ + TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_1) + { + ASSERT_EQ(merge_container_cgroups_path(nullptr, nullptr), nullptr); +@@ -347,6 +414,107 @@ TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_5) + testing::Mock::VerifyAndClearExpectations(&m_isulad_conf); + } + ++TEST_F(SpecsUnitTest, test_update_oci_container_cgroups_path) ++{ ++ parser_error err = nullptr; ++ host_config *hostspec = static_cast<host_config *>(util_common_calloc_s(sizeof(host_config))); ++ ASSERT_NE(hostspec, nullptr); ++ ++ oci_runtime_spec *ocispec = oci_runtime_spec_parse_data("{\"ociVersion\": \"1.0.1\", \"linux\": \ ++ {} }", nullptr, &err); ++ ASSERT_NE(ocispec, nullptr); ++ ++ ocispec->linux->cgroups_path = util_strdup_s("/isulad"); ++ ASSERT_EQ(update_oci_container_cgroups_path("abcdef", nullptr, nullptr), -1); ++ EXPECT_CALL(m_isulad_conf, GetCgroupParent()).WillRepeatedly(Invoke(invoke_conf_get_isulad_cgroup_parent)); ++ ASSERT_EQ(update_oci_container_cgroups_path("abcdef", ocispec, hostspec), 0); ++ ASSERT_STREQ(ocispec->linux->cgroups_path, "/var/lib/isulad/engines/lcr/abcdef"); ++ ++ free(err); ++ free_host_config(hostspec); ++ free_oci_runtime_spec(ocispec); ++ ++ testing::Mock::VerifyAndClearExpectations(&m_isulad_conf); ++} ++ ++TEST_F(SpecsUnitTest, test_update_oci_ulimit) ++{ ++ parser_error err = nullptr; ++ host_config *hostspec = static_cast<host_config *>(util_common_calloc_s(sizeof(host_config))); ++ ASSERT_NE(hostspec, nullptr); ++ ++ char *oci_config_file = json_path(OCI_RUNTIME_SPEC_FILE); ++ ASSERT_TRUE(oci_config_file != nullptr); ++ oci_runtime_spec *ocispec = oci_runtime_spec_parse_file(oci_config_file, nullptr, &err); ++ ASSERT_NE(ocispec, nullptr); ++ ++ ASSERT_EQ(update_oci_ulimit(nullptr, nullptr), -1); ++ EXPECT_CALL(m_isulad_conf, GetUlimit(_)).WillRepeatedly(Invoke(invoke_conf_get_isulad_default_ulimit)); ++ ASSERT_EQ(update_oci_ulimit(ocispec, hostspec), 0); ++ ASSERT_EQ(ocispec->process->rlimits_len, 1); ++ ASSERT_EQ(ocispec->process->rlimits[0]->hard, 8192); ++ ASSERT_EQ(ocispec->process->rlimits[0]->soft, 2048); ++ ASSERT_STREQ(ocispec->process->rlimits[0]->type, "RLIMIT_NPROC"); ++ EXPECT_CALL(m_isulad_conf, GetUlimit(_)).WillRepeatedly(Invoke(invoke_conf_get_isulad_default_ulimit_empty)); ++ ASSERT_EQ(update_oci_ulimit(ocispec, hostspec), 0); ++ ASSERT_EQ(ocispec->process->rlimits_len, 0); ++ ++ free(err); ++ free(oci_config_file); ++ free_host_config(hostspec); ++ free_oci_runtime_spec(ocispec); ++ testing::Mock::VerifyAndClearExpectations(&m_isulad_conf); ++} ++ ++TEST_F(SpecsUnitTest, test_update_devcies_for_oci_spec) ++{ ++ parser_error err = nullptr; ++ oci_runtime_spec *readonly_spec = oci_runtime_spec_parse_data("{\"ociVersion\": \"1.0.1\", \"linux\": \ ++ { \"devices\": \ ++ [ { \"type\": \"c\", \"path\": \"/dev/testA\", \ ++ \"fileMode\": 8612, \"major\": 99, \"minor\": 99} ], \ ++ \"resources\": { \"devices\": [ { \"allow\": false, \ ++ \"type\": \"a\", \"major\": -1, \ ++ \"minor\": -1, \"access\": \"rwm\" } ] } } }", nullptr, &err); ++ ASSERT_NE(readonly_spec, nullptr); ++ free(err); ++ err = nullptr; ++ host_config *hostspec = static_cast<host_config *>(util_common_calloc_s(sizeof(host_config))); ++ ASSERT_NE(hostspec, nullptr); ++ ++ oci_runtime_spec *ocispec = oci_runtime_spec_parse_data("{\"ociVersion\": \"1.0.1\", \"linux\": \ ++ { \"devices\": [ ], \ ++ \"resources\": { \"devices\": [ ] } } }", nullptr, &err); ++ ASSERT_NE(ocispec, nullptr); ++ ++ MOCK_SET(get_readonly_default_oci_spec, readonly_spec); ++ MOCK_SET_V(util_smart_calloc_s, util_smart_calloc_s_fail); ++ MOCK_SET_V(util_common_calloc_s, util_common_calloc_s_fail); ++ ++ ASSERT_EQ(update_devcies_for_oci_spec(ocispec, hostspec), -1); ++ ASSERT_EQ(update_devcies_for_oci_spec(ocispec, hostspec), -1); ++ ASSERT_EQ(update_devcies_for_oci_spec(ocispec, hostspec), -1); ++ free(ocispec->linux->devices[0]); ++ free(ocispec->linux->devices); ++ ocispec->linux->devices = NULL; ++ ocispec->linux->devices_len = 0; ++ ASSERT_EQ(update_devcies_for_oci_spec(ocispec, hostspec), -1); ++ free(ocispec->linux->devices[0]); ++ free(ocispec->linux->devices); ++ ocispec->linux->devices = NULL; ++ ocispec->linux->devices_len = 0; ++ ASSERT_EQ(update_devcies_for_oci_spec(ocispec, hostspec), 0); ++ ++ MOCK_CLEAR(get_readonly_default_oci_spec); ++ MOCK_CLEAR(util_smart_calloc_s); ++ MOCK_CLEAR(util_common_calloc_s); ++ ++ free_oci_runtime_spec(readonly_spec); ++ free_oci_runtime_spec(ocispec); ++ free_host_config(hostspec); ++ free(err); ++} ++ + /********************************* UT for merge caps *******************************************/ + struct capabilities_lens { + size_t bounding_len; +-- +2.34.1 + diff --git a/0010-open-run-container-with-dev-volume-testcase.patch b/0010-open-run-container-with-dev-volume-testcase.patch new file mode 100644 index 0000000..2381125 --- /dev/null +++ b/0010-open-run-container-with-dev-volume-testcase.patch @@ -0,0 +1,30 @@ +From 44d15a7451a922ca7266b756d3f9a83908199cb3 Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu <zhangxiaoyu58@huawei.com> +Date: Tue, 23 Jan 2024 10:35:59 +0800 +Subject: [PATCH 10/43] open run container with dev volume testcase + +Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com> +--- + CI/test_cases/container_cases/bind_special_dir.sh | 5 ++--- + 1 file changed, 2 insertions(+), 3 deletions(-) + +diff --git a/CI/test_cases/container_cases/bind_special_dir.sh b/CI/test_cases/container_cases/bind_special_dir.sh +index 545d5099..04bf437a 100755 +--- a/CI/test_cases/container_cases/bind_special_dir.sh ++++ b/CI/test_cases/container_cases/bind_special_dir.sh +@@ -40,10 +40,9 @@ function test_bind_special_dir() + # when create container in container, runc not support to mount /dev + # adapt fedora base image, we just remove rshared option of sys dir + if [ $runtime == "runc" ]; then +- c_id=`isula run -itd -v -itd --runtime=$runtime -v /sys/fs:/sys/fs:rw -v /proc:/proc -v /dev/pts:/dev/pts:rw busybox sh` ++ c_id=`isula run -itd --runtime=$runtime -v /sys/fs:/sys/fs:rw -v /proc:/proc -v /dev/pts:/dev/pts:rw busybox sh` + else +- # lxc 5.X cannot support mount /dev directory +- c_id=`isula run --runtime=$runtime -itd -v -itd -v /sys/fs:/sys/fs:rw -v /proc:/proc busybox sh` ++ c_id=`isula run --runtime=$runtime -itd -v /sys/fs:/sys/fs:rw -v /proc:/proc -v /dev:/dev:ro -v /dev/pts:/dev/pts:rw busybox sh` + fi + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + +-- +2.34.1 + diff --git a/0011-add-cpu-usage-nano-cores-for-sandbox.patch b/0011-add-cpu-usage-nano-cores-for-sandbox.patch new file mode 100644 index 0000000..27be05d --- /dev/null +++ b/0011-add-cpu-usage-nano-cores-for-sandbox.patch @@ -0,0 +1,81 @@ +From 3dc12d7806fda8d5ceee183595e993079bee4056 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Fri, 12 Jan 2024 17:38:09 +0800 +Subject: [PATCH 11/43] add cpu usage nano cores for sandbox + +Signed-off-by: jikai <jikai11@huawei.com> +--- + .../v1/v1_cri_pod_sandbox_manager_service.cc | 17 ++++++++++++++--- + .../cri/v1/v1_cri_pod_sandbox_manager_service.h | 1 + + 2 files changed, 15 insertions(+), 3 deletions(-) + +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +index a0c45111..76fa17bc 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +@@ -1024,6 +1024,7 @@ void PodSandboxManagerService::PodSandboxStatsToGRPC(const std::string &id, cons + const std::vector<Network::NetworkInterfaceStats> &netMetrics, + const std::unique_ptr<ContainerManagerService> &containerManager, + std::unique_ptr<runtime::v1::PodSandboxStats> &podStats, ++ sandbox::StatsInfo &oldStatsRec, + Errors &error) + { + std::unique_ptr<runtime::v1::PodSandboxStats> podStatsPtr( +@@ -1044,8 +1045,13 @@ void PodSandboxManagerService::PodSandboxStatsToGRPC(const std::string &id, cons + auto cpu = podStatsPtr->mutable_linux()->mutable_cpu(); + cpu->set_timestamp(timestamp); + cpu->mutable_usage_core_nano_seconds()->set_value(cgroupMetrics.cgcpu_metrics.cpu_use_nanos); +- // todo +- // cpu->mutable_usage_nano_cores()->set_value(getNanoCores()); ++ if (oldStatsRec.cpuUseNanos != 0 && timestamp > oldStatsRec.timestamp && ++ cgroupMetrics.cgcpu_metrics.cpu_use_nanos > oldStatsRec.cpuUseNanos) { ++ uint64_t usage = cgroupMetrics.cgcpu_metrics.cpu_use_nanos - oldStatsRec.cpuUseNanos; ++ uint64_t nanoSeconds = timestamp - oldStatsRec.timestamp; ++ uint64_t usage_nano_cores = (uint64_t)(((double)usage / (double)nanoSeconds) * (double)Time_Second); ++ cpu->mutable_usage_nano_cores()->set_value(usage_nano_cores); ++ } + + // Memory + auto memory = podStatsPtr->mutable_linux()->mutable_memory(); +@@ -1114,6 +1120,7 @@ auto PodSandboxManagerService::PodSandboxStats(const std::string &podSandboxID, + return nullptr; + } + auto &config = sandbox->GetSandboxConfig(); ++ auto oldStatsRec = sandbox->GetStatsInfo(); + + auto status = PodSandboxStatus(sandbox->GetId(), tmpErr); + if (error.NotEmpty()) { +@@ -1136,13 +1143,17 @@ auto PodSandboxManagerService::PodSandboxStats(const std::string &podSandboxID, + tmpErr.Clear(); + } + +- PodSandboxStatsToGRPC(sandbox->GetId(), cgroupMetrics, netMetrics, containerManager, podStats, tmpErr); ++ PodSandboxStatsToGRPC(sandbox->GetId(), cgroupMetrics, netMetrics, containerManager, podStats, oldStatsRec, tmpErr); + if (tmpErr.NotEmpty()) { + ERROR("Failed to set PodSandboxStats: %s", tmpErr.GetCMessage()); + error.Errorf("Failed to set PodSandboxStats"); + return nullptr; + } + ++ // update stats info that sandbox recorded ++ sandbox::StatsInfo newStatsRec { podStats->linux().cpu().timestamp(), podStats->linux().cpu().usage_core_nano_seconds().value() }; ++ sandbox->UpdateStatsInfo(newStatsRec); ++ + return podStats; + } + +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h +index 2bd28007..c3d98b8c 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h +@@ -123,6 +123,7 @@ private: + const std::vector<Network::NetworkInterfaceStats> &netMetrics, + const std::unique_ptr<ContainerManagerService> &containerManager, + std::unique_ptr<runtime::v1::PodSandboxStats> &podStats, ++ sandbox::StatsInfo &statsInfo, + Errors &error); + void GetFilterPodSandbox(const runtime::v1::PodSandboxStatsFilter *filter, + std::vector<std::string> &podSandboxIDs, Errors &error); +-- +2.34.1 + diff --git a/0012-sleep-some-time-in-ServiceWorkThread-to-prevent-the-.patch b/0012-sleep-some-time-in-ServiceWorkThread-to-prevent-the-.patch new file mode 100644 index 0000000..9d70f92 --- /dev/null +++ b/0012-sleep-some-time-in-ServiceWorkThread-to-prevent-the-.patch @@ -0,0 +1,27 @@ +From 384cf7870c155d41f742b1928a4cb1b56aa46c94 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Tue, 6 Feb 2024 20:05:05 +0800 +Subject: [PATCH 12/43] sleep some time in ServiceWorkThread to prevent the CPU + from being occupied all the time + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/entry/cri/streams/websocket/ws_server.cc | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/src/daemon/entry/cri/streams/websocket/ws_server.cc b/src/daemon/entry/cri/streams/websocket/ws_server.cc +index 6319a67f..a8d89b36 100644 +--- a/src/daemon/entry/cri/streams/websocket/ws_server.cc ++++ b/src/daemon/entry/cri/streams/websocket/ws_server.cc +@@ -551,6 +551,8 @@ void WebsocketServer::ServiceWorkThread(int threadid) + + while (n >= 0 && m_forceExit == 0) { + n = lws_service(m_context, 0); ++ // sleep some time to prevent the CPU from being occupied all the time ++ std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + } + +-- +2.34.1 + diff --git a/0013-restore-name-for-rename-failed.patch b/0013-restore-name-for-rename-failed.patch new file mode 100644 index 0000000..be7d59c --- /dev/null +++ b/0013-restore-name-for-rename-failed.patch @@ -0,0 +1,29 @@ +From 2df7a67ad2cb0249b18ca5eba46f9aab8f72038f Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Sun, 18 Feb 2024 11:32:55 +0800 +Subject: [PATCH 13/43] restore name for rename failed + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/executor/container_cb/execution_information.c | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/src/daemon/executor/container_cb/execution_information.c b/src/daemon/executor/container_cb/execution_information.c +index 2a71e82a..c02cc830 100644 +--- a/src/daemon/executor/container_cb/execution_information.c ++++ b/src/daemon/executor/container_cb/execution_information.c +@@ -1116,6 +1116,11 @@ static void restore_names_at_fail(container_t *cont, const char *ori_name, const + if (!container_name_index_rename(ori_name, new_name, id)) { + ERROR("Failed to restore name from \"%s\" to \"%s\" for container %s", new_name, ori_name, id); + } ++ ++ // restore name in id-name manager ++ if (!id_name_manager_rename(ori_name, new_name)) { ++ ERROR("Failed to restore name from \"%s\" to \"%s\" in id-name manager", new_name, ori_name); ++ } + } + + static int container_rename(container_t *cont, const char *new_name) +-- +2.34.1 + diff --git a/0014-2371-Allow-iSulad-to-pull-load-image-with-symlink.patch b/0014-2371-Allow-iSulad-to-pull-load-image-with-symlink.patch new file mode 100644 index 0000000..4a1fa1f --- /dev/null +++ b/0014-2371-Allow-iSulad-to-pull-load-image-with-symlink.patch @@ -0,0 +1,45 @@ +From fd4c80b8de768d7132cef0720cd46167173a653b Mon Sep 17 00:00:00 2001 +From: xuxuepeng <xuxuepeng1@huawei.com> +Date: Mon, 19 Feb 2024 01:05:18 +0000 +Subject: [PATCH 14/43] !2371 Allow iSulad to pull/load image with symlink * + Allow iSulad to pull/load image with symlink + +--- + src/utils/tar/util_archive.c | 13 ++++++++++--- + 1 file changed, 10 insertions(+), 3 deletions(-) + +diff --git a/src/utils/tar/util_archive.c b/src/utils/tar/util_archive.c +index e4c302bc..52b51162 100644 +--- a/src/utils/tar/util_archive.c ++++ b/src/utils/tar/util_archive.c +@@ -637,6 +637,11 @@ static void try_to_replace_exited_dst(const char *dst_path, struct archive_entry + } + } + ++/** ++ * This function has to be used with chroot to prevent a potential attack from manipulating ++ * the path of the file to be extracted, such as using a symbolic link to extract the file to ++ * a location outside the path. ++ */ + int archive_unpack_handler(const struct io_read_wrapper *content, const struct archive_options *options) + { + int ret = 0; +@@ -671,10 +676,12 @@ int archive_unpack_handler(const struct io_read_wrapper *content, const struct a + flags |= ARCHIVE_EXTRACT_PERM; + flags |= ARCHIVE_EXTRACT_ACL; + flags |= ARCHIVE_EXTRACT_FFLAGS; +- flags |= ARCHIVE_EXTRACT_SECURE_SYMLINKS; +- flags |= ARCHIVE_EXTRACT_SECURE_NODOTDOT; + flags |= ARCHIVE_EXTRACT_XATTR; +- flags |= ARCHIVE_EXTRACT_SECURE_NOABSOLUTEPATHS; ++ /** ++ * ARCHIVE_EXTRACT_SECURE_SYMLINKS, ARCHIVE_EXTRACT_SECURE_NODOTDOT, ++ * ARCHIVE_EXTRACT_SECURE_NOABSOLUTEPATHS flags are not set here, ++ * since this function is called after chroot, the security of the path is guaranteed. ++ */ + + a = archive_read_new(); + if (a == NULL) { +-- +2.34.1 + diff --git a/0015-Replace-http-parser-dependency-with-lcr.patch b/0015-Replace-http-parser-dependency-with-lcr.patch new file mode 100644 index 0000000..eccb44e --- /dev/null +++ b/0015-Replace-http-parser-dependency-with-lcr.patch @@ -0,0 +1,427 @@ +From fb76605985166c4d2172270c8d633ed26d62f698 Mon Sep 17 00:00:00 2001 +From: xuxuepeng <xuxuepeng1@huawei.com> +Date: Mon, 19 Feb 2024 23:52:47 +0800 +Subject: [PATCH 15/43] Replace http-parser dependency with lcr + +Signed-off-by: xuxuepeng <xuxuepeng1@huawei.com> +--- + CI/dockerfiles/Dockerfile-centos | 13 ------------- + CI/dockerfiles/Dockerfile-fedora | 1 - + CI/dockerfiles/Dockerfile-ubuntu | 1 - + CI/pr-gateway.sh | 2 +- + Dockerfile | 13 ------------- + cmake/checker.cmake | 5 ----- + docs/build_docs/guide/build_guide.md | 13 ------------- + docs/build_docs/guide/build_guide_with_rpm.md | 4 +--- + docs/build_docs/guide/build_guide_with_rpm_zh.md | 4 +--- + docs/build_docs/guide/build_guide_zh.md | 13 ------------- + .../guide/script/install_iSulad_on_Centos_7.sh | 11 ----------- + .../script/install_iSulad_on_Ubuntu_20_04_LTS.sh | 2 +- + .../isulad_build_in_openeuler.Dockerfile | 2 +- + docs/vs_other_engines/vs_docker_command.md | 1 - + iSulad.spec | 3 +-- + src/CMakeLists.txt | 1 - + src/contrib/env_checkconfig | 1 - + .../modules/image/oci/registry/registry_apiv1.c | 2 +- + .../modules/image/oci/registry/registry_apiv2.c | 2 +- + src/utils/http/CMakeLists.txt | 2 +- + src/utils/http/parser.c | 2 +- + src/utils/http/parser.h | 2 +- + src/utils/http/rest_common.c | 2 +- + 23 files changed, 12 insertions(+), 90 deletions(-) + +diff --git a/CI/dockerfiles/Dockerfile-centos b/CI/dockerfiles/Dockerfile-centos +index 1d76b4ec..af3ce035 100644 +--- a/CI/dockerfiles/Dockerfile-centos ++++ b/CI/dockerfiles/Dockerfile-centos +@@ -207,19 +207,6 @@ RUN export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH && \ + make install && \ + ldconfig + +-# install http-parser +-RUN export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH && \ +- set -x && \ +- cd ~ && \ +- git clone https://gitee.com/src-openeuler/http-parser.git && \ +- cd http-parser && \ +- git checkout -b openEuler-20.03-LTS-tag openEuler-20.03-LTS-tag && \ +- tar -xzvf http-parser-2.9.2.tar.gz && \ +- cd http-parser-2.9.2 && \ +- make -j CFLAGS="-Wno-error" && \ +- make CFLAGS="-Wno-error" install && \ +- ldconfig +- + # install libwebsockets + RUN export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH && \ + set -x && \ +diff --git a/CI/dockerfiles/Dockerfile-fedora b/CI/dockerfiles/Dockerfile-fedora +index a105cbb4..be2bf412 100644 +--- a/CI/dockerfiles/Dockerfile-fedora ++++ b/CI/dockerfiles/Dockerfile-fedora +@@ -42,7 +42,6 @@ RUN dnf update -y && dnf install -y automake \ + grpc-plugins \ + libevent-devel \ + libwebsockets-devel \ +- http-parser-devel \ + gtest-devel \ + gmock-devel \ + libarchive-devel \ +diff --git a/CI/dockerfiles/Dockerfile-ubuntu b/CI/dockerfiles/Dockerfile-ubuntu +index 2441a7ce..09a20eb5 100644 +--- a/CI/dockerfiles/Dockerfile-ubuntu ++++ b/CI/dockerfiles/Dockerfile-ubuntu +@@ -71,7 +71,6 @@ RUN apt update -y && apt upgrade -y && \ + language-pack-en \ + curl \ + cmake \ +- libhttp-parser-dev \ + libprotobuf-dev \ + libgrpc-dev \ + libgrpc++-dev \ +diff --git a/CI/pr-gateway.sh b/CI/pr-gateway.sh +index e5bf627e..e3613e8e 100755 +--- a/CI/pr-gateway.sh ++++ b/CI/pr-gateway.sh +@@ -22,7 +22,7 @@ sed -i "s#http://repo.openeuler.org#https://repo.huaweicloud.com/openeuler#g" /e + + dnf update -y + +-dnf install -y docbook2X doxygen gtest-devel gmock-devel diffutils cmake gcc-c++ yajl-devel patch make libtool libevent-devel libevhtp-devel grpc grpc-plugins grpc-devel protobuf-devel libcurl libcurl-devel sqlite-devel libarchive-devel device-mapper-devel http-parser-devel libseccomp-devel libcap-devel libselinux-devel libwebsockets libwebsockets-devel systemd-devel git chrpath ncurses-devel ++dnf install -y docbook2X doxygen gtest-devel gmock-devel diffutils cmake gcc-c++ yajl-devel patch make libtool libevent-devel libevhtp-devel grpc grpc-plugins grpc-devel protobuf-devel libcurl libcurl-devel sqlite-devel libarchive-devel device-mapper-devel libseccomp-devel libcap-devel libselinux-devel libwebsockets libwebsockets-devel systemd-devel git chrpath ncurses-devel + if [ $? -ne 0 ]; then + echo "install dependences failed" + exit 1 +diff --git a/Dockerfile b/Dockerfile +index 3b284630..a30ed12a 100644 +--- a/Dockerfile ++++ b/Dockerfile +@@ -197,19 +197,6 @@ RUN export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH && \ + make install && \ + ldconfig + +-# install http-parser +-RUN export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH && \ +- set -x && \ +- cd ~ && \ +- git clone https://gitee.com/src-openeuler/http-parser.git && \ +- cd http-parser && \ +- git checkout -b openEuler-20.03-LTS-tag openEuler-20.03-LTS-tag && \ +- tar -xzvf http-parser-2.9.2.tar.gz && \ +- cd http-parser-2.9.2 && \ +- make -j CFLAGS="-Wno-error" && \ +- make CFLAGS="-Wno-error" install && \ +- ldconfig +- + # install libwebsockets + RUN export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH && \ + set -x && \ +diff --git a/cmake/checker.cmake b/cmake/checker.cmake +index b0c395ef..13f0fd62 100644 +--- a/cmake/checker.cmake ++++ b/cmake/checker.cmake +@@ -66,11 +66,6 @@ if (ANDROID OR MUSL) + _CHECK(CRYPTO_LIBRARY "LIBSSL_LIBRARY-NOTFOUND" "libssl.so") + endif() + +-find_path(HTTP_PARSER_INCLUDE_DIR http_parser.h) +-_CHECK(HTTP_PARSER_INCLUDE_DIR "HTTP_PARSER_INCLUDE_DIR-NOTFOUND" "http_parser.h") +-find_library(HTTP_PARSER_LIBRARY http_parser) +-_CHECK(HTTP_PARSER_LIBRARY "HTTP_PARSER_LIBRARY-NOTFOUND" "libhttp_parser.so") +- + pkg_check_modules(PC_CURL "libcurl>=7.4.0") + find_path(CURL_INCLUDE_DIR "curl/curl.h" + HINTS ${PC_CURL_INCLUDEDIR} ${PC_CURL_INCLUDE_DIRS}) +diff --git a/docs/build_docs/guide/build_guide.md b/docs/build_docs/guide/build_guide.md +index 741abddd..73a0d9d8 100644 +--- a/docs/build_docs/guide/build_guide.md ++++ b/docs/build_docs/guide/build_guide.md +@@ -150,19 +150,6 @@ $ sudo -E make install + $ sudo -E ldconfig + ``` + +-#### build and install http-parser +- +-```bash +-$ git clone https://gitee.com/src-openeuler/http-parser.git +-$ cd http-parser +-$ git checkout openEuler-20.03-LTS-tag +-$ tar -xzvf http-parser-2.9.2.tar.gz +-$ cd http-parser-2.9.2 +-$ sudo -E make -j CFLAGS="-Wno-error" +-$ sudo -E make CFLAGS="-Wno-error" install +-$ sudo -E ldconfig +-``` +- + #### build and install libwebsockets + + ```bash +diff --git a/docs/build_docs/guide/build_guide_with_rpm.md b/docs/build_docs/guide/build_guide_with_rpm.md +index 181b2ef5..acf8e7c0 100644 +--- a/docs/build_docs/guide/build_guide_with_rpm.md ++++ b/docs/build_docs/guide/build_guide_with_rpm.md +@@ -29,7 +29,7 @@ BUILD BUILDROOT RPMS SOURCES SPECS SRPMS + + ```shell + dnf install -y patch automake autoconf libtool cmake make libcap libcap-devel libselinux libselinux-devel libseccomp libseccomp-devel git libcgroup tar python3 python3-pip libcurl-devel zlib-devel glibc-headers openssl-devel gcc gcc-c++ systemd-devel systemd-libs golang libtar && \ +-dnf --enablerepo=powertools install -y yajl-devel device-mapper-devel http-parser-devel && \ ++dnf --enablerepo=powertools install -y yajl-devel device-mapper-devel && \ + dnf install -y epel-release && \ + dnf --enablerepo=powertools install libuv-devel &&\ + dnf install libwebsockets-devel +@@ -281,7 +281,6 @@ rpm -Uvh libarchive-devel-3.4.3-4.x86_64.rpm + ### 9.1 install iSulad dependencies + + ```shell +-dnf --enablerepo=powertools install http-parser-devel + dnf install -y sqlite-devel + ``` + +@@ -317,7 +316,6 @@ dnf install libwebsockets-devel + then, you can install iSulad + + ```shell +-dnf --enablerepo=powertools install http-parser-devel + dnf install -y sqlite-devel.x86_64 + rpm -Uvh iSulad-2.1.0-1.x86_64.rpm + ``` +\ No newline at end of file +diff --git a/docs/build_docs/guide/build_guide_with_rpm_zh.md b/docs/build_docs/guide/build_guide_with_rpm_zh.md +index edb565e3..b9574b4e 100644 +--- a/docs/build_docs/guide/build_guide_with_rpm_zh.md ++++ b/docs/build_docs/guide/build_guide_with_rpm_zh.md +@@ -25,7 +25,7 @@ BUILD BUILDROOT RPMS SOURCES SPECS SRPMS + + ```shell + dnf install -y patch automake autoconf libtool cmake make libcap libcap-devel libselinux libselinux-devel libseccomp libseccomp-devel git libcgroup tar python3 python3-pip libcurl-devel zlib-devel glibc-headers openssl-devel gcc gcc-c++ systemd-devel systemd-libs golang libtar && \ +-dnf --enablerepo=powertools install -y yajl-devel device-mapper-devel http-parser-devel && \ ++dnf --enablerepo=powertools install -y yajl-devel device-mapper-devel && \ + dnf install -y epel-release && \ + dnf --enablerepo=powertools install libuv-devel &&\ + dnf install libwebsockets-devel +@@ -272,7 +272,6 @@ rpm -Uvh libarchive-devel-3.4.3-4.x86_64.rpm + ### 9.1 安装iSulad的依赖 + + ```shell +-dnf --enablerepo=powertools install http-parser-devel + dnf install -y sqlite-devel + ``` + +@@ -310,7 +309,6 @@ dnf install libwebsockets-devel + 再安装isulad： + + ```shell +-dnf --enablerepo=powertools install http-parser-devel + dnf install -y sqlite-devel.x86_64 + rpm -Uvh iSulad-2.1.0-1.x86_64.rpm + ``` +\ No newline at end of file +diff --git a/docs/build_docs/guide/build_guide_zh.md b/docs/build_docs/guide/build_guide_zh.md +index bfdc69dc..2d853d9c 100644 +--- a/docs/build_docs/guide/build_guide_zh.md ++++ b/docs/build_docs/guide/build_guide_zh.md +@@ -150,19 +150,6 @@ $ sudo -E make install + $ sudo -E ldconfig + ``` + +-#### 编译安装http-parser +- +-```bash +-$ git clone https://gitee.com/src-openeuler/http-parser.git +-$ cd http-parser +-$ git checkout openEuler-20.03-LTS-tag +-$ tar -xzvf http-parser-2.9.2.tar.gz +-$ cd http-parser-2.9.2 +-$ sudo -E make -j CFLAGS="-Wno-error" +-$ sudo -E make CFLAGS="-Wno-error" install +-$ sudo -E ldconfig +-``` +- + #### 编译安装libwebsockets + + ```bash +diff --git a/docs/build_docs/guide/script/install_iSulad_on_Centos_7.sh b/docs/build_docs/guide/script/install_iSulad_on_Centos_7.sh +index 3834d333..b268d777 100755 +--- a/docs/build_docs/guide/script/install_iSulad_on_Centos_7.sh ++++ b/docs/build_docs/guide/script/install_iSulad_on_Centos_7.sh +@@ -68,17 +68,6 @@ make -j $(nproc) + make install + ldconfig + +-# build http_parser +-cd $BUILD_DIR +-git clone https://gitee.com/src-openeuler/http-parser.git +-cd http-parser +-git checkout openEuler-20.03-LTS-tag +-tar -xzvf http-parser-2.9.2.tar.gz +-cd http-parser-2.9.2 +-make -j CFLAGS="-Wno-error" +-make CFLAGS="-Wno-error" install +-ldconfig +- + # build libwebsockets + cd $BUILD_DIR + git clone https://gitee.com/src-openeuler/libwebsockets.git +diff --git a/docs/build_docs/guide/script/install_iSulad_on_Ubuntu_20_04_LTS.sh b/docs/build_docs/guide/script/install_iSulad_on_Ubuntu_20_04_LTS.sh +index 4f27244e..f44bddb4 100755 +--- a/docs/build_docs/guide/script/install_iSulad_on_Ubuntu_20_04_LTS.sh ++++ b/docs/build_docs/guide/script/install_iSulad_on_Ubuntu_20_04_LTS.sh +@@ -7,7 +7,7 @@ set -e + export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH + export LD_LIBRARY_PATH=/usr/local/lib:/usr/lib:/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH + echo "/usr/local/lib" >> /etc/ld.so.conf +-apt install -y g++ libprotobuf-dev protobuf-compiler protobuf-compiler-grpc libgrpc++-dev libgrpc-dev libtool automake autoconf cmake make pkg-config libyajl-dev zlib1g-dev libselinux1-dev libseccomp-dev libcap-dev libsystemd-dev git libarchive-dev libcurl4-gnutls-dev openssl libdevmapper-dev python3 libtar0 libtar-dev libhttp-parser-dev libwebsockets-dev ++apt install -y g++ libprotobuf-dev protobuf-compiler protobuf-compiler-grpc libgrpc++-dev libgrpc-dev libtool automake autoconf cmake make pkg-config libyajl-dev zlib1g-dev libselinux1-dev libseccomp-dev libcap-dev libsystemd-dev git libarchive-dev libcurl4-gnutls-dev openssl libdevmapper-dev python3 libtar0 libtar-dev libwebsockets-dev + + BUILD_DIR=/tmp/build_isulad + +diff --git a/docs/dockerfiles/isulad_build_in_openeuler.Dockerfile b/docs/dockerfiles/isulad_build_in_openeuler.Dockerfile +index 5049f783..a081b009 100644 +--- a/docs/dockerfiles/isulad_build_in_openeuler.Dockerfile ++++ b/docs/dockerfiles/isulad_build_in_openeuler.Dockerfile +@@ -26,7 +26,7 @@ RUN dnf install -y rust rust-packaging cargo + RUN dnf install -y grpc grpc-plugins grpc-devel protobuf-devel libwebsockets libwebsockets-devel + + # depends for image module and restful client of iSulad +-RUN dnf install -y libcurl libcurl-devel libarchive-devel http-parser-devel ++RUN dnf install -y libcurl libcurl-devel libarchive-devel + + # depends for embedded image of iSulad: -DENABLE_EMBEDDED=ON + RUN dnf install -y sqlite-devel +diff --git a/docs/vs_other_engines/vs_docker_command.md b/docs/vs_other_engines/vs_docker_command.md +index 5c38d31b..d7b587ea 100644 +--- a/docs/vs_other_engines/vs_docker_command.md ++++ b/docs/vs_other_engines/vs_docker_command.md +@@ -114,7 +114,6 @@ Installing dependencies: + abseil-cpp + clibcni + grpc +- http-parser + lcr + lib-shim-v2 + libwebsockets +diff --git a/iSulad.spec b/iSulad.spec +index eda87f7a..eafa82a0 100644 +--- a/iSulad.spec ++++ b/iSulad.spec +@@ -48,7 +48,6 @@ BuildRequires: libisula-devel > %{lcrver_lower} libisula-devel < %{lcrver_upper} + BuildRequires: cmake gcc-c++ yajl-devel + BuildRequires: grpc grpc-plugins grpc-devel protobuf-devel + BuildRequires: libcurl libcurl-devel libarchive-devel device-mapper-devel +-BuildRequires: http-parser-devel + BuildRequires: libseccomp-devel libcap-devel libselinux-devel libwebsockets libwebsockets-devel + BuildRequires: systemd-devel git + BuildRequires: libevhtp-devel libevent-devel +@@ -60,7 +59,7 @@ BuildRequires: lib-shim-v2 lib-shim-v2-devel + Requires: libisula > %{lcrver_lower} libisula < %{lcrver_upper} + Requires: grpc protobuf + Requires: libcurl +-Requires: http-parser libseccomp ++Requires: libseccomp + Requires: libcap libselinux libwebsockets libarchive device-mapper + Requires: systemd + Requires: (docker-runc or runc) +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index d1bc65f9..48c1bad0 100644 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -3,7 +3,6 @@ set(CHECKED_INCLUDE_DIRS + ${STD_HEADER_SYS_PARAM} + ${LIBYAJL_INCLUDE_DIR} + ${LIBARCHIVE_INCLUDE_DIR} +- ${HTTP_PARSER_INCLUDE_DIR} + ${OPENSSL_INCLUDE_DIR} + ${CURL_INCLUDE_DIR} + ${SYSTEMD_INCLUDE_DIR} +diff --git a/src/contrib/env_checkconfig b/src/contrib/env_checkconfig +index 62a91354..9080e01a 100755 +--- a/src/contrib/env_checkconfig ++++ b/src/contrib/env_checkconfig +@@ -164,7 +164,6 @@ config_set CONFIG_SMP && echo -n "Cpuset Cgroup Result: " && config_enable CONFI + echo "" + echo "--- Third-party Packages ---" + echo -n "libyajl: " && has_lib libyajl +-echo -n "libhttp_parser: " && has_lib libhttp_parser + echo -n "libevhtp.so.1.2.16: " && has_lib libevhtp.so.1.2.16 + echo -n "libseccomp: " && has_lib libseccomp + echo -n "libcap.so: " && has_lib libcap.so +diff --git a/src/daemon/modules/image/oci/registry/registry_apiv1.c b/src/daemon/modules/image/oci/registry/registry_apiv1.c +index 414eb65c..6da24c1d 100644 +--- a/src/daemon/modules/image/oci/registry/registry_apiv1.c ++++ b/src/daemon/modules/image/oci/registry/registry_apiv1.c +@@ -18,7 +18,7 @@ + #include <stdio.h> + #include <string.h> + #include <limits.h> +-#include <http_parser.h> ++#include <isula_libutils/http_parser.h> + #include <isula_libutils/json_common.h> + #include <stdbool.h> + #include <stdlib.h> +diff --git a/src/daemon/modules/image/oci/registry/registry_apiv2.c b/src/daemon/modules/image/oci/registry/registry_apiv2.c +index 3b3bbd93..dd49fab7 100644 +--- a/src/daemon/modules/image/oci/registry/registry_apiv2.c ++++ b/src/daemon/modules/image/oci/registry/registry_apiv2.c +@@ -18,7 +18,7 @@ + #include <stdio.h> + #include <string.h> + #include <limits.h> +-#include <http_parser.h> ++#include <isula_libutils/http_parser.h> + #include <isula_libutils/json_common.h> + #include <stdbool.h> + #include <stdlib.h> +diff --git a/src/utils/http/CMakeLists.txt b/src/utils/http/CMakeLists.txt +index ad7d0747..23d92cdc 100644 +--- a/src/utils/http/CMakeLists.txt ++++ b/src/utils/http/CMakeLists.txt +@@ -20,7 +20,7 @@ target_include_directories(libhttpclient PUBLIC + + # set libhttpclient FLAGS + set_target_properties(libhttpclient PROPERTIES PREFIX "") +-target_link_libraries(libhttpclient ${HTTP_PARSER_LIBRARY} ${CURL_LIBRARY}) ++target_link_libraries(libhttpclient ${CURL_LIBRARY}) + + set_target_properties(libhttpclient PROPERTIES LINKER_LANGUAGE "C") + +diff --git a/src/utils/http/parser.c b/src/utils/http/parser.c +index cf8425e4..3e910efc 100644 +--- a/src/utils/http/parser.c ++++ b/src/utils/http/parser.c +@@ -38,7 +38,7 @@ + */ + + #include "parser.h" +-#include <http_parser.h> ++#include <isula_libutils/http_parser.h> + #include <string.h> + #include <stdlib.h> + #include <stdint.h> +diff --git a/src/utils/http/parser.h b/src/utils/http/parser.h +index ce5fe5e7..fd8f150d 100644 +--- a/src/utils/http/parser.h ++++ b/src/utils/http/parser.h +@@ -42,7 +42,7 @@ + + #include <stddef.h> + +-#include "http_parser.h" ++#include <isula_libutils/http_parser.h> + + #undef TRUE + #define TRUE 1 +diff --git a/src/utils/http/rest_common.c b/src/utils/http/rest_common.c +index 885375f2..d851ba96 100644 +--- a/src/utils/http/rest_common.c ++++ b/src/utils/http/rest_common.c +@@ -16,7 +16,7 @@ + + #include <dlfcn.h> + #include <string.h> +-#include <http_parser.h> ++#include <isula_libutils/http_parser.h> + #include <stdlib.h> + + #include "isula_libutils/log.h" +-- +2.34.1 + diff --git a/0016-add-more-detailed-log-information-for-load-sandbox.patch b/0016-add-more-detailed-log-information-for-load-sandbox.patch new file mode 100644 index 0000000..db8bf17 --- /dev/null +++ b/0016-add-more-detailed-log-information-for-load-sandbox.patch @@ -0,0 +1,26 @@ +From e4facfcd2947b5277789d58a452090b61ca2d383 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Wed, 21 Feb 2024 15:04:27 +0800 +Subject: [PATCH 16/43] add more detailed log information for load sandbox + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/sandbox/sandbox_manager.cc | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/daemon/sandbox/sandbox_manager.cc b/src/daemon/sandbox/sandbox_manager.cc +index d3db4fb4..cee444f4 100644 +--- a/src/daemon/sandbox/sandbox_manager.cc ++++ b/src/daemon/sandbox/sandbox_manager.cc +@@ -461,7 +461,7 @@ auto SandboxManager::LoadSandbox(std::string &id) -> std::shared_ptr<Sandbox> + Errors tmpError; + + if (!sandbox->Load(tmpError)) { +- ERROR("Failed to load subdir:%s", id.c_str()); ++ ERROR("Failed to load subdir:%s: %s", id.c_str(), tmpError.GetMessage().c_str()); + return nullptr; + } + +-- +2.34.1 + diff --git a/0017-bugfix-for-the-concurrency-competition-between-the-r.patch b/0017-bugfix-for-the-concurrency-competition-between-the-r.patch new file mode 100644 index 0000000..796370b --- /dev/null +++ b/0017-bugfix-for-the-concurrency-competition-between-the-r.patch @@ -0,0 +1,83 @@ +From 0099190e7f18e890185e36c5a657e9ce95179bc8 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Fri, 1 Mar 2024 15:04:09 +0800 +Subject: [PATCH 17/43] bugfix for the concurrency competition between the + reuse layer and the creation layer + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/modules/image/oci/oci_load.c | 9 ++++++++- + src/daemon/modules/image/oci/registry/registry.c | 9 ++++++++- + 2 files changed, 16 insertions(+), 2 deletions(-) + +diff --git a/src/daemon/modules/image/oci/oci_load.c b/src/daemon/modules/image/oci/oci_load.c +index 534e2647..85127f35 100644 +--- a/src/daemon/modules/image/oci/oci_load.c ++++ b/src/daemon/modules/image/oci/oci_load.c +@@ -680,6 +680,12 @@ static int oci_load_set_layers_info(load_image_t *im, const image_manifest_items + char *parent_chain_id_sha256 = ""; + char *id = NULL; + char *parent_chain_id = NULL; ++ // exist_flag is used to mark whether a non-existent layer has been encountered during this layer reuse process. ++ // 1.exist_flag is true if the layers are currently reusable; ++ // 2.exist_flag is false if encounter an uncreated layer that cannot be reused ++ // Prevent concurrent competition between the creation layer function ++ // and the reuse layer function on the im -> layer_of_hold_refs variable ++ bool exist_flag = true; + + if (im == NULL || manifest == NULL || dstdir == NULL) { + ERROR("Invalid input params image or manifest is null"); +@@ -761,7 +767,7 @@ static int oci_load_set_layers_info(load_image_t *im, const image_manifest_items + goto out; + } + +- if (storage_inc_hold_refs(id) == 0) { ++ if (exist_flag && storage_inc_hold_refs(id) == 0) { + free(im->layer_of_hold_refs); + im->layer_of_hold_refs = util_strdup_s(id); + if (parent_chain_id != NULL && storage_dec_hold_refs(parent_chain_id) != 0) { +@@ -781,6 +787,7 @@ static int oci_load_set_layers_info(load_image_t *im, const image_manifest_items + continue; + } + ++ exist_flag = false; + if (check_and_set_digest_from_tarball(im->layers[i], conf->rootfs->diff_ids[i]) != 0) { + ERROR("Check layer digest failed"); + ret = -1; +diff --git a/src/daemon/modules/image/oci/registry/registry.c b/src/daemon/modules/image/oci/registry/registry.c +index aed3057a..66fa0076 100644 +--- a/src/daemon/modules/image/oci/registry/registry.c ++++ b/src/daemon/modules/image/oci/registry/registry.c +@@ -1516,6 +1516,12 @@ static int fetch_all(pull_descriptor *desc) + struct layer_list *list = NULL; + pthread_t tid = 0; + struct timespec ts = { 0 }; ++ // exist_flag is used to mark whether a non-existent layer has been encountered during this layer reuse process. ++ // 1.exist_flag is true if the layers are currently reusable; ++ // 2.exist_flag is false if encounter an uncreated layer that cannot be reused ++ // Prevent concurrent competition between the creation layer function ++ // and the reuse layer function on the im -> layer_of_hold_refs variable ++ bool exist_flag = true; + + if (desc == NULL) { + ERROR("Invalid NULL param"); +@@ -1547,7 +1553,7 @@ static int fetch_all(pull_descriptor *desc) + + // Skip layer that already exist in local store + list = storage_layers_get_by_compress_digest(desc->layers[i].digest); +- if (list != NULL) { ++ if (exist_flag && list != NULL) { + for (j = 0; j < list->layers_len; j++) { + if ((list->layers[j]->parent == NULL && i == 0) || + (parent_chain_id != NULL && list->layers[j]->parent != NULL && +@@ -1579,6 +1585,7 @@ static int fetch_all(pull_descriptor *desc) + continue; + } + } ++ exist_flag = false; + + // parent_chain_id = NULL means no parent chain match from now on, so no longer need + // to get layers by compressed digest to reuse layer. +-- +2.34.1 + diff --git a/0018-add-concurrent-load-test.patch b/0018-add-concurrent-load-test.patch new file mode 100644 index 0000000..2256aba --- /dev/null +++ b/0018-add-concurrent-load-test.patch @@ -0,0 +1,73 @@ +From 2af906d42a155a7b779dce017a2779b96dba2b61 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Fri, 1 Mar 2024 15:04:35 +0800 +Subject: [PATCH 18/43] add concurrent load test + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + CI/test_cases/image_cases/image_load.sh | 47 +++++++++++++++++++++++++ + 1 file changed, 47 insertions(+) + +diff --git a/CI/test_cases/image_cases/image_load.sh b/CI/test_cases/image_cases/image_load.sh +index 52b713d4..a2cada5f 100755 +--- a/CI/test_cases/image_cases/image_load.sh ++++ b/CI/test_cases/image_cases/image_load.sh +@@ -79,8 +79,55 @@ function test_image_load() + return ${ret} + } + ++function test_concurrent_load() ++{ ++ local ret=0 ++ local test="isula load image test => (${FUNCNAME[@]})" ++ ++ msg_info "${test} starting..." ++ ++ # clean exist image ++ ubuntu_id=`isula inspect -f '{{.image.id}}' ubuntu` ++ busybox_id=`isula inspect -f '{{.image.id}}' busybox` ++ isula rmi $ubuntu_id $busybox_id ++ ++ concurrent_time=10 ++ for i in `seq 1 $concurrent_time` ++ do ++ isula load -i $mult_image & ++ pids[$i]=$! ++ done ++ ++ for i in `seq 1 $concurrent_time`;do ++ wait ${pids[$i]} ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - fail to do isulad load $i" && ((ret++)) ++ done ++ ++ ubuntu_id=`isula inspect -f '{{.image.id}}' ubuntu` ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - fail to inspect image: ubuntu" && ((ret++)) ++ ++ top_layer_id=$(isula inspect -f '{{.image.top_layer}}' ${ubuntu_id}) ++ ++ busybox_id=`isula inspect -f '{{.image.id}}' busybox` ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - fail to inspect image: busybox" && ((ret++)) ++ ++ # delete image after concurrent load ++ isula rmi $ubuntu_id $busybox_id ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to remove image ${ubuntu_id} and ${busybox_id}" && ((ret++)) ++ ++ ls -l /var/lib/isulad/storage/overlay-layers ++ local top_layer_dir=/var/lib/isulad/storage/overlay-layers/${top_layer_id} ++ test -e ${top_layer_dir} ++ [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - top layer dir ${top_layer_id} exist after delete image" && ((ret++)) ++ ++ msg_info "${test} finished with return ${ret}..." ++ return ${ret} ++} ++ + declare -i ans=0 + ++test_concurrent_load || ((ans++)) ++ + test_image_load || ((ans++)) + + show_result ${ans} "${curr_path}/${0}" +-- +2.34.1 + diff --git a/0019-get-the-realpath-of-the-host-path-for-archive-when-c.patch b/0019-get-the-realpath-of-the-host-path-for-archive-when-c.patch new file mode 100644 index 0000000..e1a2fda --- /dev/null +++ b/0019-get-the-realpath-of-the-host-path-for-archive-when-c.patch @@ -0,0 +1,63 @@ +From 96dfd32ee5d9a133ad63af13723402f10cd7cf7b Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Mon, 11 Mar 2024 15:50:45 +0800 +Subject: [PATCH 19/43] get the realpath of the host path for archive when cp + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/utils/tar/isulad_tar.c | 16 ++++++++++++++-- + 1 file changed, 14 insertions(+), 2 deletions(-) + +diff --git a/src/utils/tar/isulad_tar.c b/src/utils/tar/isulad_tar.c +index bbe4c3b2..fe514acc 100644 +--- a/src/utils/tar/isulad_tar.c ++++ b/src/utils/tar/isulad_tar.c +@@ -390,6 +390,7 @@ int archive_copy_to(const struct io_read_wrapper *content, const struct archive_ + { + int ret = -1; + struct archive_copy_info *dstinfo = NULL; ++ char cleanpath[PATH_MAX] = { 0 }; + char *dstdir = NULL; + char *src_base = NULL; + char *dst_base = NULL; +@@ -410,7 +411,12 @@ int archive_copy_to(const struct io_read_wrapper *content, const struct archive_ + goto cleanup; + } + +- ret = archive_chroot_untar_stream(content, dstdir, ".", src_base, dst_base, root_dir, err); ++ if (realpath(dstdir, cleanpath) == NULL) { ++ ERROR("Failed to get real path for %s", dstdir); ++ return -1; ++ } ++ ++ ret = archive_chroot_untar_stream(content, cleanpath, ".", src_base, dst_base, root_dir, err); + + cleanup: + free_archive_copy_info(dstinfo); +@@ -428,6 +434,7 @@ static int tar_resource_rebase(const char *path, const char *rebase, const char + struct stat st; + char *srcdir = NULL; + char *srcbase = NULL; ++ char cleanpath[PATH_MAX] = { 0 }; + + if (lstat(path, &st) < 0) { + SYSERROR("lstat %s failed", path); +@@ -438,9 +445,14 @@ static int tar_resource_rebase(const char *path, const char *rebase, const char + ERROR("Can not split path: %s", path); + goto cleanup; + } ++ ++ if (realpath(srcdir, cleanpath) == NULL) { ++ ERROR("Failed to get real path for %s", srcdir); ++ return -1; ++ } + + DEBUG("chroot tar stream srcdir(%s) srcbase(%s) rebase(%s)", srcdir, srcbase, rebase); +- nret = archive_chroot_tar_stream(srcdir, srcbase, srcbase, rebase, root_dir, archive_reader); ++ nret = archive_chroot_tar_stream(cleanpath, srcbase, srcbase, rebase, root_dir, archive_reader); + if (nret < 0) { + ERROR("Can not archive path: %s", path); + goto cleanup; +-- +2.34.1 + diff --git a/0020-bugfix-for-wrong-goto-branch.patch b/0020-bugfix-for-wrong-goto-branch.patch new file mode 100644 index 0000000..7c8318c --- /dev/null +++ b/0020-bugfix-for-wrong-goto-branch.patch @@ -0,0 +1,35 @@ +From c67760ce928f67d9a8beeaf2e2d51c8f2239f69e Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Tue, 12 Mar 2024 11:15:26 +0800 +Subject: [PATCH 20/43] bugfix for wrong goto branch + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/utils/tar/isulad_tar.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/src/utils/tar/isulad_tar.c b/src/utils/tar/isulad_tar.c +index fe514acc..13343922 100644 +--- a/src/utils/tar/isulad_tar.c ++++ b/src/utils/tar/isulad_tar.c +@@ -413,7 +413,7 @@ int archive_copy_to(const struct io_read_wrapper *content, const struct archive_ + + if (realpath(dstdir, cleanpath) == NULL) { + ERROR("Failed to get real path for %s", dstdir); +- return -1; ++ goto cleanup; + } + + ret = archive_chroot_untar_stream(content, cleanpath, ".", src_base, dst_base, root_dir, err); +@@ -448,7 +448,7 @@ static int tar_resource_rebase(const char *path, const char *rebase, const char + + if (realpath(srcdir, cleanpath) == NULL) { + ERROR("Failed to get real path for %s", srcdir); +- return -1; ++ goto cleanup; + } + + DEBUG("chroot tar stream srcdir(%s) srcbase(%s) rebase(%s)", srcdir, srcbase, rebase); +-- +2.34.1 + diff --git a/0021-bugfix-for-wrong-dynamic-allocation-object-type.patch b/0021-bugfix-for-wrong-dynamic-allocation-object-type.patch new file mode 100644 index 0000000..bb08db6 --- /dev/null +++ b/0021-bugfix-for-wrong-dynamic-allocation-object-type.patch @@ -0,0 +1,29 @@ +From 0ef23c6caae4a97228705574b0c8f3445c6e65dc Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?=E8=AE=B8=E7=BA=A2=E5=BC=BA?= <277922995@qq.com> +Date: Wed, 13 Mar 2024 17:00:16 +0800 +Subject: [PATCH 21/43] bugfix for wrong dynamic allocation object type +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Signed-off-by: 许红强 <277922995@qq.com> +--- + src/daemon/modules/image/image.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/daemon/modules/image/image.c b/src/daemon/modules/image/image.c +index e7aa81b8..871f5f39 100644 +--- a/src/daemon/modules/image/image.c ++++ b/src/daemon/modules/image/image.c +@@ -1535,7 +1535,7 @@ int im_tag_image(const im_tag_request *request, im_tag_response **response) + return -1; + } + +- *response = util_common_calloc_s(sizeof(im_remove_response)); ++ *response = util_common_calloc_s(sizeof(im_tag_response)); + if (*response == NULL) { + ERROR("Out of memory"); + return -1; +-- +2.34.1 + diff --git a/0022-add-swap-usage-in-cri.patch b/0022-add-swap-usage-in-cri.patch new file mode 100644 index 0000000..cb773e6 --- /dev/null +++ b/0022-add-swap-usage-in-cri.patch @@ -0,0 +1,80 @@ +From ed569ccbf7e5029e83c40521255e0e406f285bae Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Fri, 12 Jan 2024 11:31:59 +0800 +Subject: [PATCH 22/43] add swap usage in cri + +Signed-off-by: jikai <jikai11@huawei.com> +--- + .../entry/cri/v1/v1_cri_container_manager_service.cc | 11 +++++++++++ + src/daemon/executor/container_cb/execution_extend.c | 2 ++ + src/daemon/modules/api/runtime_api.h | 3 +++ + src/daemon/modules/runtime/isula/isula_rt_ops.c | 4 ++++ + 4 files changed, 20 insertions(+) + +diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +index 2dda1e16..47a33c2c 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +@@ -890,6 +890,17 @@ void ContainerManagerService::ContainerStatsToGRPC( + if (response->container_stats[i]->major_page_faults != 0u) { + container->mutable_memory()->mutable_major_page_faults()->set_value(response->container_stats[i]->major_page_faults); + } ++ ++ // Swap ++ container->mutable_swap()->set_timestamp(timestamp); ++ if (response->container_stats[i]->swap_used != 0u) { ++ container->mutable_swap()->mutable_swap_usage_bytes()->set_value(response->container_stats[i]->swap_used); ++ } ++ if (response->container_stats[i]->swap_limit >= response->container_stats[i]->swap_used) { ++ container->mutable_swap()->mutable_swap_available_bytes()->set_value(response->container_stats[i]->swap_limit ++ - response->container_stats[i]->swap_used); ++ } ++ + containerstats.push_back(std::move(container)); + } + } +diff --git a/src/daemon/executor/container_cb/execution_extend.c b/src/daemon/executor/container_cb/execution_extend.c +index 25ec5d3b..52401633 100644 +--- a/src/daemon/executor/container_cb/execution_extend.c ++++ b/src/daemon/executor/container_cb/execution_extend.c +@@ -259,6 +259,8 @@ static container_info *get_container_stats(const container_t *cont, + info->major_page_faults = einfo->major_page_faults; + info->kmem_used = einfo->kmem_used; + info->kmem_limit = einfo->kmem_limit; ++ info->swap_used = einfo->swap_used; ++ info->swap_limit = einfo->swap_limit; + info->timestamp = util_get_now_time_nanos(); + + // workingset is zero if memory used < total inactive file +diff --git a/src/daemon/modules/api/runtime_api.h b/src/daemon/modules/api/runtime_api.h +index d2679c2d..bd170c30 100644 +--- a/src/daemon/modules/api/runtime_api.h ++++ b/src/daemon/modules/api/runtime_api.h +@@ -66,6 +66,9 @@ struct runtime_container_resources_stats_info { + uint64_t cache; + uint64_t cache_total; + uint64_t inactive_file_total; ++ /* Swap usage*/ ++ uint64_t swap_used; ++ uint64_t swap_limit; + }; + + typedef struct _rt_create_params_t { +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 2c92cc59..745154bb 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -647,6 +647,10 @@ static void transform_stats_info_from_runtime(shim_client_runtime_stats *stats, + info->page_faults = memory->raw->pgfault; + info->major_page_faults = memory->raw->pgmajfault; + } ++ if (memory != NULL && memory->swap != NULL) { ++ info->swap_used = memory->swap->usage; ++ info->swap_limit = memory->swap->limit; ++ } + shim_client_runtime_stats_data_blkio *blkio = stats->data->blkio; + if (blkio == NULL) { + return; +-- +2.34.1 + diff --git a/0023-add-benchmark-result-of-perf-test-in-cri.patch b/0023-add-benchmark-result-of-perf-test-in-cri.patch new file mode 100644 index 0000000..92efaeb --- /dev/null +++ b/0023-add-benchmark-result-of-perf-test-in-cri.patch @@ -0,0 +1,256 @@ +From 16a0cf7e9c2c059cb5537f48a022e63df457f186 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Sat, 2 Mar 2024 11:49:08 +0800 +Subject: [PATCH 23/43] add benchmark, result of perf test in cri + +Signed-off-by: jikai <jikai11@huawei.com> +--- + docs/performance/cri_performance_test.md | 60 ++++++++ + tools/benchmark/cri_perf_test.sh | 168 +++++++++++++++++++++++ + 2 files changed, 228 insertions(+) + create mode 100644 docs/performance/cri_performance_test.md + create mode 100755 tools/benchmark/cri_perf_test.sh + +diff --git a/docs/performance/cri_performance_test.md b/docs/performance/cri_performance_test.md +new file mode 100644 +index 00000000..594f0da7 +--- /dev/null ++++ b/docs/performance/cri_performance_test.md +@@ -0,0 +1,60 @@ ++## Machine configuration ++ ++ARM machine: ++ ++| Configuration | Information | ++| ------------- | -------------------------------------- | ++| OS | openEuler 22.03-LTS | ++| Kernel | linux 5.10.0-136.12.0.86.oe2203.aarch64 | ++| CPU | 96 cores | ++| Memory | 128 GB | ++ ++## Version of Softwares ++ ++| Name | Version | ++| ------ | ------------------------------------------------------------ | ++| iSulad | Version: 2.1.5 , Git commit: 5ebca976dd591a5676527be1bde950e5ce93eac0 | ++| containerd | Version: v2.0.0-beta.2, Git commit: 290194fe77d48521d3ea78ec02e2e406c4bf91b6 | ++| crio | version: 1.30.0, Git commit: b43e0d63a8af3277dbfc555f62d07bb2305a72c7 | ++ ++## Test tool ++ ++tools/benchmark/cri_perf_test.sh ++ ++## Compare with other container engines ++ ++### run operator once ++ ++#### ARM ++ ++run 1 pod and 1 container ++ ++| measure | iSulad | containerd | crio | vs containerd | vs crio | ++| ----------------- | ------ | ------ | ------ | ------ | ------ | ++| time(ms) | 580 | 812 | 567 | -28.5% | 2.3% | ++| engine mem(kb) | 38704 | 66806 | 58760 | -42.0% | -34.2% | ++| shim mem(kb) | 1700 | 13876 | 4648 | -87.7% | -63.4% | ++ ++run 10 pods and 10 containers ++ ++| measure | iSulad | containerd | crio | vs containerd | vs crio | ++| ----------------- | ------ | ------ | ------ | ------ | ------ | ++| time(ms) | 1141 | 4000 | 1749 | -71.5% | -34.8% | ++| engine mem(kb) | 47688 | 82580 | 86128 | -42.2% | -44.6% | ++| shim mem(kb) | 16764 | 154872 | 46836 | -89.2% | -64.2% | ++ ++run 50 pods and 50 containers ++ ++| measure | iSulad | containerd | crio | vs containerd | vs crio | ++| ----------------- | ------ | ------ | ------ | ------ | ------ | ++| time(ms) | 4544 | 19963 | 8503 | -77.2% | -46.9% | ++| engine mem(kb) | 88700 | 134384 | 115560 | -34.0% | -23.2% | ++| shim mem(kb) | 83892 | 750924 | 233480 | -88.8% | -64.0% | ++ ++run 100 pods and 100 containers ++ ++| measure | iSulad | containerd | crio | vs containerd | vs crio | ++| ----------------- | ------ | ------ | ------ | ------ | ------ | ++| time(ms) | 10012 | 39629 | 18278 | -74.7% | -45.5% | ++| engine mem(kb) | 148464 | 185700 | 147836 | -20.0% | 0.4% | ++| shim mem(kb) | 168420 | 1506268| 462000 | -88.8% | -63.3% | +diff --git a/tools/benchmark/cri_perf_test.sh b/tools/benchmark/cri_perf_test.sh +new file mode 100755 +index 00000000..54ee24f5 +--- /dev/null ++++ b/tools/benchmark/cri_perf_test.sh +@@ -0,0 +1,168 @@ ++#!/bin/bash ++####################################################################### ++##- Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. ++# - iSulad licensed under the Mulan PSL v2. ++# - You can use this software according to the terms and conditions of the Mulan PSL v2. ++# - You may obtain a copy of Mulan PSL v2 at: ++# - http://license.coscl.org.cn/MulanPSL2 ++# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++# - PURPOSE. ++# - See the Mulan PSL v2 for more details. ++##- @Description: perf test ++##- @Author: jikai ++##- @Create: 2024-02-29 ++####################################################################### ++ ++# cri_perf_test.sh -e $engine -p $parallel ++ ++engine=isulad ++runtime="unix:///var/run/isulad.sock" ++shim="isulad-shim" ++parallel=1 ++while getopts ":e:p:" opt ++do ++ case $opt in ++ e) ++ engine=${OPTARG} ++ # compare if OPTARG is in ["isulad", "containerd", "crio"]} ++ if [ $engine == "isulad" ]; then ++ runtime="unix:///var/run/isulad.sock" ++ shim="isulad-shim" ++ elif [ $engine == "containerd" ]; then ++ runtime="unix:///var/run/containerd/containerd.sock" ++ shim="containerd-shim" ++ elif [ $engine == "crio" ]; then ++ runtime="unix:///var/run/crio/crio.sock" ++ shim="conmon" ++ else ++ echo "Unknown engine: ${OPTARG}, only support isulad, containerd, crio." ++ exit 1 ++ fi ++ ;; ++ p) ++ parallel=${OPTARG} ++ ;; ++ ?) ++ echo "Unknown parameter" ++ exit 1;; ++ esac ++done ++ ++workdir="$(pwd)" ++tmpdir="$workdir/cri_perf_test_tmpdata" ++mkdir -p $tmpdir/container/ ++mkdir -p $tmpdir/pod/ ++mkdir -p $workdir/cri_perf_test_result/ ++result_data=$workdir/cri_perf_test_result/${engine}-${parallel}-result.dat ++rm -f $result_data ++ ++# Get the interval time(ms) ++function getTiming(){ ++ start=$1 ++ end=$2 ++ ++ start_s=$(echo $start | cut -d '.' -f 1) ++ start_ns=$(echo $start | cut -d '.' -f 2) ++ end_s=$(echo $end | cut -d '.' -f 1) ++ end_ns=$(echo $end | cut -d '.' -f 2) ++ ++ time=$(( ( 10#$end_s - 10#$start_s ) * 1000 + ( 10#$end_ns / 1000000 - 10#$start_ns / 1000000 ) )) ++ ++ echo "$time" ++} ++ ++# Kill all pods and containers running ++crictl --runtime-endpoint $runtime rmp -af ++ ++# Create $parallel container.json and pod.json ++for((i=0;i<$parallel;i++)) ++do ++ cat > $tmpdir/container/container_$i.json << EOF ++{ ++ "metadata": { ++ "name": "testcontainer$i" ++ }, ++ "image": { ++ "image": "busybox" ++ }, ++ "command": [ ++ "/bin/sh", "-c", "sleep 1d" ++ ], ++ "log_path": "console$i.log", ++ "linux": { ++ "security_context": { ++ "capabilities": {} ++ } ++ } ++} ++EOF ++ ++ cat > $tmpdir/pod/pod_$i.json <<EOF ++{ ++ "metadata": { ++ "name": "testpod$i", ++ "namespace": "testns", ++ "uid": "b49ef5ee-ee30-11ed-a05b-0242ac120003", ++ "attempt": 1 ++ }, ++ "log_directory": "/tmp", ++ "linux": { ++ "security_context": { ++ "capabilities": {} ++ } ++ } ++} ++EOF ++done ++ ++# get start time ++start_time=$(date +%s.%N) ++ ++engine_pid=$(pidof $engine) ++ ++for((i=0;i<$parallel;i++)) ++do ++ crictl --runtime-endpoint $runtime run --no-pull $tmpdir/container/container_$i.json $tmpdir/pod/pod_$i.json & ++done ++ ++# wait for all the containers to finish and get end time ++end_time=$(date +%s.%N) ++boot_time=$(getTiming $start_time $end_time) ++a=`crictl --runtime-endpoint $runtime ps | grep testcontainer | wc -l` ++while [ $a -ne $parallel ]; ++do ++ a=`crictl --runtime-endpoint $runtime ps | grep testcontainer | wc -l` ++ end_time=$(date +%s.%N) ++ boot_time=$(getTiming $start_time $end_time) ++ if [ $boot_time -gt 2000000 ]; then ++ break ++ fi ++done ++ ++if [ ${boot_time} -lt 2000000 ]; then ++ echo "BootTime: ${boot_time}ms" ++ # Output to the corresponding file ++ echo "time: ${boot_time}" >> ${result_data} ++else ++ echo "${boot_time}ms is too long, please check the environment." ++fi ++ ++# get pids ++shim_pids=$(ps -ef | grep -v grep | grep -i $shim | awk '{print$2}') ++ ++# calc memory of pids ++engine_mem=$(cat /proc/$engine_pid/status | grep VmRSS | awk '{print $2}') ++shim_mem=0 ++for pid in $shim_pids ++do ++ let shim_mem+=$(cat /proc/$pid/status | grep VmRSS | awk '{print $2}') ++done ++echo "Engine Mem: ${engine_mem}KB" ++echo "engine-mem: ${engine_mem}" >> ${result_data} ++echo "Shim Mem Total: ${shim_mem}KB" ++echo "shim-mem: ${shim_mem}" >> ${result_data} ++ ++# clean resources ++crictl --runtime-endpoint $runtime rmp -af ++rm -rf $tmpdir +-- +2.34.1 + diff --git a/0024-add-support-for-systemd-cgroup-driver.patch b/0024-add-support-for-systemd-cgroup-driver.patch new file mode 100644 index 0000000..d6c363b --- /dev/null +++ b/0024-add-support-for-systemd-cgroup-driver.patch @@ -0,0 +1,523 @@ +From 167af3ce0cff3906c9976b249432d41167b15eb2 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Thu, 11 Jan 2024 17:06:57 +0800 +Subject: [PATCH 24/43] add support for systemd cgroup driver + +Signed-off-by: jikai <jikai11@huawei.com> +--- + src/cmd/isulad-shim/process.c | 6 + + src/cmd/isulad/isulad_commands.h | 2 + + src/contrib/config/daemon.json | 3 +- + src/daemon/common/cgroup.h | 4 + + src/daemon/common/cgroup_v1.c | 116 ++++++++++++++++++ + src/daemon/config/isulad_config.c | 23 ++++ + src/daemon/config/isulad_config.h | 2 + + .../cri/v1/cri_v1_runtime_runtime_service.cc | 25 ++++ + .../cri/v1/cri_v1_runtime_runtime_service.h | 4 + + .../cri/v1/v1_cri_runtime_manager_service.cc | 12 ++ + .../cri/v1/v1_cri_runtime_manager_service.h | 2 + + .../entry/cri/v1/v1_cri_runtime_service.h | 2 + + .../cri/v1/v1_cri_runtime_service_impl.cc | 5 + + .../cri/v1/v1_cri_runtime_service_impl.h | 2 + + src/daemon/executor/container_cb/execution.c | 26 ++++ + .../modules/runtime/engines/lcr/lcr_rt_ops.c | 7 ++ + .../modules/runtime/isula/isula_rt_ops.c | 1 + + src/daemon/modules/runtime/shim/shim_rt_ops.c | 6 + + src/daemon/modules/spec/specs.c | 22 +++- + 19 files changed, 268 insertions(+), 2 deletions(-) + +diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c +index bf67c414..8a4ca175 100644 +--- a/src/cmd/isulad-shim/process.c ++++ b/src/cmd/isulad-shim/process.c +@@ -1287,6 +1287,12 @@ static void get_runtime_cmd(process_t *p, const char *log_path, const char *pid_ + params[i++] = p->state->cwd; + } + } else { ++ // the --systemd-cgroup argument is not in oci spec, but up to now, ++ // the latest version of runc, crun, youki, runsc, kata-runtime all support this argument ++ // should ensure that this is supported for oci runtime ++ if (p->state->systemd_cgroup) { ++ params[i++] = "--systemd-cgroup"; ++ } + params[i++] = "create"; + params[i++] = "--bundle"; + params[i++] = p->bundle; +diff --git a/src/cmd/isulad/isulad_commands.h b/src/cmd/isulad/isulad_commands.h +index ba4b838d..cf5f65f5 100644 +--- a/src/cmd/isulad/isulad_commands.h ++++ b/src/cmd/isulad/isulad_commands.h +@@ -226,6 +226,8 @@ int command_default_ulimit_append(command_option_t *option, const char *arg); + &(cmdargs)->json_confs->storage_opts, \ + "Storage driver options", \ + command_append_array }, \ ++ { CMD_OPT_TYPE_BOOL, false, "systemd-cgroup", 0, &(cmdargs)->json_confs->systemd_cgroup, \ ++ "Use systemd cgroup driver(default false)", NULL }, \ + SUP_GROUPS_OPT(cmdargs) \ + { CMD_OPT_TYPE_CALLBACK, \ + false, \ +diff --git a/src/contrib/config/daemon.json b/src/contrib/config/daemon.json +index 966e016a..69362c26 100644 +--- a/src/contrib/config/daemon.json ++++ b/src/contrib/config/daemon.json +@@ -36,5 +36,6 @@ + "cri-runtimes": { + "kata": "io.containerd.kata.v2" + }, +- "enable-cri-v1": false ++ "enable-cri-v1": false, ++ "systemd-cgroup": false + } +diff --git a/src/daemon/common/cgroup.h b/src/daemon/common/cgroup.h +index 6664fb15..fa20f42c 100644 +--- a/src/daemon/common/cgroup.h ++++ b/src/daemon/common/cgroup.h +@@ -141,6 +141,10 @@ typedef struct { + + int common_get_cgroup_v1_metrics(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics); + ++char *common_get_init_cgroup(const char *subsystem); ++ ++char *common_get_own_cgroup(const char *subsystem); ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/common/cgroup_v1.c b/src/daemon/common/cgroup_v1.c +index 9004becc..e38fc03e 100644 +--- a/src/daemon/common/cgroup_v1.c ++++ b/src/daemon/common/cgroup_v1.c +@@ -554,3 +554,119 @@ int common_get_cgroup_v1_metrics(const char *cgroup_path, cgroup_metrics_t *cgro + return 0; + } + ++/* parse cgroup files, such as /proc/self/cgroup or /proc/1/cgroup */ ++static int parse_cgroup_file(const char *path, char ***nlist, char ***plist) ++{ ++ int ret = 0; ++ size_t length = 0; ++ __isula_auto_file FILE *fp = NULL; ++ __isula_auto_free char *pline = NULL; ++ ++ fp = util_fopen(path, "r"); ++ if (fp == NULL) { ++ return -1; ++ } ++ ++ while (getline(&pline, &length, fp) != -1) { ++ char *pos = NULL; ++ char *pos2 = NULL; ++ char *pos3 = NULL; ++ char *ptoken = NULL; ++ char *psave = NULL; ++ pos = strchr(pline, ':'); ++ if (pos == NULL) { ++ ERROR("Invalid cgroup entry: must contain at least two colons: %s", pline); ++ ret = -1; ++ goto out; ++ } ++ pos++; ++ pos2 = strchr(pos, ':'); ++ if (pos2 == NULL) { ++ ERROR("Invalid cgroup entry: must contain at least two colons: %s", pline); ++ ret = -1; ++ goto out; ++ } ++ pos3 = strchr(pos2, '\n'); ++ if (pos3 != NULL) { ++ *pos3 = '\0'; ++ } ++ *pos2 = '\0'; ++ ++ if ((pos2 - pos) == 0) { ++ INFO("Cgroup entry: %s not supported by cgroup v1", pline); ++ continue; ++ } ++ ++ for (ptoken = strtok_r(pos, ",", &psave); ptoken; ptoken = strtok_r(NULL, ",", &psave)) { ++ ret = util_array_append(nlist, ptoken); ++ if (ret != 0) { ++ ERROR("Failed to append string"); ++ goto out; ++ } ++ ++ ret = util_array_append(plist, pos2 + 1); ++ if (ret != 0) { ++ ERROR("Failed to append string"); ++ goto out; ++ } ++ } ++ } ++ ++out: ++ if (ret != 0) { ++ util_free_array(*nlist); ++ *nlist = NULL; ++ util_free_array(*plist); ++ *plist = NULL; ++ } ++ return ret; ++} ++ ++static char *common_get_cgroup_path(const char *path, const char *subsystem) ++{ ++ char **nlist = NULL, **plist = NULL; ++ size_t i = 0; ++ char *res = NULL; ++ if (path == NULL) { ++ ERROR("Invalid NULL param"); ++ return NULL; ++ } ++ ++ if (parse_cgroup_file(path, &nlist, &plist) < 0) { ++ return NULL; ++ } ++ ++ for (i = 0; i < util_array_len((const char **)nlist); i++) { ++ const char *prefix = "name="; ++ bool find_sub = (strcmp(nlist[i], subsystem) == 0 || (strncmp(nlist[i], prefix, strlen(prefix)) == 0 ++ && strcmp(nlist[i]+strlen(prefix), subsystem) == 0)); ++ if (find_sub) { ++ res = util_strdup_s(plist[i]); ++ break; ++ } ++ } ++ ++ util_free_array(nlist); ++ util_free_array(plist); ++ return res; ++} ++ ++char *common_get_init_cgroup(const char *subsystem) ++{ ++ if (common_get_cgroup_version() != CGROUP_VERSION_1) { ++ ERROR("Not implemented for cgroup v2 hierarchy"); ++ return NULL; ++ } ++ ++ return common_get_cgroup_path("/proc/1/cgroup", subsystem); ++} ++ ++char *common_get_own_cgroup(const char *subsystem) ++{ ++ if (common_get_cgroup_version() != CGROUP_VERSION_1) { ++ ERROR("Not implemented for cgroup v2 hierarchy"); ++ return NULL; ++ } ++ ++ return common_get_cgroup_path("/proc/self/cgroup", subsystem); ++} +diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c +index 51758adb..8179558e 100644 +--- a/src/daemon/config/isulad_config.c ++++ b/src/daemon/config/isulad_config.c +@@ -1762,6 +1762,8 @@ int merge_json_confs_into_global(struct service_arguments *args) + args->json_confs->enable_cri_v1 = tmp_json_confs->enable_cri_v1; + #endif + ++ args->json_confs->systemd_cgroup = tmp_json_confs->systemd_cgroup; ++ + if (merge_cri_runtimes_into_global(args, tmp_json_confs)) { + ret = -1; + goto out; +@@ -1895,3 +1897,24 @@ isulad_daemon_constants *get_isulad_daemon_constants(void) + { + return g_isulad_daemon_constants; + } ++ ++bool conf_get_systemd_cgroup() ++{ ++ bool systemd_cgroup = false; ++ struct service_arguments *conf = NULL; ++ ++ if (isulad_server_conf_rdlock() != 0) { ++ return false; ++ } ++ ++ conf = conf_get_server_conf(); ++ if (conf == NULL || conf->json_confs == NULL) { ++ goto out; ++ } ++ ++ systemd_cgroup = conf->json_confs->systemd_cgroup; ++ ++out: ++ (void)isulad_server_conf_unlock(); ++ return systemd_cgroup; ++} +diff --git a/src/daemon/config/isulad_config.h b/src/daemon/config/isulad_config.h +index 459ea331..f29cd564 100644 +--- a/src/daemon/config/isulad_config.h ++++ b/src/daemon/config/isulad_config.h +@@ -114,6 +114,8 @@ char *conf_get_isulad_monitor_fifo_path(void); + int init_isulad_daemon_constants(void); + isulad_daemon_constants *get_isulad_daemon_constants(void); + ++bool conf_get_systemd_cgroup(void); ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc +index ba9459f6..76e393f3 100644 +--- a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc ++++ b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc +@@ -632,3 +632,28 @@ grpc::Status RuntimeV1RuntimeServiceImpl::Status(grpc::ServerContext *context, + + return grpc::Status::OK; + } ++ ++grpc::Status ++RuntimeV1RuntimeServiceImpl::RuntimeConfig(grpc::ServerContext *context, ++ const runtime::v1::RuntimeConfigRequest *request, ++ runtime::v1::RuntimeConfigResponse *reply) ++{ ++ Errors error; ++ ++ if (request == nullptr) { ++ ERROR("Invalid input arguments"); ++ return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, "Invalid input arguments"); ++ } ++ ++ EVENT("Event: {Object: CRI, Type: Runtime Config}"); ++ ++ m_rService->RuntimeConfig(reply, error); ++ if (!error.Empty()) { ++ ERROR("Object: CRI, Type: Failed to get runtime config:%s", error.GetMessage().c_str()); ++ return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ } ++ ++ EVENT("Event: {Object: CRI, Type: Runtime Config}"); ++ ++ return grpc::Status::OK; ++} +diff --git a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h +index 09ebe68b..52cc6b99 100644 +--- a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h ++++ b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h +@@ -101,6 +101,10 @@ public: + grpc::Status Status(grpc::ServerContext *context, const runtime::v1::StatusRequest *request, + runtime::v1::StatusResponse *reply) override; + ++ grpc::Status RuntimeConfig(grpc::ServerContext *context, ++ const runtime::v1::RuntimeConfigRequest *request, ++ runtime::v1::RuntimeConfigResponse *reply) override; ++ + private: + std::unique_ptr<CRIV1::CRIRuntimeService> m_rService; + }; +diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.cc +index 1afac3e2..de489aae 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.cc +@@ -16,6 +16,7 @@ + #include "isula_libutils/log.h" + #include "v1_cri_helpers.h" + #include "cri_helpers.h" ++#include "isulad_config.h" + + namespace CRIV1 { + void RuntimeManagerService::UpdateRuntimeConfig(const runtime::v1::RuntimeConfig &config, Errors & /*error*/) +@@ -69,4 +70,15 @@ auto RuntimeManagerService::Status(Errors &error) -> std::unique_ptr<runtime::v1 + return status; + } + ++void RuntimeManagerService::RuntimeConfig(runtime::v1::RuntimeConfigResponse *reply, Errors &error) ++{ ++ if (reply == nullptr) { ++ ERROR("Invaliad params"); ++ error.SetError("Invalid params"); ++ return; ++ } ++ ++ reply->mutable_linux()->set_cgroup_driver(conf_get_systemd_cgroup() ? runtime::v1::SYSTEMD : runtime::v1::CGROUPFS); ++} ++ + } // namespace CRI +diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.h b/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.h +index 3800ec30..2160064e 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.h ++++ b/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.h +@@ -39,6 +39,8 @@ public: + + auto Status(Errors &error) -> std::unique_ptr<runtime::v1::RuntimeStatus>; + ++ void RuntimeConfig(runtime::v1::RuntimeConfigResponse *reply, Errors &error); ++ + private: + service_executor_t *m_cb; + std::shared_ptr<Network::PluginManager> m_pluginManager; +diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service.h b/src/daemon/entry/cri/v1/v1_cri_runtime_service.h +index 87394173..839f6724 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_runtime_service.h ++++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service.h +@@ -89,6 +89,8 @@ public: + virtual void UpdateRuntimeConfig(const runtime::v1::RuntimeConfig &config, Errors &error) = 0; + + virtual auto Status(Errors &error) -> std::unique_ptr<runtime::v1::RuntimeStatus> = 0; ++ ++ virtual void RuntimeConfig(runtime::v1::RuntimeConfigResponse *reply, Errors &error) = 0; + }; + } // namespace CRIV1 + #endif // DAEMON_ENTRY_CRI_V1_CRI_RUNTIME_SERVICE_INTERFACE_H +diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc +index 67cda5ed..aa5ae516 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc +@@ -161,4 +161,9 @@ auto CRIRuntimeServiceImpl::Status(Errors &error) -> std::unique_ptr<runtime::v1 + return m_runtimeManager->Status(error); + } + ++void CRIRuntimeServiceImpl::RuntimeConfig(runtime::v1::RuntimeConfigResponse *reply, Errors &error) ++{ ++ m_runtimeManager->RuntimeConfig(reply, error); ++} ++ + } // namespace CRIV1 +diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h +index 23866648..0a25749f 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h ++++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h +@@ -92,6 +92,8 @@ public: + + auto Status(Errors &error) -> std::unique_ptr<runtime::v1::RuntimeStatus> override; + ++ void RuntimeConfig(runtime::v1::RuntimeConfigResponse *reply, Errors &error) override; ++ + protected: + std::unique_ptr<RuntimeVersionerService> m_runtimeVersioner; + std::unique_ptr<ContainerManagerService> m_containerManager; +diff --git a/src/daemon/executor/container_cb/execution.c b/src/daemon/executor/container_cb/execution.c +index 63d8143c..7ed8e837 100644 +--- a/src/daemon/executor/container_cb/execution.c ++++ b/src/daemon/executor/container_cb/execution.c +@@ -427,6 +427,32 @@ static int cpurt_controller_init(const char *id, const host_config *host_spec) + return 0; + } + ++ if (conf_get_systemd_cgroup()) { ++ // currently it is the same as docker, yet it is unclear that ++ // if systemd cgroup is used and cgroup parent is set to a slice rather than system.slice ++ // should iSulad set cpu.rt_runtime_us and cpu.rt_period_us for the parent path? ++ // in fact, even if system.slice is used, ++ // cpu.rt_runtime_us and cpu.rt_period_us might still needed to be set manually ++ __isula_auto_free char *init_cgroup = common_get_init_cgroup("cpu"); ++ if (init_cgroup == NULL) { ++ ERROR("Failed to get init cgroup"); ++ return -1; ++ } ++ // make sure that the own cgroup path for cpu existed ++ __isula_auto_free char *own_cgroup = common_get_own_cgroup("cpu"); ++ if (own_cgroup == NULL) { ++ ERROR("Failed to get own cgroup"); ++ return -1; ++ } ++ char *new_cgroups_path = util_path_join(init_cgroup, cgroups_path); ++ if (new_cgroups_path == NULL) { ++ ERROR("Failed to join path"); ++ return -1; ++ } ++ free(cgroups_path); ++ cgroups_path = new_cgroups_path; ++ } ++ + mnt_root = sysinfo_cgroup_controller_cpurt_mnt_path(); + if (mnt_root == NULL) { + ERROR("Failed to get cpu rt controller mnt root path"); +diff --git a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c +index 8c3c5f1a..6b862958 100644 +--- a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c ++++ b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c +@@ -53,6 +53,13 @@ int rt_lcr_create(const char *name, const char *runtime, const rt_create_params_ + char *runtime_root = NULL; + struct engine_operation *engine_ops = NULL; + ++ if (conf_get_systemd_cgroup()) { ++ ERROR("Systemd cgroup not supported for lcr runtime"); ++ isulad_set_error_message("Systemd cgroup not supported for lcr runtime"); ++ ret = -1; ++ goto out; ++ } ++ + runtime_root = conf_get_routine_rootdir(runtime); + if (runtime_root == NULL) { + ERROR("Root path is NULL"); +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 745154bb..b9aba3e3 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -1157,6 +1157,7 @@ int rt_isula_create(const char *id, const char *runtime, const rt_create_params_ + p.runtime_args = (char **)runtime_args; + p.runtime_args_len = runtime_args_len; + p.attach_socket = attach_socket; ++ p.systemd_cgroup = conf_get_systemd_cgroup(); + copy_process(&p, config->process); + copy_annotations(&p, config->annotations); + +diff --git a/src/daemon/modules/runtime/shim/shim_rt_ops.c b/src/daemon/modules/runtime/shim/shim_rt_ops.c +index 81daf224..fc4d8e3a 100644 +--- a/src/daemon/modules/runtime/shim/shim_rt_ops.c ++++ b/src/daemon/modules/runtime/shim/shim_rt_ops.c +@@ -35,6 +35,7 @@ + #include "engine.h" + #include "shim_rt_monitor.h" + #include "supervisor.h" ++#include "isulad_config.h" + + #define EXIT_SIGNAL_OFFSET_X 128 + +@@ -370,6 +371,11 @@ int rt_shim_create(const char *id, const char *runtime, const rt_create_params_t + return -1; + } + ++ if (conf_get_systemd_cgroup()) { ++ ERROR("ShimV2 does not support systemd cgroup yet"); ++ return -1; ++ } ++ + exit_fifo_path = util_path_dir(params->exit_fifo); + if (exit_fifo_path == NULL) { + ERROR("%s: failed to get exit fifo dir from %s", id, params->exit_fifo); +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index 464b4fb4..b4d2b0f6 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -188,7 +188,10 @@ static char *do_get_container_cgroup_path(const host_config *host_spec) + } + + if (path == NULL) { +- // third, all faild, just use default '/isulad' ++ // third, all faild, just use default '/isulad' for cgroupfs or "system.slice" for systemd ++ if (conf_get_systemd_cgroup()) { ++ return util_strdup_s("system.slice"); ++ } + path = util_strdup_s("/isulad"); + } + +@@ -2288,6 +2291,23 @@ char *merge_container_cgroups_path(const char *id, const host_config *host_spec) + + path = do_get_container_cgroup_path(host_spec); + ++ if (conf_get_systemd_cgroup()) { ++ // systemd cgroup path has the form of [slice]:[prefix]:[name] ++#define SYSTEMD_CGROUP_PATH_LEN 3 ++ if (!util_has_suffix(path, ".slice")) { ++ ERROR("Invalid cgroup path %s for systemd", path); ++ isulad_set_error_message("Invalid cgroup path %s for systemd", path); ++ return NULL; ++ } ++ ++ // slice must not contain slashes ++ // convert test.slice/test-a.slice/test-a-b.slice to become test-a-b.slice ++ __isula_auto_free char *base = util_path_base(path); ++ const char *isulad_prefix = "isulad"; ++ const char *parts[SYSTEMD_CGROUP_PATH_LEN] = {base, isulad_prefix, id}; ++ return util_string_join(":", parts, SYSTEMD_CGROUP_PATH_LEN); ++ } ++ + return util_path_join(path, id); + } + +-- +2.34.1 + diff --git a/0025-add-ci-cases-for-systemd-cgroup-driver.patch b/0025-add-ci-cases-for-systemd-cgroup-driver.patch new file mode 100644 index 0000000..2d2ac71 --- /dev/null +++ b/0025-add-ci-cases-for-systemd-cgroup-driver.patch @@ -0,0 +1,219 @@ +From f5f100f5b244be2debebe815aaed3afad8950daf Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Tue, 6 Feb 2024 17:33:17 +0800 +Subject: [PATCH 25/43] add ci cases for systemd cgroup driver + +Signed-off-by: jikai <jikai11@huawei.com> +--- + .../container_cases/systemd_cgroup.sh | 80 +++++++++++++++++++ + test/mocks/isulad_config_mock.cc | 8 ++ + test/mocks/isulad_config_mock.h | 1 + + test/specs/specs/specs_ut.cc | 49 ++++++++++++ + 4 files changed, 138 insertions(+) + create mode 100755 CI/test_cases/container_cases/systemd_cgroup.sh + +diff --git a/CI/test_cases/container_cases/systemd_cgroup.sh b/CI/test_cases/container_cases/systemd_cgroup.sh +new file mode 100755 +index 00000000..ac1288e1 +--- /dev/null ++++ b/CI/test_cases/container_cases/systemd_cgroup.sh +@@ -0,0 +1,80 @@ ++#!/bin/bash ++# ++# attributes: isulad systemd cgroup run ++# concurrent: NO ++# spend time: 18 ++ ++####################################################################### ++##- Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. ++# - iSulad licensed under the Mulan PSL v2. ++# - You can use this software according to the terms and conditions of the Mulan PSL v2. ++# - You may obtain a copy of Mulan PSL v2 at: ++# - http://license.coscl.org.cn/MulanPSL2 ++# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++# - PURPOSE. ++# - See the Mulan PSL v2 for more details. ++##- @Description:CI ++##- @Author: jikai ++##- @Create: 2024-02-05 ++####################################################################### ++ ++curr_path=$(dirname $(readlink -f "$0")) ++data_path=$(realpath $curr_path/../data) ++source ../helpers.sh ++ ++function test_systemd_cgroup() ++{ ++ local ret=0 ++ local runtime=$1 ++ local image="busybox" ++ ++ local test="systemd cgroup driver test with (${runtime})=> (${FUNCNAME[@]})" ++ msg_info "${test} starting..." ++ ++ check_valgrind_log ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop isulad failed" && ((ret++)) ++ ++ start_isulad_with_valgrind --systemd-cgroup ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start isulad failed" && ((ret++)) ++ ++ cid1=$(isula run -tid --runtime $runtime -m 10M $image /bin/sh) ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start container failed" && ((ret++)) ++ cat /sys/fs/cgroup/memory/system.slice/isulad-$cid1.scope/memory.limit_in_bytes | grep ^10485760$ ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check memory limit failed" && ((ret++)) ++ ++ cid2=$(isula run -tid --runtime $runtime --cgroup-parent /test $image /bin/sh) ++ [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start container failed" && ((ret++)) ++ ++ cid3=$(isula run -tid --runtime $runtime -m 10M --cgroup-parent test-a-b.slice $image /bin/sh) ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start container failed" && ((ret++)) ++ cat /sys/fs/cgroup/memory/test.slice/test-a.slice/test-a-b.slice/isulad-$cid3.scope/memory.limit_in_bytes | grep ^10485760$ ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check memory limit failed" && ((ret++)) ++ ++ isula rm -f $cid1 $cid2 $cid3 ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - rm container failed" && ((ret++)) ++ ++ check_valgrind_log ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop isulad failed" && ((ret++)) ++ ++ start_isulad_with_valgrind ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start isulad failed" && ((ret++)) ++ ++ rm -rf $ulimitlog ++ ++ msg_info "${test} finished with return ${ret}..." ++ return ${ret} ++} ++ ++declare -i ans=0 ++ ++for element in ${RUNTIME_LIST[@]}; ++do ++ # lcr does not support systemd cgroup driver ++ if [ "$element" == "lcr" ];then ++ continue ++ fi ++ test_systemd_cgroup $element || ((ans++)) ++done ++ ++show_result ${ans} "${curr_path}/${0}" +diff --git a/test/mocks/isulad_config_mock.cc b/test/mocks/isulad_config_mock.cc +index 7ba4fa57..65b00563 100644 +--- a/test/mocks/isulad_config_mock.cc ++++ b/test/mocks/isulad_config_mock.cc +@@ -210,3 +210,11 @@ char *conf_get_isulad_loglevel(void) + } + return nullptr; + } ++ ++bool conf_get_systemd_cgroup(void) ++{ ++ if (g_isulad_conf_mock != nullptr) { ++ return g_isulad_conf_mock->ConfGetSystemdCgroup(); ++ } ++ return false; ++} +diff --git a/test/mocks/isulad_config_mock.h b/test/mocks/isulad_config_mock.h +index 6793fa51..d59c5938 100644 +--- a/test/mocks/isulad_config_mock.h ++++ b/test/mocks/isulad_config_mock.h +@@ -45,6 +45,7 @@ public: + MOCK_METHOD0(ConfGetSandboxStatePath, char *(void)); + MOCK_METHOD0(ConfGetEngineLogFile, char *(void)); + MOCK_METHOD0(ConfGetIsuladLogLevel, char *(void)); ++ MOCK_METHOD0(ConfGetSystemdCgroup, bool(void)); + }; + + void MockIsuladConf_SetMock(MockIsuladConf *mock); +diff --git a/test/specs/specs/specs_ut.cc b/test/specs/specs/specs_ut.cc +index 47e4ca6e..6c42216d 100644 +--- a/test/specs/specs/specs_ut.cc ++++ b/test/specs/specs/specs_ut.cc +@@ -319,6 +319,7 @@ TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_2) + ASSERT_TRUE(host_spec != nullptr); + + EXPECT_CALL(m_isulad_conf, GetCgroupParent()).WillRepeatedly(Invoke(invoke_conf_get_isulad_cgroup_parent_null)); ++ EXPECT_CALL(m_isulad_conf, ConfGetSystemdCgroup()).WillRepeatedly(Return(false)); + + merged_cp = merge_container_cgroups_path("123", host_spec); + ASSERT_NE(merged_cp, nullptr); +@@ -347,6 +348,7 @@ TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_3) + host_spec->cgroup_parent = util_strdup_s("/test"); + + EXPECT_CALL(m_isulad_conf, GetCgroupParent()).WillRepeatedly(Invoke(invoke_conf_get_isulad_cgroup_parent_null)); ++ EXPECT_CALL(m_isulad_conf, ConfGetSystemdCgroup()).WillRepeatedly(Return(false)); + + merged_cp = merge_container_cgroups_path("123", host_spec); + ASSERT_NE(merged_cp, nullptr); +@@ -373,6 +375,7 @@ TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_4) + ASSERT_TRUE(host_spec != nullptr); + + EXPECT_CALL(m_isulad_conf, GetCgroupParent()).WillRepeatedly(Invoke(invoke_conf_get_isulad_cgroup_parent)); ++ EXPECT_CALL(m_isulad_conf, ConfGetSystemdCgroup()).WillRepeatedly(Return(false)); + + merged_cp = merge_container_cgroups_path("123", host_spec); + ASSERT_NE(merged_cp, nullptr); +@@ -401,6 +404,7 @@ TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_5) + host_spec->cgroup_parent = util_strdup_s("/test"); + + EXPECT_CALL(m_isulad_conf, GetCgroupParent()).WillRepeatedly(Invoke(invoke_conf_get_isulad_cgroup_parent)); ++ EXPECT_CALL(m_isulad_conf, ConfGetSystemdCgroup()).WillRepeatedly(Return(false)); + + merged_cp = merge_container_cgroups_path("123", host_spec); + ASSERT_NE(merged_cp, nullptr); +@@ -414,6 +418,51 @@ TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_5) + testing::Mock::VerifyAndClearExpectations(&m_isulad_conf); + } + ++// systemd cgroup test ++TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_6) ++{ ++ oci_runtime_spec *oci_spec = nullptr; ++ host_config *host_spec = nullptr; ++ char *merged_cp = nullptr; ++ ++ oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec)); ++ ASSERT_TRUE(oci_spec != nullptr); ++ ++ host_spec = (host_config *)util_common_calloc_s(sizeof(host_config)); ++ ASSERT_TRUE(host_spec != nullptr); ++ ++ EXPECT_CALL(m_isulad_conf, GetCgroupParent()).WillRepeatedly(Invoke(invoke_conf_get_isulad_cgroup_parent_null)); ++ EXPECT_CALL(m_isulad_conf, ConfGetSystemdCgroup()).WillRepeatedly(Return(true)); ++ ++ merged_cp = merge_container_cgroups_path("123", host_spec); ++ ASSERT_NE(merged_cp, nullptr); ++ ASSERT_STREQ(merged_cp, "system.slice:isulad:123"); ++ free(merged_cp); ++ ++ host_spec->cgroup_parent = util_strdup_s("/test"); ++ merged_cp = merge_container_cgroups_path("123", host_spec); ++ ASSERT_EQ(merged_cp, nullptr); ++ free(host_spec->cgroup_parent); ++ ++ host_spec->cgroup_parent = util_strdup_s("test.slice"); ++ merged_cp = merge_container_cgroups_path("123", host_spec); ++ ASSERT_NE(merged_cp, nullptr); ++ ASSERT_STREQ(merged_cp, "test.slice:isulad:123"); ++ free(merged_cp); ++ free(host_spec->cgroup_parent); ++ ++ host_spec->cgroup_parent = util_strdup_s("test/test-a/test-a-b.slice"); ++ merged_cp = merge_container_cgroups_path("123", host_spec); ++ ASSERT_NE(merged_cp, nullptr); ++ ASSERT_STREQ(merged_cp, "test-a-b.slice:isulad:123"); ++ ++ free_oci_runtime_spec(oci_spec); ++ free_host_config(host_spec); ++ free(merged_cp); ++ ++ testing::Mock::VerifyAndClearExpectations(&m_isulad_conf); ++} ++ + TEST_F(SpecsUnitTest, test_update_oci_container_cgroups_path) + { + parser_error err = nullptr; +-- +2.34.1 + diff --git a/0026-move-systemd_cgroup-CI-test-to-manual-cases.patch b/0026-move-systemd_cgroup-CI-test-to-manual-cases.patch new file mode 100644 index 0000000..ea60213 --- /dev/null +++ b/0026-move-systemd_cgroup-CI-test-to-manual-cases.patch @@ -0,0 +1,18 @@ +From b93647205db5c4a5d74fb245c9b1e15ca1ffd3fe Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Sat, 16 Mar 2024 09:35:22 +0800 +Subject: [PATCH 26/43] move systemd_cgroup CI test to manual cases + +Signed-off-by: jikai <jikai11@huawei.com> +--- + CI/test_cases/{container_cases => manual_cases}/systemd_cgroup.sh | 0 + 1 file changed, 0 insertions(+), 0 deletions(-) + rename CI/test_cases/{container_cases => manual_cases}/systemd_cgroup.sh (100%) + +diff --git a/CI/test_cases/container_cases/systemd_cgroup.sh b/CI/test_cases/manual_cases/systemd_cgroup.sh +similarity index 100% +rename from CI/test_cases/container_cases/systemd_cgroup.sh +rename to CI/test_cases/manual_cases/systemd_cgroup.sh +-- +2.34.1 + diff --git a/0027-feature-add-support-for-cgroup-v2-metrics.patch b/0027-feature-add-support-for-cgroup-v2-metrics.patch new file mode 100644 index 0000000..f5b4073 --- /dev/null +++ b/0027-feature-add-support-for-cgroup-v2-metrics.patch @@ -0,0 +1,1084 @@ +From 7c7cd82619ed1f7e36d34da1afc2b417a90b3040 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Tue, 23 Jan 2024 17:18:51 +0800 +Subject: [PATCH 27/43] =?UTF-8?q?=E3=80=90feature=E3=80=91add=20support=20?= + =?UTF-8?q?for=20cgroup=20v2=20metrics?= +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/common/cgroup.c | 237 ++-------- + src/daemon/common/cgroup.h | 11 +- + src/daemon/common/cgroup_v1.c | 194 +++------ + src/daemon/common/cgroup_v2.c | 406 ++++++++++++++++++ + .../v1/v1_cri_pod_sandbox_manager_service.cc | 2 +- + .../cri_pod_sandbox_manager_service.cc | 2 +- + src/utils/cutils/utils_array.h | 6 + + 7 files changed, 506 insertions(+), 352 deletions(-) + create mode 100644 src/daemon/common/cgroup_v2.c + +diff --git a/src/daemon/common/cgroup.c b/src/daemon/common/cgroup.c +index 2d1cabb2..3c58f7fa 100644 +--- a/src/daemon/common/cgroup.c ++++ b/src/daemon/common/cgroup.c +@@ -29,32 +29,6 @@ + #include "utils_array.h" + #include "sysinfo.h" + +-// Cgroup V2 Item Definition +-#define CGROUP2_CPU_WEIGHT "cpu.weight" +-#define CGROUP2_CPU_MAX "cpu.max" +-#define CGROUP2_CPUSET_CPUS_EFFECTIVE "cpuset.cpus.effective" +-#define CGROUP2_CPUSET_MEMS_EFFECTIVE "cpuset.mems.effective" +-#define CGROUP2_CPUSET_CPUS "cpuset.cpus" +-#define CGROUP2_CPUSET_MEMS "cpuset.mems" +-#define CGROUP2_IO_WEIGHT "io.weight" +-#define CGROUP2_IO_BFQ_WEIGHT "io.bfq.weight" +-#define CGROUP2_IO_MAX "io.max" +-#define CGROUP2_MEMORY_MAX "memory.max" +-#define CGROUP2_MEMORY_LOW "memory.low" +-#define CGROUP2_MEMORY_SWAP_MAX "memory.swap.max" +-#define CGROUP2_HUGETLB_MAX "hugetlb.%s.max" +-#define CGROUP2_PIDS_MAX "pids.max" +-#define CGROUP2_FILES_LIMIT "files.limit" +- +-#define CGROUP2_CONTROLLERS_PATH CGROUP_MOUNTPOINT"/cgroup.controllers" +-#define CGROUP2_SUBTREE_CONTROLLER_PATH CGROUP_MOUNTPOINT"/cgroup.subtree_control" +-#define CGROUP2_CPUSET_CPUS_EFFECTIVE_PATH CGROUP_MOUNTPOINT"/cpuset.cpus.effective" +-#define CGROUP2_CPUSET_MEMS_EFFECTIVE_PATH CGROUP_MOUNTPOINT"/cpuset.mems.effective" +- +-#ifndef CGROUP2_SUPER_MAGIC +-#define CGROUP2_SUPER_MAGIC 0x63677270 +-#endif +- + #ifndef CGROUP_SUPER_MAGIC + #define CGROUP_SUPER_MAGIC 0x27e0eb + #endif +@@ -485,20 +459,6 @@ out: + return layers; + } + +-/* cgroup enabled */ +-static bool cgroup_enabled(const char *mountpoint, const char *name) +-{ +- char path[PATH_MAX] = { 0 }; +- int nret; +- +- nret = snprintf(path, sizeof(path), "%s/%s", mountpoint, name); +- if (nret < 0 || (size_t)nret >= sizeof(path)) { +- ERROR("Path is too long"); +- return false; +- } +- return util_file_exists(path); +-} +- + char *common_find_cgroup_subsystem_mountpoint(const cgroup_layer_t *layers, const char *subsystem) + { + size_t i; +@@ -607,188 +567,59 @@ int common_get_cgroup_version(void) + return CGROUP_VERSION_1; + } + +-static int cgroup2_enable_all() ++static int get_value_ull(const char *content, void *result) + { +- int ret = 0; +- int nret = 0; +- int n = 0; +- size_t i = 0; +- const char *space = ""; +- char *controllers_str = NULL; +- char *subtree_controller_str = NULL; +- char **controllers = NULL; +- char enable_controllers[PATH_MAX] = { 0 }; +- +- controllers_str = util_read_content_from_file(CGROUP2_CONTROLLERS_PATH); +- if (controllers_str == NULL || strlen(controllers_str) == 0 || strcmp(controllers_str, "\n") == 0) { +- WARN("no cgroup controller found"); +- goto out; +- } +- +- subtree_controller_str = util_read_content_from_file(CGROUP2_SUBTREE_CONTROLLER_PATH); +- if (subtree_controller_str != NULL && strcmp(controllers_str, subtree_controller_str) == 0) { +- goto out; +- } ++ uint64_t ull_result = 0; + +- controllers = util_string_split(controllers_str, ' '); +- if (controllers == NULL) { +- ERROR("split %s failed", controllers_str); +- ret = -1; +- goto out; +- } +- +- for (i = 0; i < util_array_len((const char **)controllers); i++) { +- nret = snprintf(enable_controllers + n, PATH_MAX - n, "%s+%s", space, controllers[i]); +- if (nret < 0 || (size_t)nret >= PATH_MAX - n) { +- ERROR("Path is too long"); +- goto out; +- } +- n += nret; +- space = " "; +- } +- ret = util_write_file(CGROUP2_SUBTREE_CONTROLLER_PATH, enable_controllers, strlen(enable_controllers), +- DEFAULT_CGROUP_FILE_MODE); +- if (ret != 0) { +- SYSERROR("write %s to %s failed", enable_controllers, CGROUP2_SUBTREE_CONTROLLER_PATH); +- goto out; ++ if (util_safe_uint64(content, &ull_result) != 0) { ++ ERROR("Failed to convert %s to uint64", content); ++ return -1; + } + +-out: +- util_free_array(controllers); +- free(controllers_str); +- free(subtree_controller_str); +- +- return ret; ++ *(uint64_t *)result = ull_result; ++ return 0; + } + +-#if defined (__ANDROID__) || defined(__MUSL__) +-static bool cgroup2_no_controller() ++int get_match_value_ull(const char *content, const char *match, void *result) + { +- char *controllers_str = NULL; ++ __isula_auto_free char *llu_string = NULL; ++ __isula_auto_free char *match_with_space = NULL; ++ __isula_auto_array_t char **lines = NULL; ++ char **worker = NULL; + +- controllers_str = util_read_content_from_file(CGROUP2_CONTROLLERS_PATH); +- if (controllers_str == NULL || strlen(controllers_str) == 0 || strcmp(controllers_str, "\n") == 0) { +- free(controllers_str); +- return true; ++ if (match == NULL) { ++ return get_value_ull(content, result); + } + +- free(controllers_str); +- return false; +-} +-#endif +- +-static int make_sure_cgroup2_isulad_path_exist() +-{ +- int ret = 0; +- +- if (util_dir_exists(CGROUP_ISULAD_PATH)) { +- return 0; ++ // match full string ++ match_with_space = util_string_append(" ", match); ++ if (match_with_space == NULL) { ++ ERROR("Failed to append string"); ++ return -1; + } + +- if (cgroup2_enable_all() != 0) { ++ lines = util_string_split(content, '\n'); ++ if (lines == NULL) { ++ ERROR("Failed to split content %s", content); + return -1; + } + +-#if defined (__ANDROID__) || defined(__MUSL__) +- if (cgroup2_no_controller()) { +- DEBUG("no cgroup controller found"); +- return 0; ++ for (worker = lines; worker && *worker; worker++) { ++ if (util_has_prefix(*worker, match_with_space)) { ++ break; ++ } + } +-#endif +- +- ret = mkdir(CGROUP_ISULAD_PATH, DEFAULT_CGROUP_DIR_MODE); +- if (ret != 0 && (errno != EEXIST || !util_dir_exists(CGROUP_ISULAD_PATH))) { ++ if (*worker == NULL) { ++ ERROR("Cannot find match string %s", match); + return -1; + } + +- return ret; +-} +- +-int common_get_cgroup_info_v2(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, +- cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, +- cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, +- cgroup_files_info_t *filesinfo, bool quiet) +-{ +- int ret = 0; +- int nret = 0; +- char *size = NULL; +- char path[PATH_MAX] = { 0 }; +- +- if (make_sure_cgroup2_isulad_path_exist() != 0) { ++ llu_string = util_sub_string(*worker, strlen(match_with_space), strlen(*worker) - strlen(match_with_space)); ++ if (llu_string == NULL) { ++ ERROR("Failed to sub string"); + return -1; + } ++ llu_string = util_trim_space(llu_string); + +- // cpu cgroup +- cpuinfo->cpu_shares = cgroup_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPU_WEIGHT); +- common_cgroup_do_log(quiet, !(cpuinfo->cpu_shares), "Your kernel does not support cgroup2 cpu weight"); +- +- cpuinfo->cpu_cfs_period = cgroup_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPU_MAX); +- cpuinfo->cpu_cfs_quota = cpuinfo->cpu_cfs_period; +- common_cgroup_do_log(quiet, !(cpuinfo->cpu_cfs_period), "Your kernel does not support cgroup2 cpu max"); +- +- cpusetinfo->cpuset = cgroup_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPUSET_CPUS_EFFECTIVE) && +- cgroup_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPUSET_CPUS) && +- cgroup_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPUSET_MEMS_EFFECTIVE) && +- cgroup_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPUSET_MEMS); +- common_cgroup_do_log(quiet, !(cpusetinfo->cpuset), "Your kernel does not support cpuset"); +- if (cpusetinfo->cpuset) { +- cpusetinfo->cpus = util_read_content_from_file(CGROUP2_CPUSET_CPUS_EFFECTIVE_PATH); +- cpusetinfo->mems = util_read_content_from_file(CGROUP2_CPUSET_MEMS_EFFECTIVE_PATH); +- if (cpusetinfo->cpus == NULL || cpusetinfo->mems == NULL) { +- ERROR("read cpus or mems failed"); +- return -1; +- } +- cpusetinfo->cpus = util_trim_space(cpusetinfo->cpus); +- cpusetinfo->mems = util_trim_space(cpusetinfo->mems); +- } +- +- // io cgroup +- blkioinfo->blkio_weight = cgroup_enabled(CGROUP_ISULAD_PATH, CGROUP2_IO_BFQ_WEIGHT) || +- cgroup_enabled(CGROUP_ISULAD_PATH, CGROUP2_IO_WEIGHT); +- blkioinfo->blkio_weight_device = blkioinfo->blkio_weight; +- common_cgroup_do_log(quiet, !(blkioinfo->blkio_weight), "Your kernel does not support cgroup2 io weight"); +- +- blkioinfo->blkio_read_bps_device = cgroup_enabled(CGROUP_ISULAD_PATH, CGROUP2_IO_MAX); +- blkioinfo->blkio_write_bps_device = blkioinfo->blkio_read_bps_device; +- blkioinfo->blkio_read_iops_device = blkioinfo->blkio_read_bps_device; +- blkioinfo->blkio_write_iops_device = blkioinfo->blkio_read_bps_device; +- common_cgroup_do_log(quiet, !(blkioinfo->blkio_read_bps_device), "Your kernel does not support cgroup2 io max"); +- +- // memory cgroup +- meminfo->limit = cgroup_enabled(CGROUP_ISULAD_PATH, CGROUP2_MEMORY_MAX); +- common_cgroup_do_log(quiet, !(meminfo->limit), "Your kernel does not support cgroup2 memory max"); +- +- meminfo->reservation = cgroup_enabled(CGROUP_ISULAD_PATH, CGROUP2_MEMORY_LOW); +- common_cgroup_do_log(quiet, !(meminfo->reservation), "Your kernel does not support cgroup2 memory low"); +- +- meminfo->swap = cgroup_enabled(CGROUP_ISULAD_PATH, CGROUP2_MEMORY_SWAP_MAX); +- common_cgroup_do_log(quiet, !(meminfo->swap), "Your kernel does not support cgroup2 memory swap max"); +- +- // pids cgroup +- pidsinfo->pidslimit = cgroup_enabled(CGROUP_ISULAD_PATH, CGROUP2_PIDS_MAX); +- common_cgroup_do_log(quiet, !(pidsinfo->pidslimit), "Your kernel does not support cgroup2 pids max"); +- +- // hugetlb cgroup +- size = get_default_huge_page_size(); +- if (size != NULL) { +- nret = snprintf(path, sizeof(path), CGROUP2_HUGETLB_MAX, size); +- if (nret < 0 || (size_t)nret >= sizeof(path)) { +- WARN("Failed to print hugetlb path"); +- ret = -1; +- goto out; +- } +- hugetlbinfo->hugetlblimit = cgroup_enabled(CGROUP_ISULAD_PATH, path); +- common_cgroup_do_log(quiet, !hugetlbinfo->hugetlblimit, "Your kernel does not support cgroup2 hugetlb limit"); +- } else { +- WARN("Your kernel does not support cgroup2 hugetlb limit"); +- } +- +- // files cgroup +- filesinfo->fileslimit = cgroup_enabled(CGROUP_ISULAD_PATH, CGROUP2_FILES_LIMIT); +- common_cgroup_do_log(quiet, !(filesinfo->fileslimit), "Your kernel does not support cgroup2 files limit"); +- +-out: +- free(size); +- +- return ret; +-} ++ return get_value_ull(llu_string, result); ++} +\ No newline at end of file +diff --git a/src/daemon/common/cgroup.h b/src/daemon/common/cgroup.h +index fa20f42c..251e3a3d 100644 +--- a/src/daemon/common/cgroup.h ++++ b/src/daemon/common/cgroup.h +@@ -31,6 +31,15 @@ extern "C" { + #define CGROUP_MOUNTPOINT "/sys/fs/cgroup" + #define CGROUP_ISULAD_PATH CGROUP_MOUNTPOINT"/isulad" + ++struct cgfile_t { ++ char *name; ++ char *file; ++ char *match; ++ int (*get_value)(const char *content, const char *match, void *result); ++}; ++ ++int get_match_value_ull(const char *content, const char *match, void *result); ++ + int common_get_cgroup_version(void); + + int common_find_cgroup_mnt_and_root(const char *subsystem, char **mountpoint, char **root); +@@ -42,7 +51,6 @@ static inline void common_cgroup_do_log(bool quiet, bool do_log, const char *msg + } + } + +- + typedef struct { + char **controllers; + char *mountpoint; +@@ -140,6 +148,7 @@ typedef struct { + } cgroup_metrics_t; + + int common_get_cgroup_v1_metrics(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics); ++int common_get_cgroup_v2_metrics(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics); + + char *common_get_init_cgroup(const char *subsystem); + +diff --git a/src/daemon/common/cgroup_v1.c b/src/daemon/common/cgroup_v1.c +index e38fc03e..e34100bc 100644 +--- a/src/daemon/common/cgroup_v1.c ++++ b/src/daemon/common/cgroup_v1.c +@@ -23,20 +23,8 @@ + + #define CGROUP_HUGETLB_LIMIT "hugetlb.%s.limit_in_bytes" + +-typedef struct { +- char *match; +-} cgfile_callback_args_t; +- +-struct cgfile_t { +- char *name; +- char *file; +- int (*get_value)(const char *content, const cgfile_callback_args_t *args, void *result); +-}; +- +-static int get_value_ll(const char *content, const cgfile_callback_args_t *args, void *result); +-static int get_value_ull(const char *content, const cgfile_callback_args_t *args, void *result); +-static int get_match_value_ull(const char *content, const cgfile_callback_args_t *args, void *result); +-static int get_value_string(const char *content, const cgfile_callback_args_t *args, void *result); ++static int get_value_ll(const char *content, const char *match, void *result); ++static int get_value_string(const char *content, const char *match, void *result); + + typedef enum { + // CPU subsystem +@@ -63,46 +51,46 @@ typedef enum { + + static struct cgfile_t g_cgroup_v1_files[] = { + // CPU subsystem +- [CPU_RT_PERIOD] = {"cpu_rt_period", "cpu.rt_period_us", get_value_ull}, +- [CPU_RT_RUNTIME] = {"cpu_rt_runtime", "cpu.rt_runtime_us", get_value_ull}, +- [CPU_SHARES] = {"cpu_shares", "cpu.shares", get_value_ull}, +- [CPU_CFS_PERIOD] = {"cpu_cfs_period", "cpu.cfs_period_us", get_value_ull}, +- [CPU_CFS_QUOTA] = {"cpu_cfs_quota", "cpu.cfs_quota_us", get_value_ll}, ++ [CPU_RT_PERIOD] = {"cpu_rt_period", "cpu.rt_period_us", NULL, get_match_value_ull}, ++ [CPU_RT_RUNTIME] = {"cpu_rt_runtime", "cpu.rt_runtime_us", NULL, get_match_value_ull}, ++ [CPU_SHARES] = {"cpu_shares", "cpu.shares", NULL, get_match_value_ull}, ++ [CPU_CFS_PERIOD] = {"cpu_cfs_period", "cpu.cfs_period_us", NULL, get_match_value_ull}, ++ [CPU_CFS_QUOTA] = {"cpu_cfs_quota", "cpu.cfs_quota_us", NULL, get_value_ll}, + // CPUSET subsystem +- [CPUSET_CPUS] = {"cpuset_cpus", "cpuset.cpus", get_value_string}, +- [CPUSET_MEMS] = {"cpuset_mems", "cpuset.mems", get_value_string}, ++ [CPUSET_CPUS] = {"cpuset_cpus", "cpuset.cpus", NULL, get_value_string}, ++ [CPUSET_MEMS] = {"cpuset_mems", "cpuset.mems", NULL, get_value_string}, + // CPUACCT subsystem +- [CPUACCT_USE_NANOS] = {"cpu_use_nanos", "cpuacct.usage", get_value_ull}, +- [CPUACCT_USE_USER] = {"cpu_use_user", "cpuacct.stat", get_match_value_ull}, +- [CPUACCT_USE_SYS] = {"cpu_use_sys", "cpuacct.stat", get_match_value_ull}, ++ [CPUACCT_USE_NANOS] = {"cpu_use_nanos", "cpuacct.usage", NULL, get_match_value_ull}, ++ [CPUACCT_USE_USER] = {"cpu_use_user", "cpuacct.stat", NULL, get_match_value_ull}, ++ [CPUACCT_USE_SYS] = {"cpu_use_sys", "cpuacct.stat", NULL, get_match_value_ull}, + // MEMORY subsystem +- [MEMORY_LIMIT] = {"mem_limit", "memory.limit_in_bytes", get_value_ull}, +- [MEMORY_USAGE] = {"mem_usage", "memory.usage_in_bytes", get_value_ull}, +- [MEMORY_SOFT_LIMIT] = {"mem_soft_limit", "memory.soft_limit_in_bytes", get_value_ull}, +- [MEMORY_KMEM_LIMIT] = {"kmem_limit", "memory.kmem.limit_in_bytes", get_value_ull}, +- [MEMORY_KMEM_USAGE] = {"kmem_usage", "memory.kmem.usage_in_bytes", get_value_ull}, +- [MEMORY_SWAPPINESS] = {"swappiness", "memory.swappiness", NULL}, +- [MEMORY_SW_LIMIT] = {"memsw_limit", "memory.memsw.limit_in_bytes", get_value_ull}, +- [MEMORY_SW_USAGE] = {"memsw_usage", "memory.memsw.usage_in_bytes", get_value_ull}, +- [MEMORY_CACHE] = {"cache", "memory.stat", get_match_value_ull}, +- [MEMORY_CACHE_TOTAL] = {"cache_total", "memory.stat", get_match_value_ull}, +- [MEMORY_TOTAL_RSS] = {"total_rss", "memory.stat", get_match_value_ull}, +- [MEMORY_TOTAL_PGFAULT] = {"total_page_fault", "memory.stat", get_match_value_ull}, +- [MEMORY_TOTAL_PGMAJFAULT] = {"total_page_majfault", "memory.stat", get_match_value_ull}, +- [MEMORY_TOTAL_INACTIVE_FILE] = {"total_inactive_file", "memory.stat", get_match_value_ull}, +- [MEMORY_OOM_CONTROL] = {"oom_control", "memory.oom_control", NULL}, ++ [MEMORY_LIMIT] = {"mem_limit", "memory.limit_in_bytes", NULL, get_match_value_ull}, ++ [MEMORY_USAGE] = {"mem_usage", "memory.usage_in_bytes", NULL, get_match_value_ull}, ++ [MEMORY_SOFT_LIMIT] = {"mem_soft_limit", "memory.soft_limit_in_bytes", NULL, get_match_value_ull}, ++ [MEMORY_KMEM_LIMIT] = {"kmem_limit", "memory.kmem.limit_in_bytes", NULL, get_match_value_ull}, ++ [MEMORY_KMEM_USAGE] = {"kmem_usage", "memory.kmem.usage_in_bytes", NULL, get_match_value_ull}, ++ [MEMORY_SWAPPINESS] = {"swappiness", "memory.swappiness", NULL, NULL}, ++ [MEMORY_SW_LIMIT] = {"memsw_limit", "memory.memsw.limit_in_bytes", NULL, get_match_value_ull}, ++ [MEMORY_SW_USAGE] = {"memsw_usage", "memory.memsw.usage_in_bytes", NULL, get_match_value_ull}, ++ [MEMORY_CACHE] = {"cache", "memory.stat", NULL, get_match_value_ull}, ++ [MEMORY_CACHE_TOTAL] = {"cache_total", "memory.stat", NULL, get_match_value_ull}, ++ [MEMORY_TOTAL_RSS] = {"total_rss", "memory.stat", "total_rss", get_match_value_ull}, ++ [MEMORY_TOTAL_PGFAULT] = {"total_page_fault", "memory.stat", "total_pgfault", get_match_value_ull}, ++ [MEMORY_TOTAL_PGMAJFAULT] = {"total_page_majfault", "memory.stat", "total_pgmajfault", get_match_value_ull}, ++ [MEMORY_TOTAL_INACTIVE_FILE] = {"total_inactive_file", "memory.stat", "total_inactive_file", get_match_value_ull}, ++ [MEMORY_OOM_CONTROL] = {"oom_control", "memory.oom_control", NULL, NULL}, + // BLKIO subsystem +- [BLKIO_WEIGTH] = {"blkio_weigth", "blkio.weight", NULL}, +- [BLKIO_WEIGTH_DEVICE] = {"blkio_weigth_device", "blkio.weight_device", NULL}, +- [BLKIO_READ_BPS] = {"blkio_read_bps", "blkio.throttle.read_bps_device", NULL}, +- [BLKIO_WRITE_BPS] = {"blkio_write_bps", "blkio.throttle.write_bps_device", NULL}, +- [BLKIO_READ_IOPS] = {"blkio_read_iops", "blkio.throttle.read_iops_device", NULL}, +- [BLKIO_WRITE_IOPS] = {"blkio_write_iops", "blkio.throttle.write_iops_device", NULL}, ++ [BLKIO_WEIGTH] = {"blkio_weigth", "blkio.weight", NULL, NULL}, ++ [BLKIO_WEIGTH_DEVICE] = {"blkio_weigth_device", "blkio.weight_device", NULL, NULL}, ++ [BLKIO_READ_BPS] = {"blkio_read_bps", "blkio.throttle.read_bps_device", NULL, NULL}, ++ [BLKIO_WRITE_BPS] = {"blkio_write_bps", "blkio.throttle.write_bps_device", NULL, NULL}, ++ [BLKIO_READ_IOPS] = {"blkio_read_iops", "blkio.throttle.read_iops_device", NULL, NULL}, ++ [BLKIO_WRITE_IOPS] = {"blkio_write_iops", "blkio.throttle.write_iops_device", NULL, NULL}, + // PIDS subsystem +- [PIDS_CURRENT] = {"pids_current", "pids.current", get_value_ull}, ++ [PIDS_CURRENT] = {"pids_current", "pids.current", NULL, get_match_value_ull}, + }; + +-static int get_value_ll(const char *content, const cgfile_callback_args_t *args, void *result) ++static int get_value_ll(const char *content, const char *match, void *result) + { + long long ll_result = 0; + +@@ -115,81 +103,7 @@ static int get_value_ll(const char *content, const cgfile_callback_args_t *args, + return 0; + } + +-static int get_value_ull(const char *content, const cgfile_callback_args_t *args, void *result) +-{ +- uint64_t ull_result = 0; +- +- if (util_safe_uint64(content, &ull_result) != 0) { +- ERROR("Failed to convert %s to uint64", content); +- return -1; +- } +- +- *(uint64_t *)result = ull_result; +- return 0; +-} +- +-static int get_match_value_ull(const char *content, const cgfile_callback_args_t *args, void *result) +-{ +- int ret = 0; +- uint64_t llu_result = 0; +- char *llu_string = NULL; +- char *match_with_space = NULL; +- char **lines = NULL; +- char **worker = NULL; +- +- if (args == NULL || args->match == NULL || strlen(args->match) == 0) { +- ERROR("Invalid arguments"); +- return -1; +- } +- +- // match full string +- match_with_space = util_string_append(" ", args->match); +- if (match_with_space == NULL) { +- ERROR("Failed to append string"); +- return -1; +- } +- +- lines = util_string_split(content, '\n'); +- if (lines == NULL) { +- ERROR("Failed to split content %s", content); +- ret = -1; +- goto out; +- } +- +- for (worker = lines; worker && *worker; worker++) { +- if (util_has_prefix(*worker, match_with_space)) { +- break; +- } +- } +- if (*worker == NULL) { +- ERROR("Cannot find match string %s", args->match); +- ret = -1; +- goto out; +- } +- +- llu_string = util_sub_string(*worker, strlen(match_with_space), strlen(*worker) - strlen(match_with_space)); +- if (llu_string == NULL) { +- ERROR("Failed to sub string"); +- ret = -1; +- goto out; +- } +- llu_string = util_trim_space(llu_string); +- +- ret = util_safe_uint64(llu_string, &llu_result); +- if (ret != 0) { +- ERROR("Failed to convert %s to uint64", llu_string); +- } else { +- *(uint64_t *)result = llu_result; +- } +- +-out: +- free(match_with_space); +- free(llu_string); +- util_free_array(lines); +- return ret; +-} +- +-static int get_value_string(const char *content, const cgfile_callback_args_t *args, void *result) ++static int get_value_string(const char *content, const char *match, void *result) + { + *(char **)result = util_strdup_s(content); + return 0; +@@ -228,7 +142,7 @@ static bool check_cgroup_v1_helper(const char *mountpoint, const cgroup_v1_files + } + + static int get_cgroup_v1_value_helper(const char *path, const cgroup_v1_files_index index, +- const cgfile_callback_args_t *args, void *result) ++ void *result) + { + int nret = 0; + char file_path[PATH_MAX] = { 0 }; +@@ -265,7 +179,7 @@ static int get_cgroup_v1_value_helper(const char *path, const cgroup_v1_files_in + util_trim_newline(content); + content = util_trim_space(content); + +- nret = g_cgroup_v1_files[index].get_value(content, args, result); ++ nret = g_cgroup_v1_files[index].get_value(content, g_cgroup_v1_files[index].match, result); + if (nret != 0) { + ERROR("%s: failed to get value", g_cgroup_v1_files[index].name); + } +@@ -308,11 +222,11 @@ static void check_cgroup_v1_cpuset(const cgroup_layer_t *layers, const bool quie + return; + } + +- if (get_cgroup_v1_value_helper(mountpoint, CPUSET_CPUS, NULL, (void *)&cpusetinfo->cpus) != 0) { ++ if (get_cgroup_v1_value_helper(mountpoint, CPUSET_CPUS, (void *)&cpusetinfo->cpus) != 0) { + ERROR("Failed to get cgroup cpuset.cpus data"); + return; + } +- if (get_cgroup_v1_value_helper(mountpoint, CPUSET_MEMS, NULL, (void *)&cpusetinfo->mems) != 0) { ++ if (get_cgroup_v1_value_helper(mountpoint, CPUSET_MEMS, (void *)&cpusetinfo->mems) != 0) { + free(cpusetinfo->cpus); + cpusetinfo->cpus = NULL; + ERROR("Failed to get cgroup cpuset.cpus data"); +@@ -463,7 +377,7 @@ static void get_cgroup_v1_metrics_cpu(const cgroup_layer_t *layers, const char * + return; + } + +- get_cgroup_v1_value_helper(path, CPUACCT_USE_NANOS, NULL, (void *)&cgroup_cpu_metrics->cpu_use_nanos); ++ get_cgroup_v1_value_helper(path, CPUACCT_USE_NANOS, (void *)&cgroup_cpu_metrics->cpu_use_nanos); + } + + static void get_cgroup_v1_metrics_memory(const cgroup_layer_t *layers, const char *cgroup_path, +@@ -472,18 +386,6 @@ static void get_cgroup_v1_metrics_memory(const cgroup_layer_t *layers, const cha + int nret = 0; + char *mountpoint = NULL; + char path[PATH_MAX] = { 0 }; +- const cgfile_callback_args_t total_inactive_file_arg = { +- .match = "total_inactive_file", +- }; +- const cgfile_callback_args_t total_rss_arg = { +- .match = "total_rss", +- }; +- const cgfile_callback_args_t total_pgfault_arg = { +- .match = "total_pgfault", +- }; +- const cgfile_callback_args_t total_pgmajfault_arg = { +- .match = "total_pgmajfault", +- }; + + mountpoint = common_find_cgroup_subsystem_mountpoint(layers, "memory"); + if (mountpoint == NULL) { +@@ -497,14 +399,14 @@ static void get_cgroup_v1_metrics_memory(const cgroup_layer_t *layers, const cha + return; + } + +- get_cgroup_v1_value_helper(path, MEMORY_LIMIT, NULL, (void *)&cgroup_mem_metrics->mem_limit); +- get_cgroup_v1_value_helper(path, MEMORY_USAGE, NULL, (void *)&cgroup_mem_metrics->mem_used); +- get_cgroup_v1_value_helper(path, MEMORY_TOTAL_RSS, &total_rss_arg, (void *)&cgroup_mem_metrics->total_rss); +- get_cgroup_v1_value_helper(path, MEMORY_TOTAL_PGFAULT, &total_pgfault_arg, ++ get_cgroup_v1_value_helper(path, MEMORY_LIMIT, (void *)&cgroup_mem_metrics->mem_limit); ++ get_cgroup_v1_value_helper(path, MEMORY_USAGE, (void *)&cgroup_mem_metrics->mem_used); ++ get_cgroup_v1_value_helper(path, MEMORY_TOTAL_RSS, (void *)&cgroup_mem_metrics->total_rss); ++ get_cgroup_v1_value_helper(path, MEMORY_TOTAL_PGFAULT, + (void *)&cgroup_mem_metrics->total_pgfault); +- get_cgroup_v1_value_helper(path, MEMORY_TOTAL_PGMAJFAULT, &total_pgmajfault_arg, ++ get_cgroup_v1_value_helper(path, MEMORY_TOTAL_PGMAJFAULT, + (void *)&cgroup_mem_metrics->total_pgmajfault); +- get_cgroup_v1_value_helper(path, MEMORY_TOTAL_INACTIVE_FILE, &total_inactive_file_arg, ++ get_cgroup_v1_value_helper(path, MEMORY_TOTAL_INACTIVE_FILE, + (void *)&cgroup_mem_metrics->total_inactive_file); + } + +@@ -527,7 +429,7 @@ static void get_cgroup_v1_metrics_pid(const cgroup_layer_t *layers, const char * + return; + } + +- get_cgroup_v1_value_helper(path, PIDS_CURRENT, NULL, (void *)&cgroup_pids_metrics->pid_current); ++ get_cgroup_v1_value_helper(path, PIDS_CURRENT, (void *)&cgroup_pids_metrics->pid_current); + } + + int common_get_cgroup_v1_metrics(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics) +diff --git a/src/daemon/common/cgroup_v2.c b/src/daemon/common/cgroup_v2.c +new file mode 100644 +index 00000000..25509bda +--- /dev/null ++++ b/src/daemon/common/cgroup_v2.c +@@ -0,0 +1,406 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-01-16 ++ * Description: provide cgroup v2 functions ++ ******************************************************************************/ ++#include "cgroup.h" ++ ++#include <stdio.h> ++#include <stdlib.h> ++#include <sys/stat.h> ++ ++#include <isula_libutils/auto_cleanup.h> ++ ++#include "utils.h" ++#include "path.h" ++#include "sysinfo.h" ++ ++// Cgroup V2 Item Definition ++#define CGROUP2_CPU_WEIGHT "cpu.weight" ++#define CGROUP2_CPU_MAX "cpu.max" ++#define CGROUP2_CPUSET_CPUS_EFFECTIVE "cpuset.cpus.effective" ++#define CGROUP2_CPUSET_MEMS_EFFECTIVE "cpuset.mems.effective" ++#define CGROUP2_CPUSET_CPUS "cpuset.cpus" ++#define CGROUP2_CPUSET_MEMS "cpuset.mems" ++#define CGROUP2_IO_WEIGHT "io.weight" ++#define CGROUP2_IO_BFQ_WEIGHT "io.bfq.weight" ++#define CGROUP2_IO_MAX "io.max" ++#define CGROUP2_MEMORY_MAX "memory.max" ++#define CGROUP2_MEMORY_LOW "memory.low" ++#define CGROUP2_MEMORY_SWAP_MAX "memory.swap.max" ++#define CGROUP2_HUGETLB_MAX "hugetlb.%s.max" ++#define CGROUP2_PIDS_MAX "pids.max" ++#define CGROUP2_FILES_LIMIT "files.limit" ++ ++#define CGROUP2_CONTROLLERS_PATH CGROUP_MOUNTPOINT"/cgroup.controllers" ++#define CGROUP2_SUBTREE_CONTROLLER_PATH CGROUP_MOUNTPOINT"/cgroup.subtree_control" ++#define CGROUP2_CPUSET_CPUS_EFFECTIVE_PATH CGROUP_MOUNTPOINT"/cpuset.cpus.effective" ++#define CGROUP2_CPUSET_MEMS_EFFECTIVE_PATH CGROUP_MOUNTPOINT"/cpuset.mems.effective" ++ ++#ifndef CGROUP2_SUPER_MAGIC ++#define CGROUP2_SUPER_MAGIC 0x63677270 ++#endif ++ ++static int get_value_ull_v2(const char *content, const char *match, void *result) ++{ ++ uint64_t ull_result = 0; ++ __isula_auto_free char *tmp_str = util_strdup_s(content); ++ ++ tmp_str = util_trim_space(tmp_str); ++ if (strcmp(tmp_str, "max") == 0) { ++ *(uint64_t *)result = UINT64_MAX; ++ return 0; ++ } ++ ++ if (util_safe_uint64(content, &ull_result) != 0) { ++ ERROR("Failed to convert %s to uint64", content); ++ return -1; ++ } ++ ++ *(uint64_t *)result = ull_result; ++ return 0; ++} ++ ++typedef enum { ++ // cpu ++ CPUACCT_USE_USER, CPUACCT_USE_SYS, CPUACCT_USE_NANOS, ++ // MEMORY subsystem ++ MEMORY_USAGE, MEMORY_LIMIT, MEMORY_ANON, ++ MEMORY_TOTAL_PGFAULT, MEMORY_TOTAL_INACTIVE_FILE, MEMORY_TOTAL_PGMAJFAULT, ++ MEMORY_CACHE, MEMORY_CACHE_TOTAL, ++ // BLKIO subsystem ++ BLKIO_READ_BPS, BLKIO_WRITE_BPS, BLKIO_READ_IOPS, BLKIO_WRITE_IOPS, ++ // PIDS subsystem ++ PIDS_CURRENT, ++ // MAX ++ CGROUP_V2_FILES_INDEX_MAXS ++} cgroup_v2_files_index; ++ ++static struct cgfile_t g_cgroup_v2_files[] = { ++ // cpu ++ [CPUACCT_USE_USER] = {"cpu_use_user", "cpu.stat", "user_usec", get_match_value_ull}, ++ [CPUACCT_USE_SYS] = {"cpu_use_sys", "cpu.stat", "system_usec", get_match_value_ull}, ++ [CPUACCT_USE_NANOS] = {"cpu_use_nanos", "cpu.stat", "usage_usec", get_match_value_ull}, ++ // memory ++ [MEMORY_USAGE] = {"mem_usage", "memory.current", NULL, get_value_ull_v2}, ++ [MEMORY_LIMIT] = {"mem_limit", "memory.max", NULL, get_value_ull_v2}, ++ [MEMORY_ANON] = {"mem_anon", "memory.stat", "anon", get_match_value_ull}, ++ [MEMORY_TOTAL_PGFAULT] = {"total_page_fault", "memory.stat", "pgfault", get_match_value_ull}, ++ [MEMORY_TOTAL_PGMAJFAULT] = {"total_page_majfault", "memory.stat", "pgmajfault", get_match_value_ull}, ++ [MEMORY_TOTAL_INACTIVE_FILE] = {"total_inactive_file", "memory.stat", "inactive_file", get_match_value_ull}, ++ [MEMORY_CACHE] = {"cache", "memory.stat", "file", get_match_value_ull}, ++ [MEMORY_CACHE_TOTAL] = {"cache_total", "memory.stat", "file", get_match_value_ull}, ++ // pids ++ [PIDS_CURRENT] = {"pids_current", "pids.current", NULL, get_value_ull_v2}, ++}; ++ ++static int get_cgroup_v2_value_helper(const char *path, const cgroup_v2_files_index index, void *result) ++{ ++ int nret = 0; ++ char file_path[PATH_MAX] = { 0 }; ++ char real_path[PATH_MAX] = { 0 }; ++ char *content = NULL; ++ ++ if (index >= CGROUP_V2_FILES_INDEX_MAXS) { ++ ERROR("Index out of range"); ++ return false; ++ } ++ ++ if (path == NULL || strlen(path) == 0 || result == NULL) { ++ ERROR("%s: Invalid arguments", g_cgroup_v2_files[index].name); ++ return -1; ++ } ++ ++ nret = snprintf(file_path, sizeof(file_path), "%s/%s", path, g_cgroup_v2_files[index].file); ++ if (nret < 0 || (size_t)nret >= sizeof(file_path)) { ++ ERROR("%s: failed to snprintf", g_cgroup_v2_files[index].name); ++ return -1; ++ } ++ ++ if (util_clean_path(file_path, real_path, sizeof(real_path)) == NULL) { ++ ERROR("%s: failed to clean path %s", g_cgroup_v2_files[index].name, file_path); ++ return -1; ++ } ++ ++ content = util_read_content_from_file(real_path); ++ if (content == NULL) { ++ ERROR("%s: failed to read file %s", g_cgroup_v2_files[index].name, real_path); ++ return -1; ++ } ++ ++ util_trim_newline(content); ++ content = util_trim_space(content); ++ ++ nret = g_cgroup_v2_files[index].get_value(content, g_cgroup_v2_files[index].match, result); ++ if (nret != 0) { ++ ERROR("%s: failed to get value", g_cgroup_v2_files[index].name); ++ } ++ ++ free(content); ++ return nret; ++} ++ ++static void get_cgroup_v2_metrics_cpu(const char *cgroup_path, cgroup_cpu_metrics_t *cgroup_cpu_metrics) ++{ ++ int nret = 0; ++ char path[PATH_MAX] = { 0 }; ++ ++ nret = snprintf(path, sizeof(path), "%s/%s", CGROUP_MOUNTPOINT, cgroup_path); ++ if (nret < 0 || (size_t)nret >= sizeof(path)) { ++ ERROR("Failed to snprintf"); ++ return; ++ } ++ ++ get_cgroup_v2_value_helper(path, CPUACCT_USE_NANOS, (void *)&cgroup_cpu_metrics->cpu_use_nanos); ++} ++ ++static void get_cgroup_v2_metrics_memory(const char *cgroup_path, cgroup_mem_metrics_t *cgroup_mem_metrics) ++{ ++ int nret = 0; ++ char path[PATH_MAX] = { 0 }; ++ ++ nret = snprintf(path, sizeof(path), "%s/%s", CGROUP_MOUNTPOINT, cgroup_path); ++ if (nret < 0 || (size_t)nret >= sizeof(path)) { ++ ERROR("Failed to snprintf"); ++ return; ++ } ++ ++ get_cgroup_v2_value_helper(path, MEMORY_LIMIT, (void *)&cgroup_mem_metrics->mem_limit); ++ get_cgroup_v2_value_helper(path, MEMORY_USAGE, (void *)&cgroup_mem_metrics->mem_used); ++ // Use Anon memory for RSS as cAdvisor on cgroupv2 ++ // see https://github.com/google/cadvisor/blob/a9858972e75642c2b1914c8d5428e33e6392c08a/container/libcontainer/handler.go#L799 ++ get_cgroup_v2_value_helper(path, MEMORY_ANON, (void *)&cgroup_mem_metrics->total_rss); ++ get_cgroup_v2_value_helper(path, MEMORY_TOTAL_PGFAULT, ++ (void *)&cgroup_mem_metrics->total_pgfault); ++ get_cgroup_v2_value_helper(path, MEMORY_TOTAL_PGMAJFAULT, ++ (void *)&cgroup_mem_metrics->total_pgmajfault); ++ get_cgroup_v2_value_helper(path, MEMORY_TOTAL_INACTIVE_FILE, ++ (void *)&cgroup_mem_metrics->total_inactive_file); ++} ++ ++static void get_cgroup_v2_metrics_pid(const char *cgroup_path, cgroup_pids_metrics_t *cgroup_pids_metrics) ++{ ++ int nret = 0; ++ char path[PATH_MAX] = { 0 }; ++ ++ nret = snprintf(path, sizeof(path), "%s/%s", CGROUP_MOUNTPOINT, cgroup_path); ++ if (nret < 0 || (size_t)nret >= sizeof(path)) { ++ ERROR("Failed to snprintf"); ++ return; ++ } ++ ++ get_cgroup_v2_value_helper(path, PIDS_CURRENT, (void *)&cgroup_pids_metrics->pid_current); ++} ++ ++static int cgroup2_enable_all() ++{ ++ int ret = 0; ++ int nret = 0; ++ int n = 0; ++ size_t i = 0; ++ const char *space = ""; ++ __isula_auto_free char *controllers_str = NULL; ++ __isula_auto_free char *subtree_controller_str = NULL; ++ __isula_auto_array_t char **controllers = NULL; ++ char enable_controllers[PATH_MAX] = { 0 }; ++ ++ controllers_str = util_read_content_from_file(CGROUP2_CONTROLLERS_PATH); ++ if (controllers_str == NULL || strlen(controllers_str) == 0 || strcmp(controllers_str, "\n") == 0) { ++ WARN("no cgroup controller found"); ++ return ret; ++ } ++ ++ subtree_controller_str = util_read_content_from_file(CGROUP2_SUBTREE_CONTROLLER_PATH); ++ if (subtree_controller_str != NULL && strcmp(controllers_str, subtree_controller_str) == 0) { ++ return ret; ++ } ++ ++ controllers = util_string_split(controllers_str, ' '); ++ if (controllers == NULL) { ++ ERROR("split %s failed", controllers_str); ++ return -1; ++ } ++ ++ for (i = 0; i < util_array_len((const char **)controllers); i++) { ++ nret = snprintf(enable_controllers + n, PATH_MAX - n, "%s+%s", space, controllers[i]); ++ if (nret < 0 || (size_t)nret >= PATH_MAX - n) { ++ ERROR("Path is too long"); ++ return -1; ++ } ++ n += nret; ++ space = " "; ++ } ++ ret = util_write_file(CGROUP2_SUBTREE_CONTROLLER_PATH, enable_controllers, strlen(enable_controllers), ++ DEFAULT_CGROUP_FILE_MODE); ++ if (ret != 0) { ++ SYSERROR("write %s to %s failed", enable_controllers, CGROUP2_SUBTREE_CONTROLLER_PATH); ++ return ret; ++ } ++ ++ return ret; ++} ++ ++#if defined (__ANDROID__) || defined(__MUSL__) ++static bool cgroup2_no_controller() ++{ ++ char *controllers_str = NULL; ++ ++ controllers_str = util_read_content_from_file(CGROUP2_CONTROLLERS_PATH); ++ if (controllers_str == NULL || strlen(controllers_str) == 0 || strcmp(controllers_str, "\n") == 0) { ++ free(controllers_str); ++ return true; ++ } ++ ++ free(controllers_str); ++ return false; ++} ++#endif ++ ++static int make_sure_cgroup2_isulad_path_exist() ++{ ++ int ret = 0; ++ ++ if (util_dir_exists(CGROUP_ISULAD_PATH)) { ++ return 0; ++ } ++ ++ if (cgroup2_enable_all() != 0) { ++ return -1; ++ } ++ ++#if defined (__ANDROID__) || defined(__MUSL__) ++ if (cgroup2_no_controller()) { ++ DEBUG("no cgroup controller found"); ++ return 0; ++ } ++#endif ++ ++ ret = mkdir(CGROUP_ISULAD_PATH, DEFAULT_CGROUP_DIR_MODE); ++ if (ret != 0 && (errno != EEXIST || !util_dir_exists(CGROUP_ISULAD_PATH))) { ++ return -1; ++ } ++ ++ return ret; ++} ++ ++/* cgroup enabled */ ++static bool cgroup_v2_enabled(const char *mountpoint, const char *name) ++{ ++ char path[PATH_MAX] = { 0 }; ++ int nret; ++ ++ nret = snprintf(path, sizeof(path), "%s/%s", mountpoint, name); ++ if (nret < 0 || (size_t)nret >= sizeof(path)) { ++ ERROR("Path is too long"); ++ return false; ++ } ++ return util_file_exists(path); ++} ++ ++int common_get_cgroup_info_v2(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, ++ cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, ++ cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, ++ cgroup_files_info_t *filesinfo, bool quiet) ++{ ++ int ret = 0; ++ int nret = 0; ++ __isula_auto_free char *size = NULL; ++ char path[PATH_MAX] = { 0 }; ++ ++ if (make_sure_cgroup2_isulad_path_exist() != 0) { ++ return -1; ++ } ++ ++ // cpu cgroup ++ cpuinfo->cpu_shares = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPU_WEIGHT); ++ common_cgroup_do_log(quiet, !(cpuinfo->cpu_shares), "Your kernel does not support cgroup2 cpu weight"); ++ ++ cpuinfo->cpu_cfs_period = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPU_MAX); ++ cpuinfo->cpu_cfs_quota = cpuinfo->cpu_cfs_period; ++ common_cgroup_do_log(quiet, !(cpuinfo->cpu_cfs_period), "Your kernel does not support cgroup2 cpu max"); ++ ++ cpusetinfo->cpuset = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPUSET_CPUS_EFFECTIVE) && ++ cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPUSET_CPUS) && ++ cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPUSET_MEMS_EFFECTIVE) && ++ cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPUSET_MEMS); ++ common_cgroup_do_log(quiet, !(cpusetinfo->cpuset), "Your kernel does not support cpuset"); ++ if (cpusetinfo->cpuset) { ++ cpusetinfo->cpus = util_read_content_from_file(CGROUP2_CPUSET_CPUS_EFFECTIVE_PATH); ++ cpusetinfo->mems = util_read_content_from_file(CGROUP2_CPUSET_MEMS_EFFECTIVE_PATH); ++ if (cpusetinfo->cpus == NULL || cpusetinfo->mems == NULL) { ++ ERROR("read cpus or mems failed"); ++ return -1; ++ } ++ cpusetinfo->cpus = util_trim_space(cpusetinfo->cpus); ++ cpusetinfo->mems = util_trim_space(cpusetinfo->mems); ++ } ++ ++ // io cgroup ++ blkioinfo->blkio_weight = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_IO_BFQ_WEIGHT) || ++ cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_IO_WEIGHT); ++ blkioinfo->blkio_weight_device = blkioinfo->blkio_weight; ++ common_cgroup_do_log(quiet, !(blkioinfo->blkio_weight), "Your kernel does not support cgroup2 io weight"); ++ ++ blkioinfo->blkio_read_bps_device = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_IO_MAX); ++ blkioinfo->blkio_write_bps_device = blkioinfo->blkio_read_bps_device; ++ blkioinfo->blkio_read_iops_device = blkioinfo->blkio_read_bps_device; ++ blkioinfo->blkio_write_iops_device = blkioinfo->blkio_read_bps_device; ++ common_cgroup_do_log(quiet, !(blkioinfo->blkio_read_bps_device), "Your kernel does not support cgroup2 io max"); ++ ++ // memory cgroup ++ meminfo->limit = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_MEMORY_MAX); ++ common_cgroup_do_log(quiet, !(meminfo->limit), "Your kernel does not support cgroup2 memory max"); ++ ++ meminfo->reservation = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_MEMORY_LOW); ++ common_cgroup_do_log(quiet, !(meminfo->reservation), "Your kernel does not support cgroup2 memory low"); ++ ++ meminfo->swap = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_MEMORY_SWAP_MAX); ++ common_cgroup_do_log(quiet, !(meminfo->swap), "Your kernel does not support cgroup2 memory swap max"); ++ ++ // pids cgroup ++ pidsinfo->pidslimit = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_PIDS_MAX); ++ common_cgroup_do_log(quiet, !(pidsinfo->pidslimit), "Your kernel does not support cgroup2 pids max"); ++ ++ // hugetlb cgroup ++ size = get_default_huge_page_size(); ++ if (size != NULL) { ++ nret = snprintf(path, sizeof(path), CGROUP2_HUGETLB_MAX, size); ++ if (nret < 0 || (size_t)nret >= sizeof(path)) { ++ WARN("Failed to print hugetlb path"); ++ return -1; ++ } ++ hugetlbinfo->hugetlblimit = cgroup_v2_enabled(CGROUP_ISULAD_PATH, path); ++ common_cgroup_do_log(quiet, !hugetlbinfo->hugetlblimit, "Your kernel does not support cgroup2 hugetlb limit"); ++ } else { ++ WARN("Your kernel does not support cgroup2 hugetlb limit"); ++ } ++ ++ // files cgroup ++ filesinfo->fileslimit = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_FILES_LIMIT); ++ common_cgroup_do_log(quiet, !(filesinfo->fileslimit), "Your kernel does not support cgroup2 files limit"); ++ ++ return ret; ++} ++ ++int common_get_cgroup_v2_metrics(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics) ++{ ++ if (cgroup_path == NULL || strlen(cgroup_path) == 0 || cgroup_metrics == NULL) { ++ ERROR("Invalid arguments"); ++ return -1; ++ } ++ ++ get_cgroup_v2_metrics_cpu(cgroup_path, &cgroup_metrics->cgcpu_metrics); ++ get_cgroup_v2_metrics_memory(cgroup_path, &cgroup_metrics->cgmem_metrics); ++ get_cgroup_v2_metrics_pid(cgroup_path, &cgroup_metrics->cgpids_metrics); ++ ++ return 0; ++} +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +index 76fa17bc..3bdc3af8 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +@@ -881,7 +881,7 @@ void PodSandboxManagerService::GetPodSandboxCgroupMetrics(const std::string &cgr + if (cgroupVersion == CGROUP_VERSION_1) { + nret = common_get_cgroup_v1_metrics(cgroupParent.c_str(), &cgroupMetrics); + } else { +- // todo: get cgroup v2 metrics ++ nret = common_get_cgroup_v2_metrics(cgroupParent.c_str(), &cgroupMetrics); + } + + if (nret != 0) { +diff --git a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +index 4d1d19eb..49a7ca54 100644 +--- a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +@@ -1337,7 +1337,7 @@ void PodSandboxManagerService::GetPodSandboxCgroupMetrics(const container_inspec + if (cgroupVersion == CGROUP_VERSION_1) { + nret = common_get_cgroup_v1_metrics(cgroupParent, &cgroupMetrics); + } else { +- // todo: get cgroup v2 metrics ++ nret = common_get_cgroup_v2_metrics(cgroupParent, &cgroupMetrics); + } + + if (nret != 0) { +diff --git a/src/utils/cutils/utils_array.h b/src/utils/cutils/utils_array.h +index 64f41496..1c084595 100644 +--- a/src/utils/cutils/utils_array.h ++++ b/src/utils/cutils/utils_array.h +@@ -18,6 +18,7 @@ + + #include <stdbool.h> + #include <stddef.h> ++#include <isula_libutils/auto_cleanup.h> + + #ifdef __cplusplus + extern "C" { +@@ -57,6 +58,11 @@ void util_free_sensitive_array(char **array); + + void util_free_sensitive_array_by_len(char **array, size_t len); + ++// define auto free function callback for char * ++define_auto_cleanup_callback(util_free_array, char *); ++// define auto free macro for char * ++#define __isula_auto_array_t auto_cleanup_tag(util_free_array) ++ + #ifdef __cplusplus + } + #endif +-- +2.34.1 + diff --git a/0028-use-supervisor-to-notify-sandbox-exit-event.patch b/0028-use-supervisor-to-notify-sandbox-exit-event.patch new file mode 100644 index 0000000..379eb90 --- /dev/null +++ b/0028-use-supervisor-to-notify-sandbox-exit-event.patch @@ -0,0 +1,254 @@ +From 835185f7c4739993c2ca26d737bb0a45277ad932 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Wed, 20 Mar 2024 15:48:42 +0800 +Subject: [PATCH 28/43] use supervisor to notify sandbox exit event + +Signed-off-by: jikai <jikai11@huawei.com> +--- + src/daemon/modules/api/container_api.h | 2 +- + .../modules/container/restore/restore.c | 6 +++- + .../modules/container/supervisor/supervisor.c | 15 +++++++- + .../modules/service/service_container.c | 9 +++-- + .../controller/shim/shim_controller.cc | 35 ++----------------- + src/daemon/sandbox/sandbox_ops.cc | 23 ++++++++++++ + src/daemon/sandbox/sandbox_ops.h | 2 ++ + 7 files changed, 53 insertions(+), 39 deletions(-) + +diff --git a/src/daemon/modules/api/container_api.h b/src/daemon/modules/api/container_api.h +index 4602d244..43d66d64 100644 +--- a/src/daemon/modules/api/container_api.h ++++ b/src/daemon/modules/api/container_api.h +@@ -270,7 +270,7 @@ bool container_is_valid_state_string(const char *state); + void container_update_health_monitor(const char *container_id); + + extern int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_info, const char *name, +- const char *runtime); ++ const char *runtime, bool sandbox_container); + + extern char *container_exit_fifo_create(const char *cont_state_path); + +diff --git a/src/daemon/modules/container/restore/restore.c b/src/daemon/modules/container/restore/restore.c +index 2669ea22..76868e28 100644 +--- a/src/daemon/modules/container/restore/restore.c ++++ b/src/daemon/modules/container/restore/restore.c +@@ -57,6 +57,7 @@ static int restore_supervisor(const container_t *cont) + char *statepath = cont->state_path; + char *runtime = cont->runtime; + pid_ppid_info_t pid_info = { 0 }; ++ bool sandbox_container = false; + + nret = snprintf(container_state, sizeof(container_state), "%s/%s", statepath, id); + if (nret < 0 || (size_t)nret >= sizeof(container_state)) { +@@ -90,8 +91,11 @@ static int restore_supervisor(const container_t *cont) + pid_info.ppid = cont->state->state->p_pid; + pid_info.start_time = cont->state->state->start_time; + pid_info.pstart_time = cont->state->state->p_start_time; ++#ifdef ENABLE_CRI_API_V1 ++ sandbox_container = is_sandbox_container(cont->common_config->sandbox_info); ++#endif + +- if (container_supervisor_add_exit_monitor(exit_fifo_fd, &pid_info, id, runtime)) { ++ if (container_supervisor_add_exit_monitor(exit_fifo_fd, &pid_info, id, runtime, sandbox_container)) { + ERROR("Failed to add exit monitor to supervisor"); + ret = -1; + goto out; +diff --git a/src/daemon/modules/container/supervisor/supervisor.c b/src/daemon/modules/container/supervisor/supervisor.c +index 1f9a043c..63289283 100644 +--- a/src/daemon/modules/container/supervisor/supervisor.c ++++ b/src/daemon/modules/container/supervisor/supervisor.c +@@ -38,6 +38,9 @@ + #include "container_api.h" + #include "event_type.h" + #include "utils_file.h" ++#ifdef ENABLE_CRI_API_V1 ++#include "sandbox_ops.h" ++#endif + + pthread_mutex_t g_supervisor_lock = PTHREAD_MUTEX_INITIALIZER; + struct epoll_descr g_supervisor_descr; +@@ -47,6 +50,7 @@ struct supervisor_handler_data { + int exit_code; + char *name; + char *runtime; ++ bool is_sandbox_container; + pid_ppid_info_t pid_info; + }; + +@@ -211,6 +215,14 @@ retry: + + (void)isulad_monitor_send_container_event(name, STOPPED, (int)pid, data->exit_code, NULL, NULL); + ++#ifdef ENABLE_CRI_API_V1 ++ if (data->is_sandbox_container) { ++ if (sandbox_on_sandbox_exit(name, data->exit_code) < 0) { ++ ERROR("Failed to handle sandbox %s exit", name); ++ } ++ } ++#endif ++ + supervisor_handler_data_free(data); + + DAEMON_CLEAR_ERRMSG(); +@@ -259,7 +271,7 @@ static int supervisor_exit_cb(int fd, uint32_t events, void *cbdata, struct epol + + /* supervisor add exit monitor */ + int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_info, const char *name, +- const char *runtime) ++ const char *runtime, bool sandbox_container) + { + int ret = 0; + struct supervisor_handler_data *data = NULL; +@@ -285,6 +297,7 @@ int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_inf + data->fd = fd; + data->name = util_strdup_s(name); + data->runtime = util_strdup_s(runtime); ++ data->is_sandbox_container = sandbox_container; + data->pid_info.pid = pid_info->pid; + data->pid_info.start_time = pid_info->start_time; + data->pid_info.ppid = pid_info->ppid; +diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c +index a3606a82..7b34cc7f 100644 +--- a/src/daemon/modules/service/service_container.c ++++ b/src/daemon/modules/service/service_container.c +@@ -275,13 +275,14 @@ static void clean_resources_on_failure(const container_t *cont, const char *engi + return; + } + +-static int do_post_start_on_success(const char *id, const char *runtime, const char *pidfile, int exit_fifo_fd, ++static int do_post_start_on_success(const char *id, const char *runtime, bool sandbox_container, ++ const char *pidfile, int exit_fifo_fd, + const pid_ppid_info_t *pid_info) + { + int ret = 0; + + // exit_fifo_fd was closed in container_supervisor_add_exit_monitor +- if (container_supervisor_add_exit_monitor(exit_fifo_fd, pid_info, id, runtime)) { ++ if (container_supervisor_add_exit_monitor(exit_fifo_fd, pid_info, id, runtime, sandbox_container)) { + ERROR("Failed to add exit monitor to supervisor"); + ret = -1; + } +@@ -749,6 +750,7 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo + oci_runtime_spec *oci_spec = NULL; + rt_create_params_t create_params = { 0 }; + rt_start_params_t start_params = { 0 }; ++ bool sandbox_container; + + nret = snprintf(bundle, sizeof(bundle), "%s/%s", cont->root_path, id); + if (nret < 0 || (size_t)nret >= sizeof(bundle)) { +@@ -897,6 +899,7 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo + if (cont->common_config->sandbox_info != NULL) { + create_params.task_addr = cont->common_config->sandbox_info->task_address; + } ++ sandbox_container = is_sandbox_container(cont->common_config->sandbox_info); + #endif + + if (runtime_create(id, runtime, &create_params) != 0) { +@@ -921,7 +924,7 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo + + ret = runtime_start(id, runtime, &start_params, pid_info); + if (ret == 0) { +- if (do_post_start_on_success(id, runtime, pidfile, exit_fifo_fd, pid_info) != 0) { ++ if (do_post_start_on_success(id, runtime, sandbox_container, pidfile, exit_fifo_fd, pid_info) != 0) { + ERROR("Failed to do post start on runtime start success"); + ret = -1; + goto clean_resources; +diff --git a/src/daemon/sandbox/controller/shim/shim_controller.cc b/src/daemon/sandbox/controller/shim/shim_controller.cc +index 39fcf8ea..593fade9 100644 +--- a/src/daemon/sandbox/controller/shim/shim_controller.cc ++++ b/src/daemon/sandbox/controller/shim/shim_controller.cc +@@ -397,39 +397,8 @@ bool ShimController::Stop(const std::string &sandboxId, uint32_t timeoutSecs, Er + + bool ShimController::Wait(std::shared_ptr<SandboxStatusCallback> cb, const std::string &sandboxId, Errors &error) + { +- std::thread([this, cb, sandboxId]() { +- if (m_cb == nullptr || m_cb->container.wait == nullptr) { +- ERROR("Unimplemented callback"); +- return; +- } +- +- auto requestWrapper = makeUniquePtrCStructWrapper<container_wait_request>(free_container_wait_request); +- if (requestWrapper == nullptr) { +- ERROR("Out of memory"); +- return; +- } +- auto request = requestWrapper->get(); +- request->id = isula_strdup_s(sandboxId.c_str()); +- request->condition = WAIT_CONDITION_STOPPED; +- container_wait_response *response { nullptr }; +- +- int ret = m_cb->container.wait(request, &response); +- auto responseWrapper = makeUniquePtrCStructWrapper<container_wait_response>(response, free_container_wait_response); +- +- if (ret != 0) { +- std::string msg = (response != nullptr && response->errmsg != nullptr) ? response->errmsg : "internal"; +- ERROR("Failed to wait sandbox %s: %s", sandboxId.c_str(), msg.c_str()); +- return; +- } +- +- ControllerExitInfo info; +- auto currentTime = std::chrono::high_resolution_clock::now(); +- auto duration = currentTime.time_since_epoch(); +- info.exitedAt = std::chrono::duration_cast<std::chrono::nanoseconds>(duration).count(); +- info.exitStatus = response->exit_code; +- cb->OnSandboxExit(info); +- }).detach(); +- ++ // ShimController will use sandbox_on_exit callback of supervisor in lower container level ++ // to notify the sandbox exit event + return true; + } + +diff --git a/src/daemon/sandbox/sandbox_ops.cc b/src/daemon/sandbox/sandbox_ops.cc +index 005063c0..b7fb40bf 100644 +--- a/src/daemon/sandbox/sandbox_ops.cc ++++ b/src/daemon/sandbox/sandbox_ops.cc +@@ -18,6 +18,7 @@ + #include <isula_libutils/log.h> + + #include "controller_manager.h" ++#include "sandbox_manager.h" + #include "namespace.h" + #include "utils.h" + +@@ -175,3 +176,25 @@ int sandbox_purge_exec(const container_config_v2_common_config *config, const ch + { + return do_sandbox_purge(config, exec_id); + } ++ ++int sandbox_on_sandbox_exit(const char *sandbox_id, int exit_code) ++{ ++ if (nullptr == sandbox_id) { ++ ERROR("Invalid parameter: sandbox_id"); ++ return -1; ++ } ++ ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(sandbox_id); ++ if (nullptr == sandbox) { ++ ERROR("Sandbox %s not found", sandbox_id); ++ return -1; ++ } ++ ++ sandbox::ControllerExitInfo info; ++ auto currentTime = std::chrono::high_resolution_clock::now(); ++ auto duration = currentTime.time_since_epoch(); ++ info.exitedAt = std::chrono::duration_cast<std::chrono::nanoseconds>(duration).count(); ++ info.exitStatus = exit_code; ++ sandbox->OnSandboxExit(info); ++ return 0; ++} +diff --git a/src/daemon/sandbox/sandbox_ops.h b/src/daemon/sandbox/sandbox_ops.h +index bef884fb..8189efd6 100644 +--- a/src/daemon/sandbox/sandbox_ops.h ++++ b/src/daemon/sandbox/sandbox_ops.h +@@ -36,6 +36,8 @@ int sandbox_purge_container(const container_config_v2_common_config *config); + + int sandbox_purge_exec(const container_config_v2_common_config *config, const char *exec_id); + ++int sandbox_on_sandbox_exit(const char *sandbox_id, int exit_code); ++ + #ifdef __cplusplus + } + #endif +-- +2.34.1 + diff --git a/0029-refactor-cgroup-module.patch b/0029-refactor-cgroup-module.patch new file mode 100644 index 0000000..f5e853d --- /dev/null +++ b/0029-refactor-cgroup-module.patch @@ -0,0 +1,2538 @@ +From c26604ff3150babae729890c549f2784212073a1 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Wed, 20 Mar 2024 15:53:56 +0800 +Subject: [PATCH 29/43] refactor cgroup module + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/cmd/isulad/main.c | 6 + + src/daemon/common/CMakeLists.txt | 4 + + src/daemon/common/cgroup.c | 625 ---------------- + src/daemon/common/cgroup/CMakeLists.txt | 5 + + src/daemon/common/cgroup/cgroup.c | 136 ++++ + src/daemon/common/cgroup/cgroup.h | 48 ++ + src/daemon/common/cgroup/cgroup_common.c | 131 ++++ + .../{cgroup.h => cgroup/cgroup_common.h} | 62 +- + src/daemon/common/{ => cgroup}/cgroup_v1.c | 699 +++++++++++++++--- + src/daemon/common/cgroup/cgroup_v1.h | 30 + + src/daemon/common/{ => cgroup}/cgroup_v2.c | 162 ++-- + src/daemon/common/cgroup/cgroup_v2.h | 30 + + src/daemon/common/sysinfo.c | 33 +- + src/daemon/common/sysinfo.h | 4 +- + .../v1/v1_cri_pod_sandbox_manager_service.cc | 13 +- + .../cri_pod_sandbox_manager_service.cc | 13 +- + src/daemon/executor/container_cb/execution.c | 13 +- + 17 files changed, 1124 insertions(+), 890 deletions(-) + delete mode 100644 src/daemon/common/cgroup.c + create mode 100644 src/daemon/common/cgroup/CMakeLists.txt + create mode 100644 src/daemon/common/cgroup/cgroup.c + create mode 100644 src/daemon/common/cgroup/cgroup.h + create mode 100644 src/daemon/common/cgroup/cgroup_common.c + rename src/daemon/common/{cgroup.h => cgroup/cgroup_common.h} (60%) + rename src/daemon/common/{ => cgroup}/cgroup_v1.c (52%) + create mode 100644 src/daemon/common/cgroup/cgroup_v1.h + rename src/daemon/common/{ => cgroup}/cgroup_v2.c (88%) + create mode 100644 src/daemon/common/cgroup/cgroup_v2.h + +diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c +index 7c0c072e..9fa87bdb 100644 +--- a/src/cmd/isulad/main.c ++++ b/src/cmd/isulad/main.c +@@ -82,6 +82,7 @@ + #include "network_api.h" + #endif + #include "id_name_manager.h" ++#include "cgroup.h" + + sem_t g_daemon_shutdown_sem; + sem_t g_daemon_wait_shutdown_sem; +@@ -1706,6 +1707,11 @@ static int pre_init_daemon(int argc, char **argv) + goto out; + } + ++ if (cgroup_ops_init() != 0) { ++ ERROR("Failed to init cgroup"); ++ goto out; ++ } ++ + if (server_conf_parse_save(argc, (const char **)argv)) { + ERROR("%s", g_isulad_errmsg ? g_isulad_errmsg : "Failed to parse and save server conf"); + goto out; +diff --git a/src/daemon/common/CMakeLists.txt b/src/daemon/common/CMakeLists.txt +index d634507b..e88578dd 100644 +--- a/src/daemon/common/CMakeLists.txt ++++ b/src/daemon/common/CMakeLists.txt +@@ -9,16 +9,20 @@ if (GRPC_CONNECTOR) + add_subdirectory(cri) + endif() + ++add_subdirectory(cgroup) ++ + set(local_daemon_common_srcs ${daemon_common_top_srcs}) + + set(DAEMON_COMMON_SRCS + ${COMMON_CRI_SRCS} ++ ${COMMON_CGROUP_SRCS} + ${local_daemon_common_srcs} + PARENT_SCOPE + ) + + set(DAEMON_COMMON_INCS + ${COMMON_CRI_INCS} ++ ${COMMON_CGROUP_INCS} + ${CMAKE_CURRENT_SOURCE_DIR} + PARENT_SCOPE + ) +diff --git a/src/daemon/common/cgroup.c b/src/daemon/common/cgroup.c +deleted file mode 100644 +index 3c58f7fa..00000000 +--- a/src/daemon/common/cgroup.c ++++ /dev/null +@@ -1,625 +0,0 @@ +-/****************************************************************************** +- * Copyright (c) Huawei Technologies Co., Ltd. 2017-2023. All rights reserved. +- * iSulad licensed under the Mulan PSL v2. +- * You can use this software according to the terms and conditions of the Mulan PSL v2. +- * You may obtain a copy of Mulan PSL v2 at: +- * http://license.coscl.org.cn/MulanPSL2 +- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +- * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +- * PURPOSE. +- * See the Mulan PSL v2 for more details. +- * Author: zhangxiaoyu +- * Create: 2023-03-29 +- * Description: provide cgroup functions +- ******************************************************************************/ +-#include "cgroup.h" +- +-#include <stdio.h> +-#include <stdlib.h> +-#include <string.h> +-#include <errno.h> +-#include <sys/vfs.h> +-#include <linux/magic.h> +-#include <sys/stat.h> +- +-#include <isula_libutils/auto_cleanup.h> +- +-#include "err_msg.h" +-#include "utils.h" +-#include "utils_array.h" +-#include "sysinfo.h" +- +-#ifndef CGROUP_SUPER_MAGIC +-#define CGROUP_SUPER_MAGIC 0x27e0eb +-#endif +- +-static cgroup_layer_t *new_cgroup_layer(size_t len) +-{ +- cgroup_layer_t *layers = NULL; +- +- if (len == 0) { +- return NULL; +- } +- +- layers = (cgroup_layer_t *)util_common_calloc_s(sizeof(cgroup_layer_t)); +- if (layers == NULL) { +- ERROR("Out of memory"); +- return NULL; +- } +- +- layers->items = (cgroup_layers_item **)util_smart_calloc_s(sizeof(cgroup_layers_item *), len); +- if (layers->items == NULL) { +- ERROR("Out of memory"); +- free(layers); +- return NULL; +- } +- +- layers->len = 0; +- layers->cap = len; +- +- return layers; +-} +- +-static int add_cgroup_layer(cgroup_layer_t *layers, char **clist, char *mountpoint) +-{ +-#define CGROUP_LAYER_MAX_CAPABILITY 1024 +- size_t new_size; +- cgroup_layers_item *newh = NULL; +- cgroup_layers_item **tmp = NULL; +- +- if (layers->len >= CGROUP_LAYER_MAX_CAPABILITY) { +- ERROR("Too many cgroup layers"); +- return -1; +- } +- +- newh = util_common_calloc_s(sizeof(cgroup_layers_item)); +- if (newh == NULL) { +- ERROR("Out of memory"); +- return -1; +- } +- newh->controllers = clist; +- newh->mountpoint = mountpoint; +- +- if (layers->len < layers->cap) { +- goto out; +- } +- +- if (layers->cap > CGROUP_LAYER_MAX_CAPABILITY / 2) { +- new_size = CGROUP_LAYER_MAX_CAPABILITY; +- } else { +- new_size = layers->cap * 2; +- } +- +- if (util_mem_realloc((void **)&tmp, new_size * sizeof(cgroup_layers_item *), +- layers->items, layers->cap * sizeof(cgroup_layers_item *)) != 0) { +- ERROR("Failed to realloc memory"); +- free(newh); +- return -1; +- } +- +- layers->items = tmp; +- tmp = NULL; +- layers->cap = new_size; +- +-out: +- layers->items[layers->len] = newh; +- layers->len++; +- return 0; +-} +- +-void common_free_cgroup_layer(cgroup_layer_t *layers) +-{ +- size_t i; +- +- if (layers == NULL) { +- return; +- } +- +- for (i = 0; i < layers->len && layers->items[i]; i++) { +- free(layers->items[i]->mountpoint); +- layers->items[i]->mountpoint = NULL; +- util_free_array(layers->items[i]->controllers); +- layers->items[i]->controllers = NULL; +- free(layers->items[i]); +- layers->items[i] = NULL; +- } +- +- free(layers->items); +- layers->items = NULL; +- layers->len = 0; +- layers->cap = 0; +- +- free(layers); +-} +- +-static int append_subsystem_to_list(char ***klist, char ***nlist, const char *ptoken) +-{ +- int ret = 0; +- +- if (strncmp(ptoken, "name=", strlen("name=")) == 0) { +- ret = util_array_append(nlist, ptoken); +- if (ret != 0) { +- ERROR("Failed to append string"); +- return -1; +- } +- } else { +- ret = util_array_append(klist, ptoken); +- if (ret != 0) { +- ERROR("Failed to append string"); +- return -1; +- } +- } +- +- return 0; +-} +- +-static int get_cgroup_subsystems(char ***klist, char ***nlist) +-{ +- int ret = 0; +- size_t length = 0; +- FILE *fp = NULL; +- char *pline = NULL; +- +- fp = util_fopen("/proc/self/cgroup", "r"); +- if (fp == NULL) { +- return -1; +- } +- +- while (getline(&pline, &length, fp) != -1) { +- char *pos = NULL; +- char *pos2 = NULL; +- char *ptoken = NULL; +- char *psave = NULL; +- pos = strchr(pline, ':'); +- if (pos == NULL) { +- ERROR("Invalid cgroup entry: must contain at least two colons: %s", pline); +- ret = -1; +- goto out; +- } +- pos++; +- pos2 = strchr(pos, ':'); +- if (pos2 == NULL) { +- ERROR("Invalid cgroup entry: must contain at least two colons: %s", pline); +- ret = -1; +- goto out; +- } +- *pos2 = '\0'; +- +- if ((pos2 - pos) == 0) { +- INFO("Not supported cgroup entry: %s", pline); +- continue; +- } +- +- for (ptoken = strtok_r(pos, ",", &psave); ptoken; ptoken = strtok_r(NULL, ",", &psave)) { +- if (append_subsystem_to_list(klist, nlist, ptoken)) { +- goto out; +- } +- } +- } +- +-out: +- free(pline); +- fclose(fp); +- if (ret != 0) { +- util_free_array(*klist); +- *klist = NULL; +- util_free_array(*nlist); +- *nlist = NULL; +- } +- return ret; +-} +- +-static int append_controller(const char **klist, const char **nlist, char ***clist, const char *entry) +-{ +- int ret = 0; +- char *dup_entry = NULL; +- +- if (util_array_contain(klist, entry) && util_array_contain(nlist, entry)) { +- ERROR("Refusing to use ambiguous controller \"%s\"", entry); +- ERROR("It is both a named and kernel subsystem"); +- return -1; +- } +- +- if (strncmp(entry, "name=", 5) == 0) { +- dup_entry = util_strdup_s(entry); +- } else if (util_array_contain(klist, entry)) { +- dup_entry = util_strdup_s(entry); +- } else { +- dup_entry = util_string_append(entry, "name="); +- } +- if (dup_entry == NULL) { +- ERROR("Out of memory"); +- return -1; +- } +- +- ret = util_array_append(clist, dup_entry); +- if (ret != 0) { +- ERROR("Failed to append array"); +- } +- +- free(dup_entry); +- return ret; +-} +- +-static inline bool is_cgroup_mountpoint(const char *mp) +-{ +- return strncmp(mp, "/sys/fs/cgroup/", strlen("/sys/fs/cgroup/")) == 0; +-} +- +-static char **cgroup_get_controllers(const char **klist, const char **nlist, const char *line) +-{ +- int index; +- char *dup = NULL; +- char *pos2 = NULL; +- char *tok = NULL; +- const char *pos = line; +- char *psave = NULL; +- char *sep = ","; +- char **pret = NULL; +- +- // line example +- // 108 99 0:55 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime - tmpfs tmpfs rw,mode=755 +- for (index = 0; index < 4; index++) { +- pos = strchr(pos, ' '); +- if (pos == NULL) { +- ERROR("Invalid mountinfo format \"%s\"", line); +- return NULL; +- } +- pos++; +- } +- +- if (!is_cgroup_mountpoint(pos)) { +- return NULL; +- } +- +- pos += strlen("/sys/fs/cgroup/"); +- pos2 = strchr(pos, ' '); +- if (pos2 == NULL) { +- ERROR("Invalid mountinfo format \"%s\"", line); +- return NULL; +- } +- +- *pos2 = '\0'; +- dup = util_strdup_s(pos); +- *pos2 = ' '; +- +- for (tok = strtok_r(dup, sep, &psave); tok; tok = strtok_r(NULL, sep, &psave)) { +- if (append_controller(klist, nlist, &pret, tok)) { +- ERROR("Failed to append controller"); +- util_free_array(pret); +- pret = NULL; +- break; +- } +- } +- +- free(dup); +- +- return pret; +-} +- +-int cgroup_get_mountpoint_and_root(char *pline, char **mountpoint, char **root) +-{ +- int index; +- char *posmp = NULL; +- char *posrt = NULL; +- char *pos = pline; +- +- // find root +- // line example +- // 108 99 0:55 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime - tmpfs tmpfs rw,mode=755 +- for (index = 0; index < 3; index++) { +- pos = strchr(pos, ' '); +- if (pos == NULL) { +- return -1; +- } +- pos++; +- } +- posrt = pos; +- +- // find mountpoint +- pos = strchr(pos, ' '); +- if (pos == NULL) { +- return -1; +- } +- +- *pos = '\0'; +- if (root != NULL) { +- *root = util_strdup_s(posrt); +- } +- +- pos++; +- posmp = pos; +- +- if (!is_cgroup_mountpoint(posmp)) { +- return -1; +- } +- +- pos = strchr(pos + strlen("/sys/fs/cgroup/"), ' '); +- if (pos == NULL) { +- return -1; +- } +- *pos = '\0'; +- +- if (mountpoint != NULL) { +- *mountpoint = util_strdup_s(posmp); +- } +- +- return 0; +-} +- +-static bool lists_intersect(const char **controllers, const char **list) +-{ +- int index; +- +- if (controllers == NULL || list == NULL) { +- return false; +- } +- +- for (index = 0; controllers[index]; index++) { +- if (util_array_contain(list, controllers[index])) { +- return true; +- } +- } +- +- return false; +-} +- +-static bool controller_list_is_dup(const cgroup_layer_t *llist, const char **clist) +-{ +- size_t index; +- +- if (llist == NULL) { +- return false; +- } +- +- for (index = 0; index < llist->len && llist->items[index]; index++) { +- if (lists_intersect((const char **)llist->items[index]->controllers, (const char **)clist)) { +- return true; +- } +- } +- +- return false; +-} +- +-cgroup_layer_t *common_cgroup_layers_find(void) +-{ +- int nret; +- int ret = 0; +- FILE *fp = NULL; +- size_t length = 0; +- const size_t cgroup_layer_item_num = 10; +- char *pline = NULL; +- char **klist = NULL; +- char **nlist = NULL; +- cgroup_layer_t *layers = NULL; +- +- layers = new_cgroup_layer(cgroup_layer_item_num); +- if (layers == NULL) { +- ERROR("Failed to new cgroup layer"); +- return NULL; +- } +- +- ret = get_cgroup_subsystems(&klist, &nlist); +- if (ret != 0) { +- ERROR("Failed to retrieve available legacy cgroup controllers\n"); +- goto out; +- } +- +- fp = util_fopen("/proc/self/mountinfo", "r"); +- if (fp == NULL) { +- ERROR("Failed to open \"/proc/self/mountinfo\"\n"); +- ret = -1; +- goto out; +- } +- +- while (getline(&pline, &length, fp) != -1) { +- char *mountpoint = NULL; +- char **clist = NULL; +- int mret; +- +- clist = cgroup_get_controllers((const char **)klist, (const char **)nlist, pline); +- if (clist == NULL) { +- goto list_out; +- } +- +- if (controller_list_is_dup(layers, (const char **)clist)) { +- goto list_out; +- } +- +- mret = cgroup_get_mountpoint_and_root(pline, &mountpoint, NULL); +- if (mret != 0 || mountpoint == NULL) { +- ERROR("Failed parsing mountpoint from \"%s\"\n", pline); +- goto list_out; +- } +- +- nret = add_cgroup_layer(layers, clist, mountpoint); +- if (nret != 0) { +- ERROR("Failed to add hierarchies"); +- goto list_out; +- } +- +- continue; +-list_out: +- util_free_array(clist); +- free(mountpoint); +- } +-out: +- util_free_array(klist); +- util_free_array(nlist); +- if (fp != NULL) { +- fclose(fp); +- } +- free(pline); +- +- if (ret != 0) { +- common_free_cgroup_layer(layers); +- return NULL; +- } +- +- return layers; +-} +- +-char *common_find_cgroup_subsystem_mountpoint(const cgroup_layer_t *layers, const char *subsystem) +-{ +- size_t i; +- +- for (i = 0; i < layers->len && layers->items[i]; i++) { +- char **cit = NULL; +- +- for (cit = layers->items[i]->controllers; cit && *cit; cit++) { +- if (strcmp(*cit, subsystem) == 0) { +- return layers->items[i]->mountpoint; +- } +- } +- } +- return NULL; +-} +- +-/* find cgroup mountpoint and root */ +-int common_find_cgroup_mnt_and_root(const char *subsystem, char **mountpoint, char **root) +-{ +- int ret = 0; +- FILE *fp = NULL; +- size_t length = 0; +- char *pline = NULL; +- +- if (subsystem == NULL) { +- ERROR("Empty subsystem"); +- return -1; +- } +- +- fp = util_fopen("/proc/self/mountinfo", "r"); +- if (fp == NULL) { +- ERROR("Failed to open \"/proc/self/mountinfo\"\n"); +- ret = -1; +- goto free_out; +- } +- +- while (getline(&pline, &length, fp) != -1) { +- char *dup = NULL; +- char *p = NULL; +- char *tok = NULL; +- char *mp = NULL; +- char *rt = NULL; +- char *saveptr = NULL; +- char *sep = ","; +- int mret; +- +- mret = cgroup_get_mountpoint_and_root(pline, &mp, &rt); +- if (mret != 0 || mp == NULL || rt == NULL) { +- goto mp_out; +- } +- +- p = mp; +- p += strlen("/sys/fs/cgroup/"); +- dup = util_strdup_s(p); +- if (dup == NULL) { +- ERROR("Out of memory"); +- free(mp); +- ret = -1; +- goto free_out; +- } +- +- for (tok = strtok_r(dup, sep, &saveptr); tok; tok = strtok_r(NULL, sep, &saveptr)) { +- if (strcmp(tok, subsystem) != 0) { +- continue; +- } +- if (mountpoint != NULL) { +- *mountpoint = mp; +- } else { +- free(mp); +- } +- if (root != NULL) { +- *root = rt; +- } else { +- free(rt); +- } +- free(dup); +- goto free_out; +- } +- free(dup); +-mp_out: +- free(mp); +- free(rt); +- continue; +- } +-free_out: +- if (fp != NULL) { +- fclose(fp); +- } +- free(pline); +- return ret; +-} +- +-int common_get_cgroup_version(void) +-{ +- struct statfs fs = { 0 }; +- +- if (statfs(CGROUP_MOUNTPOINT, &fs) != 0) { +- SYSERROR("failed to statfs %s", CGROUP_MOUNTPOINT); +- return -1; +- } +- +- if (fs.f_type == CGROUP2_SUPER_MAGIC) { +- return CGROUP_VERSION_2; +- } +- +- return CGROUP_VERSION_1; +-} +- +-static int get_value_ull(const char *content, void *result) +-{ +- uint64_t ull_result = 0; +- +- if (util_safe_uint64(content, &ull_result) != 0) { +- ERROR("Failed to convert %s to uint64", content); +- return -1; +- } +- +- *(uint64_t *)result = ull_result; +- return 0; +-} +- +-int get_match_value_ull(const char *content, const char *match, void *result) +-{ +- __isula_auto_free char *llu_string = NULL; +- __isula_auto_free char *match_with_space = NULL; +- __isula_auto_array_t char **lines = NULL; +- char **worker = NULL; +- +- if (match == NULL) { +- return get_value_ull(content, result); +- } +- +- // match full string +- match_with_space = util_string_append(" ", match); +- if (match_with_space == NULL) { +- ERROR("Failed to append string"); +- return -1; +- } +- +- lines = util_string_split(content, '\n'); +- if (lines == NULL) { +- ERROR("Failed to split content %s", content); +- return -1; +- } +- +- for (worker = lines; worker && *worker; worker++) { +- if (util_has_prefix(*worker, match_with_space)) { +- break; +- } +- } +- if (*worker == NULL) { +- ERROR("Cannot find match string %s", match); +- return -1; +- } +- +- llu_string = util_sub_string(*worker, strlen(match_with_space), strlen(*worker) - strlen(match_with_space)); +- if (llu_string == NULL) { +- ERROR("Failed to sub string"); +- return -1; +- } +- llu_string = util_trim_space(llu_string); +- +- return get_value_ull(llu_string, result); +-} +\ No newline at end of file +diff --git a/src/daemon/common/cgroup/CMakeLists.txt b/src/daemon/common/cgroup/CMakeLists.txt +new file mode 100644 +index 00000000..e8c1726c +--- /dev/null ++++ b/src/daemon/common/cgroup/CMakeLists.txt +@@ -0,0 +1,5 @@ ++# get current directory sources files ++aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_common_cgroup_srcs) ++ ++set(COMMON_CGROUP_SRCS ${local_common_cgroup_srcs} PARENT_SCOPE) ++set(COMMON_CGROUP_INCS ${CMAKE_CURRENT_SOURCE_DIR} PARENT_SCOPE) +diff --git a/src/daemon/common/cgroup/cgroup.c b/src/daemon/common/cgroup/cgroup.c +new file mode 100644 +index 00000000..837b514a +--- /dev/null ++++ b/src/daemon/common/cgroup/cgroup.c +@@ -0,0 +1,136 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2017-2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhangxiaoyu ++ * Create: 2023-03-29 ++ * Description: provide cgroup functions ++ ******************************************************************************/ ++#include "cgroup.h" ++ ++#include <stdio.h> ++#include <stdlib.h> ++#include <string.h> ++#include <errno.h> ++#include <sys/vfs.h> ++#include <linux/magic.h> ++#include <sys/stat.h> ++ ++#include <isula_libutils/auto_cleanup.h> ++ ++#include "err_msg.h" ++#include "utils.h" ++#include "utils_array.h" ++#include "sysinfo.h" ++#include "cgroup_v1.h" ++#include "cgroup_v2.h" ++#include "path.h" ++ ++#ifndef CGROUP_SUPER_MAGIC ++#define CGROUP_SUPER_MAGIC 0x27e0eb ++#endif ++ ++static cgroup_ops g_cgroup_ops; ++ ++static int get_cgroup_version_for_init(void) ++{ ++ struct statfs fs = { 0 }; ++ ++ if (statfs(CGROUP_MOUNTPOINT, &fs) != 0) { ++ SYSERROR("failed to statfs %s", CGROUP_MOUNTPOINT); ++ return -1; ++ } ++ ++ if (fs.f_type == CGROUP2_SUPER_MAGIC) { ++ return CGROUP_VERSION_2; ++ } ++ ++ return CGROUP_VERSION_1; ++} ++ ++/* connect client ops init */ ++int cgroup_ops_init(void) ++{ ++ (void)memset(&g_cgroup_ops, 0, sizeof(g_cgroup_ops)); ++ int cgroupVersion = get_cgroup_version_for_init(); ++ if (cgroupVersion < 0) { ++ ERROR("Invalid cgroup version"); ++ return -1; ++ } ++ ++ if (cgroupVersion == CGROUP_VERSION_1) { ++ return cgroup_v1_ops_init(&g_cgroup_ops); ++ } else { ++ return cgroup_v2_ops_init(&g_cgroup_ops); ++ } ++} ++ ++int common_get_cgroup_version(void) ++{ ++ if (g_cgroup_ops.get_cgroup_version == NULL) { ++ ERROR("Unimplemented get_cgroup_version ops"); ++ return -1; ++ } ++ ++ return g_cgroup_ops.get_cgroup_version(); ++} ++ ++int common_get_cgroup_info(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, ++ cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, ++ cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, ++ cgroup_files_info_t *filesinfo, bool quiet) ++{ ++ if (g_cgroup_ops.get_cgroup_info == NULL) { ++ ERROR("Unimplemented get_cgroup_info ops"); ++ return -1; ++ } ++ ++ return g_cgroup_ops.get_cgroup_info(meminfo, cpuinfo, hugetlbinfo, blkioinfo, cpusetinfo, pidsinfo, filesinfo, quiet); ++} ++ ++int common_get_cgroup_metrics(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics) ++{ ++ if (g_cgroup_ops.get_cgroup_metrics == NULL) { ++ ERROR("Unimplemented get_cgroup_metrics ops"); ++ return -1; ++ } ++ ++ return g_cgroup_ops.get_cgroup_metrics(cgroup_path, cgroup_metrics); ++} ++ ++int common_get_cgroup_mnt_and_root_path(const char *subsystem, char **mountpoint, char **root) ++{ ++ if (g_cgroup_ops.get_cgroup_mnt_and_root_path == NULL) { ++ ERROR("Unimplemented get_cgroup_mnt_and_root_path ops"); ++ return -1; ++ } ++ ++ return g_cgroup_ops.get_cgroup_mnt_and_root_path(subsystem, mountpoint, root); ++} ++ ++// only for cgroup v1 ++char *common_get_init_cgroup_path(const char *subsystem) ++{ ++ if (g_cgroup_ops.get_init_cgroup_path == NULL) { ++ ERROR("Unimplemented get_init_cgroup_path ops"); ++ return NULL; ++ } ++ ++ return g_cgroup_ops.get_init_cgroup_path(subsystem); ++} ++ ++char *common_get_own_cgroup_path(const char *subsystem) ++{ ++ if (g_cgroup_ops.get_own_cgroup_path == NULL) { ++ ERROR("Unimplemented get_own_cgroup_path ops"); ++ return NULL; ++ } ++ ++ return g_cgroup_ops.get_own_cgroup_path(subsystem); ++} +\ No newline at end of file +diff --git a/src/daemon/common/cgroup/cgroup.h b/src/daemon/common/cgroup/cgroup.h +new file mode 100644 +index 00000000..1efc3ca6 +--- /dev/null ++++ b/src/daemon/common/cgroup/cgroup.h +@@ -0,0 +1,48 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2017-2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhangxiaoyu ++ * Create: 2023-03-29 ++ * Description: provide cgroup definition ++ ******************************************************************************/ ++#ifndef DAEMON_COMMON_CGROUP_H ++#define DAEMON_COMMON_CGROUP_H ++ ++#include <stdbool.h> ++#include <stdint.h> ++#include <stdio.h> ++ ++#include <isula_libutils/log.h> ++ ++#include "cgroup_common.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++int cgroup_ops_init(void); ++ ++int common_get_cgroup_version(void); ++int common_get_cgroup_info(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, ++ cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, ++ cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, ++ cgroup_files_info_t *filesinfo, bool quiet); ++int common_get_cgroup_metrics(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics); ++int common_get_cgroup_mnt_and_root_path(const char *subsystem, char **mountpoint, char **root); ++ ++// only for cgroup v1 ++char *common_get_init_cgroup_path(const char *subsystem); ++char *common_get_own_cgroup_path(const char *subsystem); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif // DAEMON_COMMON_CGROUP_H +diff --git a/src/daemon/common/cgroup/cgroup_common.c b/src/daemon/common/cgroup/cgroup_common.c +new file mode 100644 +index 00000000..0fb9405c +--- /dev/null ++++ b/src/daemon/common/cgroup/cgroup_common.c +@@ -0,0 +1,131 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-03-22 ++ * Description: provide cgroup common func definition ++ ******************************************************************************/ ++#include "cgroup_common.h" ++ ++#include <stdio.h> ++#include <stdlib.h> ++#include <string.h> ++#include <errno.h> ++#include <sys/vfs.h> ++#include <linux/magic.h> ++#include <sys/stat.h> ++ ++#include <isula_libutils/auto_cleanup.h> ++ ++#include "err_msg.h" ++#include "utils.h" ++#include "utils_array.h" ++#include "sysinfo.h" ++#include "cgroup_v1.h" ++#include "cgroup_v2.h" ++#include "path.h" ++ ++static int get_value_ull(const char *content, void *result) ++{ ++ uint64_t ull_result = 0; ++ ++ if (util_safe_uint64(content, &ull_result) != 0) { ++ ERROR("Failed to convert %s to uint64", content); ++ return -1; ++ } ++ ++ *(uint64_t *)result = ull_result; ++ return 0; ++} ++ ++int get_match_value_ull(const char *content, const char *match, void *result) ++{ ++ __isula_auto_free char *llu_string = NULL; ++ __isula_auto_free char *match_with_space = NULL; ++ __isula_auto_array_t char **lines = NULL; ++ char **worker = NULL; ++ ++ if (match == NULL) { ++ return get_value_ull(content, result); ++ } ++ ++ // match full string ++ match_with_space = util_string_append(" ", match); ++ if (match_with_space == NULL) { ++ ERROR("Failed to append string"); ++ return -1; ++ } ++ ++ lines = util_string_split(content, '\n'); ++ if (lines == NULL) { ++ ERROR("Failed to split content %s", content); ++ return -1; ++ } ++ ++ for (worker = lines; worker && *worker; worker++) { ++ if (util_has_prefix(*worker, match_with_space)) { ++ break; ++ } ++ } ++ if (*worker == NULL) { ++ ERROR("Cannot find match string %s", match); ++ return -1; ++ } ++ ++ llu_string = util_sub_string(*worker, strlen(match_with_space), strlen(*worker) - strlen(match_with_space)); ++ if (llu_string == NULL) { ++ ERROR("Failed to sub string"); ++ return -1; ++ } ++ llu_string = util_trim_space(llu_string); ++ ++ return get_value_ull(llu_string, result); ++} ++ ++int get_cgroup_value_helper(const char *path, struct cgfile_t *cgfile, void *result) ++{ ++ int nret = 0; ++ char file_path[PATH_MAX] = { 0 }; ++ char real_path[PATH_MAX] = { 0 }; ++ char *content = NULL; ++ ++ if (path == NULL || strlen(path) == 0 || result == NULL) { ++ ERROR("%s: Invalid arguments", cgfile->name); ++ return -1; ++ } ++ ++ nret = snprintf(file_path, sizeof(file_path), "%s/%s", path, cgfile->file); ++ if (nret < 0 || (size_t)nret >= sizeof(file_path)) { ++ ERROR("%s: failed to snprintf", cgfile->name); ++ return -1; ++ } ++ ++ if (util_clean_path(file_path, real_path, sizeof(real_path)) == NULL) { ++ ERROR("%s: failed to clean path %s", cgfile->name, file_path); ++ return -1; ++ } ++ ++ content = util_read_content_from_file(real_path); ++ if (content == NULL) { ++ ERROR("%s: failed to read file %s", cgfile->name, real_path); ++ return -1; ++ } ++ ++ util_trim_newline(content); ++ content = util_trim_space(content); ++ ++ nret = cgfile->get_value(content, cgfile->match, result); ++ if (nret != 0) { ++ ERROR("%s: failed to get value", cgfile->name); ++ } ++ ++ free(content); ++ return nret; ++} +\ No newline at end of file +diff --git a/src/daemon/common/cgroup.h b/src/daemon/common/cgroup/cgroup_common.h +similarity index 60% +rename from src/daemon/common/cgroup.h +rename to src/daemon/common/cgroup/cgroup_common.h +index 251e3a3d..2a0935cb 100644 +--- a/src/daemon/common/cgroup.h ++++ b/src/daemon/common/cgroup/cgroup_common.h +@@ -1,5 +1,5 @@ + /****************************************************************************** +- * Copyright (c) Huawei Technologies Co., Ltd. 2017-2023. All rights reserved. ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * iSulad licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: +@@ -8,12 +8,12 @@ + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. +- * Author: zhangxiaoyu +- * Create: 2023-03-29 +- * Description: provide cgroup definition ++ * Author: zhongtao ++ * Create: 2024-03-22 ++ * Description: provide cgroup common func definition + ******************************************************************************/ +-#ifndef DAEMON_COMMON_CGROUP_H +-#define DAEMON_COMMON_CGROUP_H ++#ifndef DAEMON_COMMON_CGROUP_COMMON_H ++#define DAEMON_COMMON_CGROUP_COMMON_H + + #include <stdbool.h> + #include <stdint.h> +@@ -40,10 +40,6 @@ struct cgfile_t { + + int get_match_value_ull(const char *content, const char *match, void *result); + +-int common_get_cgroup_version(void); +- +-int common_find_cgroup_mnt_and_root(const char *subsystem, char **mountpoint, char **root); +- + static inline void common_cgroup_do_log(bool quiet, bool do_log, const char *msg) + { + if (!quiet && do_log) { +@@ -51,23 +47,7 @@ static inline void common_cgroup_do_log(bool quiet, bool do_log, const char *msg + } + } + +-typedef struct { +- char **controllers; +- char *mountpoint; +-} cgroup_layers_item; +- +-typedef struct { +- cgroup_layers_item **items; +- size_t len; +- size_t cap; +-} cgroup_layer_t; +- +-char *common_find_cgroup_subsystem_mountpoint(const cgroup_layer_t *layers, const char *subsystem); +- +-cgroup_layer_t *common_cgroup_layers_find(void); +- +-void common_free_cgroup_layer(cgroup_layer_t *layers); +- ++int get_cgroup_value_helper(const char *path, struct cgfile_t *cgfile, void *result); + + typedef struct { + bool limit; +@@ -113,17 +93,6 @@ typedef struct { + bool fileslimit; + } cgroup_files_info_t; + +-int common_get_cgroup_info_v1(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, +- cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, +- cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, +- cgroup_files_info_t *filesinfo, bool quiet); +- +-int common_get_cgroup_info_v2(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, +- cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, +- cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, +- cgroup_files_info_t *filesinfo, bool quiet); +- +- + typedef struct { + uint64_t cpu_use_nanos; + } cgroup_cpu_metrics_t; +@@ -147,15 +116,22 @@ typedef struct { + cgroup_pids_metrics_t cgpids_metrics; + } cgroup_metrics_t; + +-int common_get_cgroup_v1_metrics(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics); +-int common_get_cgroup_v2_metrics(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics); ++typedef struct { ++ int (*get_cgroup_version)(void); ++ int (*get_cgroup_info)(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, ++ cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, ++ cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, ++ cgroup_files_info_t *filesinfo, bool quiet); ++ int (*get_cgroup_metrics)(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics); + +-char *common_get_init_cgroup(const char *subsystem); ++ int (*get_cgroup_mnt_and_root_path)(const char *subsystem, char **mountpoint, char **root); + +-char *common_get_own_cgroup(const char *subsystem); ++ char *(*get_init_cgroup_path)(const char *subsystem); ++ char *(*get_own_cgroup_path)(const char *subsystem); ++} cgroup_ops; + + #ifdef __cplusplus + } + #endif + +-#endif // DAEMON_COMMON_CGROUP_H ++#endif // DAEMON_COMMON_CGROUP_COMMON_H +diff --git a/src/daemon/common/cgroup_v1.c b/src/daemon/common/cgroup/cgroup_v1.c +similarity index 52% +rename from src/daemon/common/cgroup_v1.c +rename to src/daemon/common/cgroup/cgroup_v1.c +index e34100bc..51cf7512 100644 +--- a/src/daemon/common/cgroup_v1.c ++++ b/src/daemon/common/cgroup/cgroup_v1.c +@@ -18,13 +18,31 @@ + #include <stdlib.h> + + #include "utils.h" +-#include "path.h" + #include "sysinfo.h" ++#include "err_msg.h" + + #define CGROUP_HUGETLB_LIMIT "hugetlb.%s.limit_in_bytes" ++#define CGROUP_MOUNT_PATH_PREFIX "/sys/fs/cgroup/" + +-static int get_value_ll(const char *content, const char *match, void *result); +-static int get_value_string(const char *content, const char *match, void *result); ++ ++static int get_value_ll(const char *content, const char *match, void *result) ++{ ++ long long ll_result = 0; ++ ++ if (util_safe_llong(content, &ll_result) != 0) { ++ ERROR("Failed to convert %s to long long", content); ++ return -1; ++ } ++ ++ *(int64_t *)result = (int64_t)ll_result; ++ return 0; ++} ++ ++static int get_value_string(const char *content, const char *match, void *result) ++{ ++ *(char **)result = util_strdup_s(content); ++ return 0; ++} + + typedef enum { + // CPU subsystem +@@ -90,105 +108,579 @@ static struct cgfile_t g_cgroup_v1_files[] = { + [PIDS_CURRENT] = {"pids_current", "pids.current", NULL, get_match_value_ull}, + }; + +-static int get_value_ll(const char *content, const char *match, void *result) ++typedef struct { ++ char **controllers; ++ char *mountpoint; ++} cgroup_layers_item; ++ ++typedef struct { ++ cgroup_layers_item **items; ++ size_t len; ++ size_t cap; ++} cgroup_layer_t; ++ ++static char *common_find_cgroup_subsystem_mountpoint(const cgroup_layer_t *layers, const char *subsystem) + { +- long long ll_result = 0; ++ size_t i; + +- if (util_safe_llong(content, &ll_result) != 0) { +- ERROR("Failed to convert %s to long long", content); ++ for (i = 0; i < layers->len && layers->items[i]; i++) { ++ char **cit = NULL; ++ ++ for (cit = layers->items[i]->controllers; cit && *cit; cit++) { ++ if (strcmp(*cit, subsystem) == 0) { ++ return layers->items[i]->mountpoint; ++ } ++ } ++ } ++ return NULL; ++} ++ ++ ++static cgroup_layer_t *new_cgroup_layer(size_t len) ++{ ++ cgroup_layer_t *layers = NULL; ++ ++ if (len == 0) { ++ return NULL; ++ } ++ ++ layers = (cgroup_layer_t *)util_common_calloc_s(sizeof(cgroup_layer_t)); ++ if (layers == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ layers->items = (cgroup_layers_item **)util_smart_calloc_s(sizeof(cgroup_layers_item *), len); ++ if (layers->items == NULL) { ++ ERROR("Out of memory"); ++ free(layers); ++ return NULL; ++ } ++ ++ layers->len = 0; ++ layers->cap = len; ++ ++ return layers; ++} ++ ++static int add_cgroup_layer(cgroup_layer_t *layers, char **clist, char *mountpoint) ++{ ++#define CGROUP_LAYER_MAX_CAPABILITY 1024 ++ size_t new_size; ++ cgroup_layers_item *newh = NULL; ++ cgroup_layers_item **tmp = NULL; ++ ++ if (layers->len >= CGROUP_LAYER_MAX_CAPABILITY) { ++ ERROR("Too many cgroup layers"); + return -1; + } + +- *(int64_t *)result = (int64_t)ll_result; ++ newh = util_common_calloc_s(sizeof(cgroup_layers_item)); ++ if (newh == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ newh->controllers = clist; ++ newh->mountpoint = mountpoint; ++ ++ if (layers->len < layers->cap) { ++ goto out; ++ } ++ ++ if (layers->cap > CGROUP_LAYER_MAX_CAPABILITY / 2) { ++ new_size = CGROUP_LAYER_MAX_CAPABILITY; ++ } else { ++ new_size = layers->cap * 2; ++ } ++ ++ if (util_mem_realloc((void **)&tmp, new_size * sizeof(cgroup_layers_item *), ++ layers->items, layers->cap * sizeof(cgroup_layers_item *)) != 0) { ++ ERROR("Failed to realloc memory"); ++ free(newh); ++ return -1; ++ } ++ ++ layers->items = tmp; ++ tmp = NULL; ++ layers->cap = new_size; ++ ++out: ++ layers->items[layers->len] = newh; ++ layers->len++; + return 0; + } + +-static int get_value_string(const char *content, const char *match, void *result) ++static void common_free_cgroup_layer(cgroup_layer_t *layers) + { +- *(char **)result = util_strdup_s(content); ++ size_t i; ++ ++ if (layers == NULL) { ++ return; ++ } ++ ++ for (i = 0; i < layers->len && layers->items[i]; i++) { ++ free(layers->items[i]->mountpoint); ++ layers->items[i]->mountpoint = NULL; ++ util_free_array(layers->items[i]->controllers); ++ layers->items[i]->controllers = NULL; ++ free(layers->items[i]); ++ layers->items[i] = NULL; ++ } ++ ++ free(layers->items); ++ layers->items = NULL; ++ layers->len = 0; ++ layers->cap = 0; ++ ++ free(layers); ++} ++ ++static int append_subsystem_to_list(char ***klist, char ***nlist, const char *ptoken) ++{ ++ int ret = 0; ++ ++ if (strncmp(ptoken, "name=", strlen("name=")) == 0) { ++ ret = util_array_append(nlist, ptoken); ++ if (ret != 0) { ++ ERROR("Failed to append string"); ++ return -1; ++ } ++ } else { ++ ret = util_array_append(klist, ptoken); ++ if (ret != 0) { ++ ERROR("Failed to append string"); ++ return -1; ++ } ++ } ++ + return 0; + } + +-static bool check_cgroup_v1_helper(const char *mountpoint, const cgroup_v1_files_index index, const bool quiet) ++static int get_cgroup_subsystems(char ***klist, char ***nlist) + { +- int nret = 0; +- char path[PATH_MAX] = { 0 }; ++ int ret = 0; ++ size_t length = 0; ++ FILE *fp = NULL; ++ char *pline = NULL; + +- if (index >= CGROUP_V1_FILES_INDEX_MAXS) { +- ERROR("Index out of range"); +- return false; ++ fp = util_fopen("/proc/self/cgroup", "r"); ++ if (fp == NULL) { ++ return -1; + } + +- if (mountpoint == NULL) { +- ERROR("%s: invalid arguments", g_cgroup_v1_files[index].name); +- return false; ++ while (getline(&pline, &length, fp) != -1) { ++ char *pos = NULL; ++ char *pos2 = NULL; ++ char *ptoken = NULL; ++ char *psave = NULL; ++ pos = strchr(pline, ':'); ++ if (pos == NULL) { ++ ERROR("Invalid cgroup entry: must contain at least two colons: %s", pline); ++ ret = -1; ++ goto out; ++ } ++ pos++; ++ pos2 = strchr(pos, ':'); ++ if (pos2 == NULL) { ++ ERROR("Invalid cgroup entry: must contain at least two colons: %s", pline); ++ ret = -1; ++ goto out; ++ } ++ *pos2 = '\0'; ++ ++ if ((pos2 - pos) == 0) { ++ INFO("Not supported cgroup entry: %s", pline); ++ continue; ++ } ++ ++ for (ptoken = strtok_r(pos, ",", &psave); ptoken; ptoken = strtok_r(NULL, ",", &psave)) { ++ if (append_subsystem_to_list(klist, nlist, ptoken)) { ++ goto out; ++ } ++ } + } + +- nret = snprintf(path, sizeof(path), "%s/%s", mountpoint, g_cgroup_v1_files[index].file); +- if (nret < 0 || (size_t)nret >= sizeof(path)) { +- ERROR("%s: failed to snprintf", g_cgroup_v1_files[index].name); +- return false; ++out: ++ free(pline); ++ fclose(fp); ++ if (ret != 0) { ++ util_free_array(*klist); ++ *klist = NULL; ++ util_free_array(*nlist); ++ *nlist = NULL; + } ++ return ret; ++} + +- if (util_file_exists(path)) { +- return true; ++static int append_controller(const char **klist, const char **nlist, char ***clist, const char *entry) ++{ ++ int ret = 0; ++ char *dup_entry = NULL; ++ ++ if (util_array_contain(klist, entry) && util_array_contain(nlist, entry)) { ++ ERROR("Refusing to use ambiguous controller \"%s\"", entry); ++ ERROR("It is both a named and kernel subsystem"); ++ return -1; + } + +- if (!quiet) { +- WARN("Your kernel does not support cgroup %s", g_cgroup_v1_files[index].name); ++ if (strncmp(entry, "name=", 5) == 0) { ++ dup_entry = util_strdup_s(entry); ++ } else if (util_array_contain(klist, entry)) { ++ dup_entry = util_strdup_s(entry); ++ } else { ++ dup_entry = util_string_append(entry, "name="); ++ } ++ if (dup_entry == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ ++ ret = util_array_append(clist, dup_entry); ++ if (ret != 0) { ++ ERROR("Failed to append array"); ++ } ++ ++ free(dup_entry); ++ return ret; ++} ++ ++static inline bool is_cgroup_mountpoint(const char *mp) ++{ ++ return strncmp(mp, CGROUP_MOUNT_PATH_PREFIX, strlen(CGROUP_MOUNT_PATH_PREFIX)) == 0; ++} ++ ++static char **cgroup_get_controllers(const char **klist, const char **nlist, const char *line) ++{ ++ int index; ++ char *dup = NULL; ++ char *pos2 = NULL; ++ char *tok = NULL; ++ const char *pos = line; ++ char *psave = NULL; ++ char *sep = ","; ++ char **pret = NULL; ++ ++ // line example ++ // 108 99 0:55 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime - tmpfs tmpfs rw,mode=755 ++ for (index = 0; index < 4; index++) { ++ pos = strchr(pos, ' '); ++ if (pos == NULL) { ++ ERROR("Invalid mountinfo format \"%s\"", line); ++ return NULL; ++ } ++ pos++; ++ } ++ ++ if (!is_cgroup_mountpoint(pos)) { ++ return NULL; ++ } ++ ++ pos += strlen(CGROUP_MOUNT_PATH_PREFIX); ++ pos2 = strchr(pos, ' '); ++ if (pos2 == NULL) { ++ ERROR("Invalid mountinfo format \"%s\"", line); ++ return NULL; ++ } ++ ++ *pos2 = '\0'; ++ dup = util_strdup_s(pos); ++ *pos2 = ' '; ++ ++ for (tok = strtok_r(dup, sep, &psave); tok; tok = strtok_r(NULL, sep, &psave)) { ++ if (append_controller(klist, nlist, &pret, tok)) { ++ ERROR("Failed to append controller"); ++ util_free_array(pret); ++ pret = NULL; ++ break; ++ } ++ } ++ ++ free(dup); ++ ++ return pret; ++} ++ ++static bool lists_intersect(const char **controllers, const char **list) ++{ ++ int index; ++ ++ if (controllers == NULL || list == NULL) { ++ return false; ++ } ++ ++ for (index = 0; controllers[index]; index++) { ++ if (util_array_contain(list, controllers[index])) { ++ return true; ++ } + } + + return false; + } + +-static int get_cgroup_v1_value_helper(const char *path, const cgroup_v1_files_index index, +- void *result) ++static bool controller_list_is_dup(const cgroup_layer_t *llist, const char **clist) + { +- int nret = 0; +- char file_path[PATH_MAX] = { 0 }; +- char real_path[PATH_MAX] = { 0 }; +- char *content = NULL; ++ size_t index; + +- if (index >= CGROUP_V1_FILES_INDEX_MAXS) { +- ERROR("Index out of range"); ++ if (llist == NULL) { + return false; + } + +- if (path == NULL || strlen(path) == 0 || result == NULL) { +- ERROR("%s: Invalid arguments", g_cgroup_v1_files[index].name); ++ for (index = 0; index < llist->len && llist->items[index]; index++) { ++ if (lists_intersect((const char **)llist->items[index]->controllers, (const char **)clist)) { ++ return true; ++ } ++ } ++ ++ return false; ++} ++ ++static int cgroup_get_mountpoint_and_root(char *pline, char **mountpoint, char **root) ++{ ++ int index; ++ char *posmp = NULL; ++ char *posrt = NULL; ++ char *pos = pline; ++ ++ // find root ++ // line example ++ // 108 99 0:55 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime - tmpfs tmpfs rw,mode=755 ++ for (index = 0; index < 3; index++) { ++ pos = strchr(pos, ' '); ++ if (pos == NULL) { ++ return -1; ++ } ++ pos++; ++ } ++ posrt = pos; ++ ++ // find mountpoint ++ pos = strchr(pos, ' '); ++ if (pos == NULL) { + return -1; + } + +- nret = snprintf(file_path, sizeof(file_path), "%s/%s", path, g_cgroup_v1_files[index].file); +- if (nret < 0 || (size_t)nret >= sizeof(file_path)) { +- ERROR("%s: failed to snprintf", g_cgroup_v1_files[index].name); ++ *pos = '\0'; ++ if (root != NULL) { ++ *root = util_strdup_s(posrt); ++ } ++ ++ pos++; ++ posmp = pos; ++ ++ if (!is_cgroup_mountpoint(posmp)) { + return -1; + } + +- if (util_clean_path(file_path, real_path, sizeof(real_path)) == NULL) { +- ERROR("%s: failed to clean path %s", g_cgroup_v1_files[index].name, file_path); ++ pos = strchr(pos + strlen(CGROUP_MOUNT_PATH_PREFIX), ' '); ++ if (pos == NULL) { + return -1; + } ++ *pos = '\0'; ++ ++ if (mountpoint != NULL) { ++ *mountpoint = util_strdup_s(posmp); ++ } ++ ++ return 0; ++} ++ ++/* find cgroup mountpoint and root */ ++static int get_cgroup_mnt_and_root_path_v1(const char *subsystem, char **mountpoint, char **root) ++{ ++ int ret = 0; ++ FILE *fp = NULL; ++ size_t length = 0; ++ char *pline = NULL; + +- content = util_read_content_from_file(real_path); +- if (content == NULL) { +- ERROR("%s: failed to read file %s", g_cgroup_v1_files[index].name, real_path); ++ if (subsystem == NULL) { ++ ERROR("Empty subsystem"); + return -1; + } + +- util_trim_newline(content); +- content = util_trim_space(content); ++ fp = util_fopen("/proc/self/mountinfo", "r"); ++ if (fp == NULL) { ++ ERROR("Failed to open \"/proc/self/mountinfo\"\n"); ++ ret = -1; ++ goto free_out; ++ } ++ ++ while (getline(&pline, &length, fp) != -1) { ++ char *dup = NULL; ++ char *p = NULL; ++ char *tok = NULL; ++ char *mp = NULL; ++ char *rt = NULL; ++ char *saveptr = NULL; ++ char *sep = ","; ++ int mret; ++ ++ mret = cgroup_get_mountpoint_and_root(pline, &mp, &rt); ++ if (mret != 0 || mp == NULL || rt == NULL) { ++ goto mp_out; ++ } ++ ++ p = mp; ++ p += strlen(CGROUP_MOUNT_PATH_PREFIX); ++ dup = util_strdup_s(p); ++ if (dup == NULL) { ++ ERROR("Out of memory"); ++ free(mp); ++ ret = -1; ++ goto free_out; ++ } ++ ++ for (tok = strtok_r(dup, sep, &saveptr); tok; tok = strtok_r(NULL, sep, &saveptr)) { ++ if (strcmp(tok, subsystem) != 0) { ++ continue; ++ } ++ if (mountpoint != NULL) { ++ *mountpoint = mp; ++ } else { ++ free(mp); ++ } ++ if (root != NULL) { ++ *root = rt; ++ } else { ++ free(rt); ++ } ++ free(dup); ++ goto free_out; ++ } ++ free(dup); ++mp_out: ++ free(mp); ++ free(rt); ++ continue; ++ } ++free_out: ++ if (fp != NULL) { ++ fclose(fp); ++ } ++ free(pline); ++ return ret; ++} ++ ++static cgroup_layer_t *common_cgroup_layers_find(void) ++{ ++ int nret; ++ int ret = 0; ++ FILE *fp = NULL; ++ size_t length = 0; ++ const size_t cgroup_layer_item_num = 10; ++ char *pline = NULL; ++ char **klist = NULL; ++ char **nlist = NULL; ++ cgroup_layer_t *layers = NULL; ++ ++ layers = new_cgroup_layer(cgroup_layer_item_num); ++ if (layers == NULL) { ++ ERROR("Failed to new cgroup layer"); ++ return NULL; ++ } ++ ++ ret = get_cgroup_subsystems(&klist, &nlist); ++ if (ret != 0) { ++ ERROR("Failed to retrieve available legacy cgroup controllers\n"); ++ goto out; ++ } ++ ++ fp = util_fopen("/proc/self/mountinfo", "r"); ++ if (fp == NULL) { ++ ERROR("Failed to open \"/proc/self/mountinfo\"\n"); ++ ret = -1; ++ goto out; ++ } ++ ++ while (getline(&pline, &length, fp) != -1) { ++ char *mountpoint = NULL; ++ char **clist = NULL; ++ int mret; ++ ++ clist = cgroup_get_controllers((const char **)klist, (const char **)nlist, pline); ++ if (clist == NULL) { ++ goto list_out; ++ } ++ ++ if (controller_list_is_dup(layers, (const char **)clist)) { ++ goto list_out; ++ } ++ ++ mret = cgroup_get_mountpoint_and_root(pline, &mountpoint, NULL); ++ if (mret != 0 || mountpoint == NULL) { ++ ERROR("Failed parsing mountpoint from \"%s\"\n", pline); ++ goto list_out; ++ } ++ ++ nret = add_cgroup_layer(layers, clist, mountpoint); ++ if (nret != 0) { ++ ERROR("Failed to add hierarchies"); ++ goto list_out; ++ } ++ ++ continue; ++list_out: ++ util_free_array(clist); ++ free(mountpoint); ++ } ++out: ++ util_free_array(klist); ++ util_free_array(nlist); ++ if (fp != NULL) { ++ fclose(fp); ++ } ++ free(pline); ++ ++ if (ret != 0) { ++ common_free_cgroup_layer(layers); ++ return NULL; ++ } ++ ++ return layers; ++} ++ ++static int get_cgroup_v1_value_helper(const char *path, const cgroup_v1_files_index index, void *result) ++{ ++ if (index >= CGROUP_V1_FILES_INDEX_MAXS) { ++ ERROR("Index out of range"); ++ return false; ++ } ++ ++ return get_cgroup_value_helper(path, &g_cgroup_v1_files[index], result); ++} ++ ++static bool check_cgroup_v1_file_exists(const char *mountpoint, const cgroup_v1_files_index index, const bool quiet) ++{ ++ int nret = 0; ++ char path[PATH_MAX] = { 0 }; ++ ++ if (index >= CGROUP_V1_FILES_INDEX_MAXS) { ++ ERROR("Index out of range"); ++ return false; ++ } ++ ++ if (mountpoint == NULL) { ++ ERROR("%s: invalid arguments", g_cgroup_v1_files[index].name); ++ return false; ++ } ++ ++ nret = snprintf(path, sizeof(path), "%s/%s", mountpoint, g_cgroup_v1_files[index].file); ++ if (nret < 0 || (size_t)nret >= sizeof(path)) { ++ ERROR("%s: failed to snprintf", g_cgroup_v1_files[index].name); ++ return false; ++ } + +- nret = g_cgroup_v1_files[index].get_value(content, g_cgroup_v1_files[index].match, result); +- if (nret != 0) { +- ERROR("%s: failed to get value", g_cgroup_v1_files[index].name); ++ if (util_file_exists(path)) { ++ return true; + } + +- free(content); +- return nret; ++ if (!quiet) { ++ WARN("Your kernel does not support cgroup %s", g_cgroup_v1_files[index].name); ++ } ++ ++ return false; + } + +-static void check_cgroup_v1_cpu(const cgroup_layer_t *layers, const bool quiet, cgroup_cpu_info_t *cpuinfo) ++static void get_cgroup_v1_cpu_info(const cgroup_layer_t *layers, const bool quiet, cgroup_cpu_info_t *cpuinfo) + { + char *mountpoint = NULL; + +@@ -198,14 +690,14 @@ static void check_cgroup_v1_cpu(const cgroup_layer_t *layers, const bool quiet, + return; + } + +- cpuinfo->cpu_rt_period = check_cgroup_v1_helper(mountpoint, CPU_RT_PERIOD, quiet); +- cpuinfo->cpu_rt_runtime = check_cgroup_v1_helper(mountpoint, CPU_RT_RUNTIME, quiet); +- cpuinfo->cpu_shares = check_cgroup_v1_helper(mountpoint, CPU_SHARES, quiet); +- cpuinfo->cpu_cfs_period = check_cgroup_v1_helper(mountpoint, CPU_CFS_PERIOD, quiet); +- cpuinfo->cpu_cfs_quota = check_cgroup_v1_helper(mountpoint, CPU_CFS_QUOTA, quiet); ++ cpuinfo->cpu_rt_period = check_cgroup_v1_file_exists(mountpoint, CPU_RT_PERIOD, quiet); ++ cpuinfo->cpu_rt_runtime = check_cgroup_v1_file_exists(mountpoint, CPU_RT_RUNTIME, quiet); ++ cpuinfo->cpu_shares = check_cgroup_v1_file_exists(mountpoint, CPU_SHARES, quiet); ++ cpuinfo->cpu_cfs_period = check_cgroup_v1_file_exists(mountpoint, CPU_CFS_PERIOD, quiet); ++ cpuinfo->cpu_cfs_quota = check_cgroup_v1_file_exists(mountpoint, CPU_CFS_QUOTA, quiet); + } + +-static void check_cgroup_v1_cpuset(const cgroup_layer_t *layers, const bool quiet, cgroup_cpuset_info_t *cpusetinfo) ++static void get_cgroup_v1_cpuset_info(const cgroup_layer_t *layers, const bool quiet, cgroup_cpuset_info_t *cpusetinfo) + { + char *mountpoint = NULL; + +@@ -215,10 +707,10 @@ static void check_cgroup_v1_cpuset(const cgroup_layer_t *layers, const bool quie + return; + } + +- if (!check_cgroup_v1_helper(mountpoint, CPUSET_CPUS, quiet)) { ++ if (!check_cgroup_v1_file_exists(mountpoint, CPUSET_CPUS, quiet)) { + return; + } +- if (!check_cgroup_v1_helper(mountpoint, CPUSET_MEMS, quiet)) { ++ if (!check_cgroup_v1_file_exists(mountpoint, CPUSET_MEMS, quiet)) { + return; + } + +@@ -238,7 +730,7 @@ static void check_cgroup_v1_cpuset(const cgroup_layer_t *layers, const bool quie + cpusetinfo->cpuset = true; + } + +-static void check_cgroup_v1_mem(const cgroup_layer_t *layers, const bool quiet, cgroup_mem_info_t *meminfo) ++static void get_cgroup_v1_mem_info(const cgroup_layer_t *layers, const bool quiet, cgroup_mem_info_t *meminfo) + { + char *mountpoint = NULL; + +@@ -248,15 +740,15 @@ static void check_cgroup_v1_mem(const cgroup_layer_t *layers, const bool quiet, + return; + } + +- meminfo->limit = check_cgroup_v1_helper(mountpoint, MEMORY_LIMIT, quiet); +- meminfo->swap = check_cgroup_v1_helper(mountpoint, MEMORY_SW_LIMIT, quiet); +- meminfo->reservation = check_cgroup_v1_helper(mountpoint, MEMORY_SOFT_LIMIT, quiet); +- meminfo->oomkilldisable = check_cgroup_v1_helper(mountpoint, MEMORY_OOM_CONTROL, quiet); +- meminfo->swappiness = check_cgroup_v1_helper(mountpoint, MEMORY_SWAPPINESS, quiet); +- meminfo->kernel = check_cgroup_v1_helper(mountpoint, MEMORY_KMEM_LIMIT, quiet); ++ meminfo->limit = check_cgroup_v1_file_exists(mountpoint, MEMORY_LIMIT, quiet); ++ meminfo->swap = check_cgroup_v1_file_exists(mountpoint, MEMORY_SW_LIMIT, quiet); ++ meminfo->reservation = check_cgroup_v1_file_exists(mountpoint, MEMORY_SOFT_LIMIT, quiet); ++ meminfo->oomkilldisable = check_cgroup_v1_file_exists(mountpoint, MEMORY_OOM_CONTROL, quiet); ++ meminfo->swappiness = check_cgroup_v1_file_exists(mountpoint, MEMORY_SWAPPINESS, quiet); ++ meminfo->kernel = check_cgroup_v1_file_exists(mountpoint, MEMORY_KMEM_LIMIT, quiet); + } + +-static void check_cgroup_v1_blkio(const cgroup_layer_t *layers, const bool quiet, cgroup_blkio_info_t *blkioinfo) ++static void get_cgroup_v1_blkio_info(const cgroup_layer_t *layers, const bool quiet, cgroup_blkio_info_t *blkioinfo) + { + char *mountpoint = NULL; + +@@ -266,15 +758,15 @@ static void check_cgroup_v1_blkio(const cgroup_layer_t *layers, const bool quiet + return; + } + +- blkioinfo->blkio_weight = check_cgroup_v1_helper(mountpoint, BLKIO_WEIGTH, quiet); +- blkioinfo->blkio_weight_device = check_cgroup_v1_helper(mountpoint, BLKIO_WEIGTH_DEVICE, quiet); +- blkioinfo->blkio_read_bps_device = check_cgroup_v1_helper(mountpoint, BLKIO_READ_BPS, quiet); +- blkioinfo->blkio_write_bps_device = check_cgroup_v1_helper(mountpoint, BLKIO_WRITE_BPS, quiet); +- blkioinfo->blkio_read_iops_device = check_cgroup_v1_helper(mountpoint, BLKIO_READ_IOPS, quiet); +- blkioinfo->blkio_write_iops_device = check_cgroup_v1_helper(mountpoint, BLKIO_WRITE_IOPS, quiet); ++ blkioinfo->blkio_weight = check_cgroup_v1_file_exists(mountpoint, BLKIO_WEIGTH, quiet); ++ blkioinfo->blkio_weight_device = check_cgroup_v1_file_exists(mountpoint, BLKIO_WEIGTH_DEVICE, quiet); ++ blkioinfo->blkio_read_bps_device = check_cgroup_v1_file_exists(mountpoint, BLKIO_READ_BPS, quiet); ++ blkioinfo->blkio_write_bps_device = check_cgroup_v1_file_exists(mountpoint, BLKIO_WRITE_BPS, quiet); ++ blkioinfo->blkio_read_iops_device = check_cgroup_v1_file_exists(mountpoint, BLKIO_READ_IOPS, quiet); ++ blkioinfo->blkio_write_iops_device = check_cgroup_v1_file_exists(mountpoint, BLKIO_WRITE_IOPS, quiet); + } + +-static void check_cgroup_v1_hugetlb(const cgroup_layer_t *layers, const bool quiet, cgroup_hugetlb_info_t *hugetlbinfo) ++static void get_cgroup_v1_hugetlb_info(const cgroup_layer_t *layers, const bool quiet, cgroup_hugetlb_info_t *hugetlbinfo) + { + int nret; + char *mountpoint = NULL; +@@ -306,7 +798,7 @@ free_out: + free(defaultpagesize); + } + +-static void check_cgroup_v1_pids(const cgroup_layer_t *layers, const bool quiet, cgroup_pids_info_t *pidsinfo) ++static void get_cgroup_v1_pids_info(const cgroup_layer_t *layers, const bool quiet, cgroup_pids_info_t *pidsinfo) + { + char *mountpoint = NULL; + +@@ -319,7 +811,7 @@ static void check_cgroup_v1_pids(const cgroup_layer_t *layers, const bool quiet, + pidsinfo->pidslimit = true; + } + +-static void check_cgroup_v1_files(const cgroup_layer_t *layers, const bool quiet, cgroup_files_info_t *filesinfo) ++static void get_cgroup_v1_files_info(const cgroup_layer_t *layers, const bool quiet, cgroup_files_info_t *filesinfo) + { + char *mountpoint = NULL; + +@@ -332,7 +824,7 @@ static void check_cgroup_v1_files(const cgroup_layer_t *layers, const bool quiet + filesinfo->fileslimit = true; + } + +-int common_get_cgroup_info_v1(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, ++static int get_cgroup_info_v1(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, + cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, + cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, + cgroup_files_info_t *filesinfo, bool quiet) +@@ -345,13 +837,13 @@ int common_get_cgroup_info_v1(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpu + return -1; + } + +- check_cgroup_v1_cpu(layers, quiet, cpuinfo); +- check_cgroup_v1_cpuset(layers, quiet, cpusetinfo); +- check_cgroup_v1_mem(layers, quiet, meminfo); +- check_cgroup_v1_blkio(layers, quiet, blkioinfo); +- check_cgroup_v1_hugetlb(layers, quiet, hugetlbinfo); +- check_cgroup_v1_pids(layers, quiet, pidsinfo); +- check_cgroup_v1_files(layers, quiet, filesinfo); ++ get_cgroup_v1_cpu_info(layers, quiet, cpuinfo); ++ get_cgroup_v1_cpuset_info(layers, quiet, cpusetinfo); ++ get_cgroup_v1_mem_info(layers, quiet, meminfo); ++ get_cgroup_v1_blkio_info(layers, quiet, blkioinfo); ++ get_cgroup_v1_hugetlb_info(layers, quiet, hugetlbinfo); ++ get_cgroup_v1_pids_info(layers, quiet, pidsinfo); ++ get_cgroup_v1_files_info(layers, quiet, filesinfo); + + common_free_cgroup_layer(layers); + +@@ -432,7 +924,7 @@ static void get_cgroup_v1_metrics_pid(const cgroup_layer_t *layers, const char * + get_cgroup_v1_value_helper(path, PIDS_CURRENT, (void *)&cgroup_pids_metrics->pid_current); + } + +-int common_get_cgroup_v1_metrics(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics) ++static int get_cgroup_metrics_v1(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics) + { + cgroup_layer_t *layers = NULL; + +@@ -553,22 +1045,31 @@ static char *common_get_cgroup_path(const char *path, const char *subsystem) + return res; + } + +-char *common_get_init_cgroup(const char *subsystem) ++char *get_init_cgroup_path_v1(const char *subsystem) + { +- if (common_get_cgroup_version() != CGROUP_VERSION_1) { +- ERROR("Not implemented for cgroup v2 hierarchy"); +- return NULL; +- } +- + return common_get_cgroup_path("/proc/1/cgroup", subsystem); + } + +-char *common_get_own_cgroup(const char *subsystem) ++char *get_own_cgroup_v1(const char *subsystem) + { +- if (common_get_cgroup_version() != CGROUP_VERSION_1) { +- ERROR("Not implemented for cgroup v2 hierarchy"); +- return NULL; +- } +- + return common_get_cgroup_path("/proc/self/cgroup", subsystem); + } ++ ++int get_cgroup_version_v1() ++{ ++ return CGROUP_VERSION_1; ++} ++ ++int cgroup_v1_ops_init(cgroup_ops *ops) ++{ ++ if (ops == NULL) { ++ return -1; ++ } ++ ops->get_cgroup_version = get_cgroup_version_v1; ++ ops->get_cgroup_info = get_cgroup_info_v1; ++ ops->get_cgroup_metrics = get_cgroup_metrics_v1; ++ ops->get_cgroup_mnt_and_root_path = get_cgroup_mnt_and_root_path_v1; ++ ops->get_init_cgroup_path = get_init_cgroup_path_v1; ++ ops->get_own_cgroup_path = get_own_cgroup_v1; ++ return 0; ++} +\ No newline at end of file +diff --git a/src/daemon/common/cgroup/cgroup_v1.h b/src/daemon/common/cgroup/cgroup_v1.h +new file mode 100644 +index 00000000..c2696725 +--- /dev/null ++++ b/src/daemon/common/cgroup/cgroup_v1.h +@@ -0,0 +1,30 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-02-19 ++ * Description: provide cgroup v1 definition ++ ******************************************************************************/ ++#ifndef DAEMON_COMMON_CGROUP_CGROUP_V1_H ++#define DAEMON_COMMON_CGROUP_CGROUP_V1_H ++ ++#include "cgroup.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++int cgroup_v1_ops_init(cgroup_ops *ops); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif // DAEMON_COMMON_CGROUP_CGROUP_V1_H +diff --git a/src/daemon/common/cgroup_v2.c b/src/daemon/common/cgroup/cgroup_v2.c +similarity index 88% +rename from src/daemon/common/cgroup_v2.c +rename to src/daemon/common/cgroup/cgroup_v2.c +index 25509bda..65cf90d8 100644 +--- a/src/daemon/common/cgroup_v2.c ++++ b/src/daemon/common/cgroup/cgroup_v2.c +@@ -105,48 +105,12 @@ static struct cgfile_t g_cgroup_v2_files[] = { + + static int get_cgroup_v2_value_helper(const char *path, const cgroup_v2_files_index index, void *result) + { +- int nret = 0; +- char file_path[PATH_MAX] = { 0 }; +- char real_path[PATH_MAX] = { 0 }; +- char *content = NULL; +- + if (index >= CGROUP_V2_FILES_INDEX_MAXS) { + ERROR("Index out of range"); + return false; + } + +- if (path == NULL || strlen(path) == 0 || result == NULL) { +- ERROR("%s: Invalid arguments", g_cgroup_v2_files[index].name); +- return -1; +- } +- +- nret = snprintf(file_path, sizeof(file_path), "%s/%s", path, g_cgroup_v2_files[index].file); +- if (nret < 0 || (size_t)nret >= sizeof(file_path)) { +- ERROR("%s: failed to snprintf", g_cgroup_v2_files[index].name); +- return -1; +- } +- +- if (util_clean_path(file_path, real_path, sizeof(real_path)) == NULL) { +- ERROR("%s: failed to clean path %s", g_cgroup_v2_files[index].name, file_path); +- return -1; +- } +- +- content = util_read_content_from_file(real_path); +- if (content == NULL) { +- ERROR("%s: failed to read file %s", g_cgroup_v2_files[index].name, real_path); +- return -1; +- } +- +- util_trim_newline(content); +- content = util_trim_space(content); +- +- nret = g_cgroup_v2_files[index].get_value(content, g_cgroup_v2_files[index].match, result); +- if (nret != 0) { +- ERROR("%s: failed to get value", g_cgroup_v2_files[index].name); +- } +- +- free(content); +- return nret; ++ return get_cgroup_value_helper(path, &g_cgroup_v2_files[index], result); + } + + static void get_cgroup_v2_metrics_cpu(const char *cgroup_path, cgroup_cpu_metrics_t *cgroup_cpu_metrics) +@@ -306,28 +270,18 @@ static bool cgroup_v2_enabled(const char *mountpoint, const char *name) + return util_file_exists(path); + } + +-int common_get_cgroup_info_v2(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, +- cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, +- cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, +- cgroup_files_info_t *filesinfo, bool quiet) ++static void get_cgroup_v2_cpu_info(const bool quiet, cgroup_cpu_info_t *cpuinfo) + { +- int ret = 0; +- int nret = 0; +- __isula_auto_free char *size = NULL; +- char path[PATH_MAX] = { 0 }; +- +- if (make_sure_cgroup2_isulad_path_exist() != 0) { +- return -1; +- } +- +- // cpu cgroup + cpuinfo->cpu_shares = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPU_WEIGHT); + common_cgroup_do_log(quiet, !(cpuinfo->cpu_shares), "Your kernel does not support cgroup2 cpu weight"); + + cpuinfo->cpu_cfs_period = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPU_MAX); + cpuinfo->cpu_cfs_quota = cpuinfo->cpu_cfs_period; + common_cgroup_do_log(quiet, !(cpuinfo->cpu_cfs_period), "Your kernel does not support cgroup2 cpu max"); ++} + ++static int get_cgroup_v2_cpuset_info(const bool quiet, cgroup_cpuset_info_t *cpusetinfo) ++{ + cpusetinfo->cpuset = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPUSET_CPUS_EFFECTIVE) && + cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPUSET_CPUS) && + cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPUSET_MEMS_EFFECTIVE) && +@@ -343,8 +297,23 @@ int common_get_cgroup_info_v2(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpu + cpusetinfo->cpus = util_trim_space(cpusetinfo->cpus); + cpusetinfo->mems = util_trim_space(cpusetinfo->mems); + } ++ return 0; ++} ++ ++static void get_cgroup_v2_mem_info(const bool quiet, cgroup_mem_info_t *meminfo) ++{ ++ meminfo->limit = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_MEMORY_MAX); ++ common_cgroup_do_log(quiet, !(meminfo->limit), "Your kernel does not support cgroup2 memory max"); ++ ++ meminfo->reservation = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_MEMORY_LOW); ++ common_cgroup_do_log(quiet, !(meminfo->reservation), "Your kernel does not support cgroup2 memory low"); + +- // io cgroup ++ meminfo->swap = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_MEMORY_SWAP_MAX); ++ common_cgroup_do_log(quiet, !(meminfo->swap), "Your kernel does not support cgroup2 memory swap max"); ++} ++ ++static void get_cgroup_v2_blkio_info(const bool quiet, cgroup_blkio_info_t *blkioinfo) ++{ + blkioinfo->blkio_weight = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_IO_BFQ_WEIGHT) || + cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_IO_WEIGHT); + blkioinfo->blkio_weight_device = blkioinfo->blkio_weight; +@@ -355,22 +324,14 @@ int common_get_cgroup_info_v2(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpu + blkioinfo->blkio_read_iops_device = blkioinfo->blkio_read_bps_device; + blkioinfo->blkio_write_iops_device = blkioinfo->blkio_read_bps_device; + common_cgroup_do_log(quiet, !(blkioinfo->blkio_read_bps_device), "Your kernel does not support cgroup2 io max"); ++} + +- // memory cgroup +- meminfo->limit = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_MEMORY_MAX); +- common_cgroup_do_log(quiet, !(meminfo->limit), "Your kernel does not support cgroup2 memory max"); +- +- meminfo->reservation = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_MEMORY_LOW); +- common_cgroup_do_log(quiet, !(meminfo->reservation), "Your kernel does not support cgroup2 memory low"); +- +- meminfo->swap = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_MEMORY_SWAP_MAX); +- common_cgroup_do_log(quiet, !(meminfo->swap), "Your kernel does not support cgroup2 memory swap max"); +- +- // pids cgroup +- pidsinfo->pidslimit = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_PIDS_MAX); +- common_cgroup_do_log(quiet, !(pidsinfo->pidslimit), "Your kernel does not support cgroup2 pids max"); ++static int get_cgroup_v2_hugetlb_info(const bool quiet, cgroup_hugetlb_info_t *hugetlbinfo) ++{ ++ __isula_auto_free char *size = NULL; ++ int nret = 0; ++ char path[PATH_MAX] = { 0 }; + +- // hugetlb cgroup + size = get_default_huge_page_size(); + if (size != NULL) { + nret = snprintf(path, sizeof(path), CGROUP2_HUGETLB_MAX, size); +@@ -383,15 +344,55 @@ int common_get_cgroup_info_v2(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpu + } else { + WARN("Your kernel does not support cgroup2 hugetlb limit"); + } ++ return 0; ++} ++ ++static void get_cgroup_v2_pids_info(const bool quiet, cgroup_pids_info_t *pidsinfo) ++{ ++ pidsinfo->pidslimit = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_PIDS_MAX); ++ common_cgroup_do_log(quiet, !(pidsinfo->pidslimit), "Your kernel does not support cgroup2 pids max"); ++ ++} + +- // files cgroup ++static void get_cgroup_v2_files_info(const bool quiet, cgroup_files_info_t *filesinfo) ++{ + filesinfo->fileslimit = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_FILES_LIMIT); + common_cgroup_do_log(quiet, !(filesinfo->fileslimit), "Your kernel does not support cgroup2 files limit"); + +- return ret; + } + +-int common_get_cgroup_v2_metrics(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics) ++static int get_cgroup_info_v2(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, ++ cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, ++ cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, ++ cgroup_files_info_t *filesinfo, bool quiet) ++{ ++ int ret = 0; ++ if (make_sure_cgroup2_isulad_path_exist() != 0) { ++ return -1; ++ } ++ ++ get_cgroup_v2_cpu_info(quiet, cpuinfo); ++ ++ ret = get_cgroup_v2_cpuset_info(quiet, cpusetinfo); ++ if (ret != 0) { ++ return ret; ++ } ++ ++ get_cgroup_v2_mem_info(quiet, meminfo); ++ get_cgroup_v2_blkio_info(quiet, blkioinfo); ++ ++ ret = get_cgroup_v2_hugetlb_info(quiet, hugetlbinfo); ++ if (ret != 0) { ++ return ret; ++ } ++ ++ get_cgroup_v2_pids_info(quiet, pidsinfo); ++ get_cgroup_v2_files_info(quiet, filesinfo); ++ ++ return 0; ++} ++ ++static int get_cgroup_metrics_v2(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics) + { + if (cgroup_path == NULL || strlen(cgroup_path) == 0 || cgroup_metrics == NULL) { + ERROR("Invalid arguments"); +@@ -404,3 +405,26 @@ int common_get_cgroup_v2_metrics(const char *cgroup_path, cgroup_metrics_t *cgro + + return 0; + } ++ ++static int get_cgroup_mnt_and_root_v2(const char *subsystem, char **mountpoint, char **root) ++{ ++ *mountpoint = util_strdup_s(CGROUP_ISULAD_PATH); ++ return 0; ++} ++ ++int get_cgroup_version_v2() ++{ ++ return CGROUP_VERSION_2; ++} ++ ++int cgroup_v2_ops_init(cgroup_ops *ops) ++{ ++ if (ops == NULL) { ++ return -1; ++ } ++ ops->get_cgroup_version = get_cgroup_version_v2; ++ ops->get_cgroup_info = get_cgroup_info_v2; ++ ops->get_cgroup_metrics = get_cgroup_metrics_v2; ++ ops->get_cgroup_mnt_and_root_path = get_cgroup_mnt_and_root_v2; ++ return 0; ++} +\ No newline at end of file +diff --git a/src/daemon/common/cgroup/cgroup_v2.h b/src/daemon/common/cgroup/cgroup_v2.h +new file mode 100644 +index 00000000..0e8e4818 +--- /dev/null ++++ b/src/daemon/common/cgroup/cgroup_v2.h +@@ -0,0 +1,30 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-02-19 ++ * Description: provide cgroup v2 definition ++ ******************************************************************************/ ++#ifndef DAEMON_COMMON_CGROUP_CGROUP_V2_H ++#define DAEMON_COMMON_CGROUP_CGROUP_V2_H ++ ++#include "cgroup.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++int cgroup_v2_ops_init(cgroup_ops *ops); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif // DAEMON_COMMON_CGROUP_CGROUP_V2_H +diff --git a/src/daemon/common/sysinfo.c b/src/daemon/common/sysinfo.c +index a082f717..e369c3e3 100644 +--- a/src/daemon/common/sysinfo.c ++++ b/src/daemon/common/sysinfo.c +@@ -188,15 +188,10 @@ static char **get_huge_page_sizes() + struct dirent *info_archivo = NULL; + int cgroup_version = 0; + +- cgroup_version = common_get_cgroup_version(); +- if (cgroup_version == CGROUP_VERSION_2) { +- hugetlbmp = util_strdup_s(CGROUP_ISULAD_PATH); +- } else { +- ret = common_find_cgroup_mnt_and_root("hugetlb", &hugetlbmp, NULL); +- if (ret != 0 || hugetlbmp == NULL) { +- ERROR("Hugetlb cgroup not supported"); +- return NULL; +- } ++ ret = common_get_cgroup_mnt_and_root_path("hugetlb", &hugetlbmp, NULL); ++ if (ret != 0 || hugetlbmp == NULL) { ++ ERROR("Hugetlb cgroup not supported"); ++ return NULL; + } + + dir = opendir(hugetlbmp); +@@ -211,6 +206,7 @@ static char **get_huge_page_sizes() + char *pos = NULL; + char *dot2 = NULL; + ++ cgroup_version = common_get_cgroup_version(); + if (cgroup_version == CGROUP_VERSION_2) { + if (!is_hugetlb_max(info_archivo->d_name)) { + continue; +@@ -375,7 +371,6 @@ void free_sysinfo(sysinfo_t *sysinfo) + /* get sys info */ + sysinfo_t *get_sys_info(bool quiet) + { +- int cgroup_version = 0; + sysinfo_t *sysinfo = NULL; + int ret = 0; + +@@ -388,21 +383,9 @@ sysinfo_t *get_sys_info(bool quiet) + sysinfo->ncpus = get_nprocs(); + sysinfo->ncpus_conf = get_nprocs_conf(); + +- cgroup_version = common_get_cgroup_version(); +- if (cgroup_version < 0) { +- ret = -1; +- goto out; +- } +- +- if (cgroup_version == CGROUP_VERSION_1) { +- ret = common_get_cgroup_info_v1(&sysinfo->cgmeminfo, &sysinfo->cgcpuinfo, &sysinfo->hugetlbinfo, ++ ret = common_get_cgroup_info(&sysinfo->cgmeminfo, &sysinfo->cgcpuinfo, &sysinfo->hugetlbinfo, + &sysinfo->blkioinfo, &sysinfo->cpusetinfo, &sysinfo->pidsinfo, + &sysinfo->filesinfo, quiet); +- } else { +- ret = common_get_cgroup_info_v2(&sysinfo->cgmeminfo, &sysinfo->cgcpuinfo, &sysinfo->hugetlbinfo, +- &sysinfo->blkioinfo, &sysinfo->cpusetinfo, &sysinfo->pidsinfo, +- &sysinfo->filesinfo, quiet); +- } + if (ret != 0) { + goto out; + } +@@ -563,7 +546,7 @@ free_out: + return minfos; + } + +-char *sysinfo_cgroup_controller_cpurt_mnt_path(void) ++char *sysinfo_get_cpurt_mnt_path(void) + { + int nret = 0; + __isula_auto_free char *mnt = NULL; +@@ -583,7 +566,7 @@ char *sysinfo_cgroup_controller_cpurt_mnt_path(void) + return NULL; + } + +- nret = common_find_cgroup_mnt_and_root("cpu", &mnt, &root); ++ nret = common_get_cgroup_mnt_and_root_path("cpu", &mnt, &root); + if (nret != 0 || mnt == NULL || root == NULL) { + ERROR("Can not find cgroup mnt and root path for subsystem 'cpu'"); + isulad_set_error_message("Can not find cgroup mnt and root path for subsystem 'cpu'"); +diff --git a/src/daemon/common/sysinfo.h b/src/daemon/common/sysinfo.h +index cb44d1c5..6142487b 100644 +--- a/src/daemon/common/sysinfo.h ++++ b/src/daemon/common/sysinfo.h +@@ -95,13 +95,13 @@ mountinfo_t *find_mount_info(mountinfo_t **minfos, const char *dir); + + void free_mounts_info(mountinfo_t **minfos); + +-char *sysinfo_cgroup_controller_cpurt_mnt_path(void); +- + // define auto free function callback for sysinfo_t + define_auto_cleanup_callback(free_sysinfo, sysinfo_t) + // define auto free macro for sysinfo_t + #define __isula_auto_sysinfo_t auto_cleanup_tag(free_sysinfo) + ++char *sysinfo_get_cpurt_mnt_path(void); ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +index 3bdc3af8..f125e714 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +@@ -872,18 +872,7 @@ void PodSandboxManagerService::GetPodSandboxCgroupMetrics(const std::string &cgr + return; + } + +- auto cgroupVersion = common_get_cgroup_version(); +- if (cgroupVersion < 0) { +- error.Errorf("Invalid cgroup version"); +- return; +- } +- +- if (cgroupVersion == CGROUP_VERSION_1) { +- nret = common_get_cgroup_v1_metrics(cgroupParent.c_str(), &cgroupMetrics); +- } else { +- nret = common_get_cgroup_v2_metrics(cgroupParent.c_str(), &cgroupMetrics); +- } +- ++ nret = common_get_cgroup_metrics(cgroupParent.c_str(), &cgroupMetrics); + if (nret != 0) { + error.Errorf("Failed to get cgroup metrics"); + } +diff --git a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +index 49a7ca54..af6b5fff 100644 +--- a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +@@ -1328,18 +1328,7 @@ void PodSandboxManagerService::GetPodSandboxCgroupMetrics(const container_inspec + return; + } + +- auto cgroupVersion = common_get_cgroup_version(); +- if (cgroupVersion < 0) { +- error.Errorf("Invalid cgroup version"); +- return; +- } +- +- if (cgroupVersion == CGROUP_VERSION_1) { +- nret = common_get_cgroup_v1_metrics(cgroupParent, &cgroupMetrics); +- } else { +- nret = common_get_cgroup_v2_metrics(cgroupParent, &cgroupMetrics); +- } +- ++ nret = common_get_cgroup_metrics(cgroupParent, &cgroupMetrics); + if (nret != 0) { + error.Errorf("Failed to get cgroup metrics"); + } +diff --git a/src/daemon/executor/container_cb/execution.c b/src/daemon/executor/container_cb/execution.c +index 7ed8e837..88c6b354 100644 +--- a/src/daemon/executor/container_cb/execution.c ++++ b/src/daemon/executor/container_cb/execution.c +@@ -413,6 +413,13 @@ static int cpurt_controller_init(const char *id, const host_config *host_spec) + char *dirpath = NULL; + int64_t cpu_rt_period = 0; + int64_t cpu_rt_runtime = 0; ++ int cgroup_version = 0; ++ ++ // cgroup v2 is not support cpurt ++ cgroup_version = common_get_cgroup_version(); ++ if (cgroup_version == CGROUP_VERSION_2) { ++ return 0; ++ } + + cgroups_path = merge_container_cgroups_path(id, host_spec); + if (cgroups_path == NULL || strcmp(cgroups_path, "/") == 0 || strcmp(cgroups_path, ".") == 0) { +@@ -433,13 +440,13 @@ static int cpurt_controller_init(const char *id, const host_config *host_spec) + // should iSulad set cpu.rt_runtime_us and cpu.rt_period_us for the parent path? + // in fact, even if system.slice is used, + // cpu.rt_runtime_us and cpu.rt_period_us might still needed to be set manually +- __isula_auto_free char *init_cgroup = common_get_init_cgroup("cpu"); ++ __isula_auto_free char *init_cgroup = common_get_init_cgroup_path("cpu"); + if (init_cgroup == NULL) { + ERROR("Failed to get init cgroup"); + return -1; + } + // make sure that the own cgroup path for cpu existed +- __isula_auto_free char *own_cgroup = common_get_own_cgroup("cpu"); ++ __isula_auto_free char *own_cgroup = common_get_own_cgroup_path("cpu"); + if (own_cgroup == NULL) { + ERROR("Failed to get own cgroup"); + return -1; +@@ -453,7 +460,7 @@ static int cpurt_controller_init(const char *id, const host_config *host_spec) + cgroups_path = new_cgroups_path; + } + +- mnt_root = sysinfo_cgroup_controller_cpurt_mnt_path(); ++ mnt_root = sysinfo_get_cpurt_mnt_path(); + if (mnt_root == NULL) { + ERROR("Failed to get cpu rt controller mnt root path"); + return -1; +-- +2.34.1 + diff --git a/0030-adaptor-unit-test-for-cgroup-module.patch b/0030-adaptor-unit-test-for-cgroup-module.patch new file mode 100644 index 0000000..66fd4dd --- /dev/null +++ b/0030-adaptor-unit-test-for-cgroup-module.patch @@ -0,0 +1,351 @@ +From 59e7ea0f16e83e0bdbc39bdc41d1ade8d3db885e Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Thu, 22 Feb 2024 09:52:30 +0800 +Subject: [PATCH 30/43] adaptor unit test for cgroup module + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/modules/image/CMakeLists.txt | 6 ++++-- + test/cgroup/cpu/CMakeLists.txt | 7 +++++-- + test/cgroup/cpu/cgroup_cpu_ut.cc | 12 ++++++++++-- + test/image/oci/oci_config_merge/CMakeLists.txt | 7 +++++-- + test/image/oci/registry/CMakeLists.txt | 7 +++++-- + test/network/network_ns/CMakeLists.txt | 7 +++++-- + test/runtime/isula/CMakeLists.txt | 7 +++++-- + test/runtime/lcr/CMakeLists.txt | 7 +++++-- + .../execute/execution_extend/CMakeLists.txt | 6 +++++- + test/specs/specs/CMakeLists.txt | 7 +++++-- + test/specs/specs_extend/CMakeLists.txt | 7 +++++-- + test/specs/verify/CMakeLists.txt | 7 +++++-- + test/volume/CMakeLists.txt | 7 +++++-- + 13 files changed, 69 insertions(+), 25 deletions(-) + +diff --git a/src/daemon/modules/image/CMakeLists.txt b/src/daemon/modules/image/CMakeLists.txt +index f8bc5840..d8b78ce1 100644 +--- a/src/daemon/modules/image/CMakeLists.txt ++++ b/src/daemon/modules/image/CMakeLists.txt +@@ -68,8 +68,9 @@ set(LIB_ISULAD_IMG_SRCS + ${CMAKE_SOURCE_DIR}/src/utils/buffer/buffer.c + ${CMAKE_SOURCE_DIR}/src/daemon/common/err_msg.c + ${CMAKE_SOURCE_DIR}/src/daemon/common/sysinfo.c +- ${CMAKE_SOURCE_DIR}/src/daemon/common/cgroup.c +- ${CMAKE_SOURCE_DIR}/src/daemon/common/cgroup_v1.c ++ ${CMAKE_SOURCE_DIR}/src/daemon/common/cgroup/cgroup.c ++ ${CMAKE_SOURCE_DIR}/src/daemon/common/cgroup/cgroup_v1.c ++ ${CMAKE_SOURCE_DIR}/src/daemon/common/cgroup/cgroup_v2.c + ${CMAKE_SOURCE_DIR}/src/utils/tar/util_gzip.c + ${CMAKE_SOURCE_DIR}/src/daemon/config/isulad_config.c + ${CMAKE_SOURCE_DIR}/src/daemon/config/daemon_arguments.c +@@ -102,6 +103,7 @@ target_include_directories(${LIB_ISULAD_IMG} PUBLIC + ${CMAKE_SOURCE_DIR}/src/utils/tar + ${CMAKE_SOURCE_DIR}/src/daemon/config + ${CMAKE_SOURCE_DIR}/src/daemon/common ++ ${CMAKE_SOURCE_DIR}/src/daemon/common/cgroup + ${CMAKE_SOURCE_DIR}/src/daemon/modules/spec/ + ${CMAKE_SOURCE_DIR}/src/utils/cutils + ${CMAKE_SOURCE_DIR}/src/utils/cutils/map +diff --git a/test/cgroup/cpu/CMakeLists.txt b/test/cgroup/cpu/CMakeLists.txt +index 32fe0a23..30bfc417 100644 +--- a/test/cgroup/cpu/CMakeLists.txt ++++ b/test/cgroup/cpu/CMakeLists.txt +@@ -6,8 +6,10 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/path.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/sysinfo.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v2.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_common.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/command_parser.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/daemon_arguments.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/isulad_config.c +@@ -20,6 +22,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/common + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup + ${CMAKE_BINARY_DIR}/conf + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd +diff --git a/test/cgroup/cpu/cgroup_cpu_ut.cc b/test/cgroup/cpu/cgroup_cpu_ut.cc +index eaee90c0..6e6e04f4 100644 +--- a/test/cgroup/cpu/cgroup_cpu_ut.cc ++++ b/test/cgroup/cpu/cgroup_cpu_ut.cc +@@ -69,13 +69,21 @@ TEST(CgroupCpuUnitTest, test_common_find_cgroup_mnt_and_root) + { + char *mnt = NULL; + char *root = NULL; +- ASSERT_EQ(common_find_cgroup_mnt_and_root(nullptr, &mnt, &root), -1); ++ ++ int ret = cgroup_ops_init(); ++ ASSERT_EQ(ret, 0); ++ ++ ASSERT_EQ(common_get_cgroup_mnt_and_root_path(nullptr, &mnt, &root), -1); + } + + TEST(CgroupCpuUnitTest, test_sysinfo_cgroup_controller_cpurt_mnt_path) + { + MOCK_SET(util_common_calloc_s, nullptr); + ASSERT_EQ(get_sys_info(true), nullptr); +- ASSERT_EQ(sysinfo_cgroup_controller_cpurt_mnt_path(), nullptr); ++ ++ int ret = cgroup_ops_init(); ++ ASSERT_EQ(ret, 0); ++ ++ ASSERT_EQ(sysinfo_get_cpurt_mnt_path(), nullptr); + MOCK_CLEAR(util_common_calloc_s); + } +diff --git a/test/image/oci/oci_config_merge/CMakeLists.txt b/test/image/oci/oci_config_merge/CMakeLists.txt +index 90809080..d13ec738 100644 +--- a/test/image/oci/oci_config_merge/CMakeLists.txt ++++ b/test/image/oci/oci_config_merge/CMakeLists.txt +@@ -25,8 +25,10 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/err_msg.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/config/isulad_config.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/sysinfo.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup/cgroup.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup/cgroup_v2.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup/cgroup_common.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/cmd/command_parser.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/config/daemon_arguments.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/image/oci/oci_ut_common.cc +@@ -55,6 +57,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/utils/cutils/map + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/volume + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/api + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/runtime/engines +diff --git a/test/image/oci/registry/CMakeLists.txt b/test/image/oci/registry/CMakeLists.txt +index 77a7907e..5b5bc3f5 100644 +--- a/test/image/oci/registry/CMakeLists.txt ++++ b/test/image/oci/registry/CMakeLists.txt +@@ -26,8 +26,10 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/image_store/image_type.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/registry_type.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/sysinfo.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup/cgroup.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup/cgroup_v2.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup/cgroup_common.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/image_store/image_store.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/registry/registry.c +@@ -58,6 +60,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/api + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/image_store +diff --git a/test/network/network_ns/CMakeLists.txt b/test/network/network_ns/CMakeLists.txt +index 50520427..71b8039d 100644 +--- a/test/network/network_ns/CMakeLists.txt ++++ b/test/network/network_ns/CMakeLists.txt +@@ -30,8 +30,10 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/daemon_arguments.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/sysinfo.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v2.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_common.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/command_parser.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/image/oci/oci_ut_common.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/image_mock.cc +@@ -55,6 +57,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/options + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/api + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/volume + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/runtime +diff --git a/test/runtime/isula/CMakeLists.txt b/test/runtime/isula/CMakeLists.txt +index cc1178b8..c1f0a5cc 100644 +--- a/test/runtime/isula/CMakeLists.txt ++++ b/test/runtime/isula/CMakeLists.txt +@@ -21,8 +21,10 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/mainloop.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/sysinfo.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v2.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_common.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/command_parser.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/daemon_arguments.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/image/oci/oci_ut_common.cc +@@ -51,6 +53,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/container + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/container/restart_manager + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/container/health_check +diff --git a/test/runtime/lcr/CMakeLists.txt b/test/runtime/lcr/CMakeLists.txt +index 424a6101..c3b93d67 100644 +--- a/test/runtime/lcr/CMakeLists.txt ++++ b/test/runtime/lcr/CMakeLists.txt +@@ -18,8 +18,10 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map/rb_tree.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/sysinfo.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v2.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_common.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/command_parser.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/daemon_arguments.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/image/oci/oci_ut_common.cc +@@ -39,6 +41,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/api + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/container + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/container/restart_manager +diff --git a/test/services/execution/execute/execution_extend/CMakeLists.txt b/test/services/execution/execute/execution_extend/CMakeLists.txt +index 68e0f443..f0875fd7 100644 +--- a/test/services/execution/execute/execution_extend/CMakeLists.txt ++++ b/test/services/execution/execute/execution_extend/CMakeLists.txt +@@ -21,7 +21,10 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/mainloop.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/filters.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common/err_msg.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common/cgroup.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common/cgroup/cgroup.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common/cgroup/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common/cgroup/cgroup_v2.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common/cgroup/cgroup_common.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/events_sender/event_sender.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/console/console.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/utils.c +@@ -59,6 +62,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/console + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common/cgroup + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/api + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image +diff --git a/test/specs/specs/CMakeLists.txt b/test/specs/specs/CMakeLists.txt +index 508123fa..45f688f9 100644 +--- a/test/specs/specs/CMakeLists.txt ++++ b/test/specs/specs/CMakeLists.txt +@@ -28,8 +28,10 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/spec/specs_security.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/sysinfo.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v2.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_common.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/command_parser.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/daemon_arguments.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/image/oci/oci_ut_common.cc +@@ -54,6 +56,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/api + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/volume + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/runtime +diff --git a/test/specs/specs_extend/CMakeLists.txt b/test/specs/specs_extend/CMakeLists.txt +index bf4b378e..1b737089 100644 +--- a/test/specs/specs_extend/CMakeLists.txt ++++ b/test/specs/specs_extend/CMakeLists.txt +@@ -28,8 +28,10 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/spec/specs_security.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/sysinfo.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v2.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_common.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/command_parser.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/daemon_arguments.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/image/oci/oci_ut_common.cc +@@ -54,6 +56,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/volume + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/api + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/container +diff --git a/test/specs/verify/CMakeLists.txt b/test/specs/verify/CMakeLists.txt +index 0e60a39e..b0602127 100644 +--- a/test/specs/verify/CMakeLists.txt ++++ b/test/specs/verify/CMakeLists.txt +@@ -23,8 +23,10 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map/rb_tree.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/sysinfo.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v2.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_common.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/spec/verify.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/image/oci/oci_ut_common.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/containers_store_mock.cc +@@ -50,6 +52,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/api + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/volume + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/runtime +diff --git a/test/volume/CMakeLists.txt b/test/volume/CMakeLists.txt +index cc309352..27d07330 100644 +--- a/test/volume/CMakeLists.txt ++++ b/test/volume/CMakeLists.txt +@@ -20,8 +20,10 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/utils/sha256/sha256.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/err_msg.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/sysinfo.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup/cgroup.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup/cgroup_v2.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup/cgroup_common.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/cmd/command_parser.c + volume_ut.cc) + +@@ -34,6 +36,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/utils/http + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/modules/api + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/modules/volume + ${CMAKE_BINARY_DIR}/conf + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/utils/sha256 +-- +2.34.1 + diff --git a/0031-cgroup-v2-does-not-support-isulad-setting-cpu_rt-opt.patch b/0031-cgroup-v2-does-not-support-isulad-setting-cpu_rt-opt.patch new file mode 100644 index 0000000..ecca55a --- /dev/null +++ b/0031-cgroup-v2-does-not-support-isulad-setting-cpu_rt-opt.patch @@ -0,0 +1,71 @@ +From 8e11a1eea62cb8061f1613379ff83bd9a721fa50 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Wed, 31 Jan 2024 18:10:46 +0800 +Subject: [PATCH 31/43] cgroup v2 does not support isulad setting cpu_rt + options + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/cmd/isulad/isulad_commands.c | 32 ++++++++++++++++++++++++++++++++ + 1 file changed, 32 insertions(+) + +diff --git a/src/cmd/isulad/isulad_commands.c b/src/cmd/isulad/isulad_commands.c +index 5fb55689..619e36d1 100644 +--- a/src/cmd/isulad/isulad_commands.c ++++ b/src/cmd/isulad/isulad_commands.c +@@ -34,6 +34,7 @@ + #include "utils_verify.h" + #include "opt_ulimit.h" + #include "opt_log.h" ++#include "sysinfo.h" + + const char isulad_desc[] = "GLOBAL OPTIONS:"; + const char isulad_usage[] = "[global options]"; +@@ -411,6 +412,33 @@ out: + return ret; + } + ++static int check_args_cpu_rt(const struct service_arguments *args) ++{ ++ int ret = 0; ++ __isula_auto_sysinfo_t sysinfo_t *sysinfo = NULL; ++ ++ sysinfo = get_sys_info(true); ++ if (sysinfo == NULL) { ++ COMMAND_ERROR("Failed to get system info"); ++ ERROR("Failed to get system info"); ++ return -1; ++ } ++ ++ if (!(sysinfo->cgcpuinfo.cpu_rt_period) && args->json_confs->cpu_rt_period != 0) { ++ COMMAND_ERROR("Invalid --cpu-rt-period: Your kernel does not support cgroup rt period"); ++ ERROR("Invalid --cpu-rt-period: Your kernel does not support cgroup rt period"); ++ return -1; ++ } ++ ++ if (!(sysinfo->cgcpuinfo.cpu_rt_runtime) && args->json_confs->cpu_rt_runtime != 0) { ++ COMMAND_ERROR("Invalid --cpu-rt-runtime: Your kernel does not support cgroup rt runtime"); ++ ERROR("Invalid --cpu-rt-runtime: Your kernel does not support cgroup rt runtime"); ++ return -1; ++ } ++ ++ return ret; ++} ++ + int check_args(struct service_arguments *args) + { + int ret = 0; +@@ -471,6 +499,10 @@ int check_args(struct service_arguments *args) + goto out; + } + ++ if (check_args_cpu_rt(args) != 0) { ++ ret = -1; ++ goto out; ++ } + out: + return ret; + } +-- +2.34.1 + diff --git a/0032-add-test-that-isulad-cannot-set-cpu_rt-parameters-wh.patch b/0032-add-test-that-isulad-cannot-set-cpu_rt-parameters-wh.patch new file mode 100644 index 0000000..1568edb --- /dev/null +++ b/0032-add-test-that-isulad-cannot-set-cpu_rt-parameters-wh.patch @@ -0,0 +1,30 @@ +From 1ab0f4608fb749b50aa6f8d8188db23aa8a6e1ac Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Thu, 1 Feb 2024 10:48:45 +0800 +Subject: [PATCH 32/43] add test that isulad cannot set cpu_rt parameters when + adding cgroup v2 + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + CI/test_cases/container_cases/cpu_rt.sh | 5 ++++- + 1 file changed, 4 insertions(+), 1 deletion(-) + +diff --git a/CI/test_cases/container_cases/cpu_rt.sh b/CI/test_cases/container_cases/cpu_rt.sh +index bdc43a5e..23d3baed 100755 +--- a/CI/test_cases/container_cases/cpu_rt.sh ++++ b/CI/test_cases/container_cases/cpu_rt.sh +@@ -106,7 +106,10 @@ function test_kernel_without_cpurt() + + msg_info "${test} starting..." + +- start_isulad_without_valgrind --cpu-rt-period 1000000 --cpu-rt-runtime 950000 ++ isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 2>&1 | grep 'Your kernel does not support cgroup rt period' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - kernel does not support cpu-rt, but start isulad with cpu-rt success" && ((ret++)) ++ ++ start_isulad_without_valgrind + + isula pull ${image} + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${image}" && return ${FAILURE} +-- +2.34.1 + diff --git a/0033-fix-sandbox-container-bool-value-uninitialized.patch b/0033-fix-sandbox-container-bool-value-uninitialized.patch new file mode 100644 index 0000000..2f88424 --- /dev/null +++ b/0033-fix-sandbox-container-bool-value-uninitialized.patch @@ -0,0 +1,26 @@ +From f62df3dedbbe11bb56e6da7dd610c573fd3ed828 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Mon, 25 Mar 2024 10:01:56 +0800 +Subject: [PATCH 33/43] fix sandbox container bool value uninitialized + +Signed-off-by: jikai <jikai11@huawei.com> +--- + src/daemon/modules/service/service_container.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c +index 7b34cc7f..a8090d5a 100644 +--- a/src/daemon/modules/service/service_container.c ++++ b/src/daemon/modules/service/service_container.c +@@ -750,7 +750,7 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo + oci_runtime_spec *oci_spec = NULL; + rt_create_params_t create_params = { 0 }; + rt_start_params_t start_params = { 0 }; +- bool sandbox_container; ++ bool sandbox_container = false; + + nret = snprintf(bundle, sizeof(bundle), "%s/%s", cont->root_path, id); + if (nret < 0 || (size_t)nret >= sizeof(bundle)) { +-- +2.34.1 + diff --git a/0034-bugfix-for-cpurt.sh.patch b/0034-bugfix-for-cpurt.sh.patch new file mode 100644 index 0000000..d8b52bb --- /dev/null +++ b/0034-bugfix-for-cpurt.sh.patch @@ -0,0 +1,47 @@ +From 411483ad9b2a0c50190f9b56779d41889c895014 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Wed, 27 Mar 2024 10:29:11 +0800 +Subject: [PATCH 34/43] bugfix for cpurt.sh + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + CI/test_cases/container_cases/cpu_rt.sh | 9 ++++----- + 1 file changed, 4 insertions(+), 5 deletions(-) + +diff --git a/CI/test_cases/container_cases/cpu_rt.sh b/CI/test_cases/container_cases/cpu_rt.sh +index 23d3baed..64dcd81f 100755 +--- a/CI/test_cases/container_cases/cpu_rt.sh ++++ b/CI/test_cases/container_cases/cpu_rt.sh +@@ -109,7 +109,7 @@ function test_kernel_without_cpurt() + isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 2>&1 | grep 'Your kernel does not support cgroup rt period' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - kernel does not support cpu-rt, but start isulad with cpu-rt success" && ((ret++)) + +- start_isulad_without_valgrind ++ start_isulad_with_valgrind + + isula pull ${image} + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${image}" && return ${FAILURE} +@@ -194,6 +194,9 @@ function do_test() + test_cpurt_isulad_abnormal $runtime || ((ret++)) + test_isula_update_normal $runtime || ((ret++)) + test_isula_update_abnormal $runtime || ((ret++)) ++ stop_isulad_without_valgrind ++ # set cpu-rt to the initial state ++ start_isulad_with_valgrind --cpu-rt-period 1000000 --cpu-rt-runtime 0 + else + test_kernel_without_cpurt $runtime || ((ans++)) + fi +@@ -211,10 +214,6 @@ do + + do_test $element || ((ans++)) + +- stop_isulad_without_valgrind +- # set cpu-rt to the initial state +- start_isulad_with_valgrind --cpu-rt-period 1000000 --cpu-rt-runtime 0 +- + isula rm -f $(isula ps -aq) + done + +-- +2.34.1 + diff --git a/0035-monitor-cgroup-oom-killed-event-and-update-to-cri-of.patch b/0035-monitor-cgroup-oom-killed-event-and-update-to-cri-of.patch new file mode 100644 index 0000000..85f7f62 --- /dev/null +++ b/0035-monitor-cgroup-oom-killed-event-and-update-to-cri-of.patch @@ -0,0 +1,868 @@ +From 947cf87a87ec49409ae509e5142b8134454d1547 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Thu, 28 Mar 2024 12:51:09 +0000 +Subject: [PATCH 35/43] monitor cgroup oom killed event and update to cri of + container + +Signed-off-by: jikai <jikai11@huawei.com> +--- + src/daemon/common/cgroup/cgroup.c | 91 +++++++++- + src/daemon/common/cgroup/cgroup.h | 5 + + src/daemon/common/cgroup/cgroup_common.h | 13 ++ + src/daemon/common/cgroup/cgroup_v1.c | 160 ++++++++++++++++++ + src/daemon/common/cgroup/cgroup_v2.c | 138 ++++++++++++++- + .../v1/v1_cri_container_manager_service.cc | 3 + + src/daemon/modules/api/container_api.h | 5 +- + .../container/container_events_handler.c | 12 +- + .../modules/container/container_state.c | 15 ++ + .../modules/container/restore/restore.c | 10 +- + .../modules/container/supervisor/supervisor.c | 54 +++++- + src/daemon/modules/events/collector.c | 7 +- + .../modules/service/service_container.c | 11 +- + 13 files changed, 498 insertions(+), 26 deletions(-) + +diff --git a/src/daemon/common/cgroup/cgroup.c b/src/daemon/common/cgroup/cgroup.c +index 837b514a..d3f1445a 100644 +--- a/src/daemon/common/cgroup/cgroup.c ++++ b/src/daemon/common/cgroup/cgroup.c +@@ -133,4 +133,93 @@ char *common_get_own_cgroup_path(const char *subsystem) + } + + return g_cgroup_ops.get_own_cgroup_path(subsystem); +-} +\ No newline at end of file ++} ++ ++char *common_convert_cgroup_path(const char *cgroup_path) ++{ ++ char *token = NULL; ++ char result[PATH_MAX + 1] = {0}; ++ __isula_auto_array_t char **arr = NULL; ++ ++ if (cgroup_path == NULL) { ++ ERROR("Invalid NULL cgroup path"); ++ return NULL; ++ } ++ ++ // for cgroup fs cgroup path, return directly ++ if (!util_has_suffix(cgroup_path, ".slice")) { ++ return util_strdup_s(cgroup_path); ++ } ++ ++ // for systemd cgroup, cgroup_path should have the form slice:prefix:id, ++ // convert it to a true path, such as from test-a.slice:isulad:id ++ // to test.slice/test-a.slice/isulad-id.scope ++ arr = util_string_split_n(cgroup_path, ':', 3); ++ if (arr == NULL || util_array_len((const char **)arr) != 3) { ++ ERROR("Invalid systemd cgroup parent"); ++ return NULL; ++ } ++ ++ token = strchr(arr[0], '-'); ++ while (token != NULL) { ++ *token = '\0'; ++ if (strlen(arr[0]) > PATH_MAX || strlen(result) + 1 + strlen(".slice") > ++ PATH_MAX - strlen(arr[0])) { ++ ERROR("Invalid systemd cgroup parent: exceeds max length of path"); ++ *token = '-'; ++ return NULL; ++ } ++ if (result[0] != '\0') { ++ strcat(result, "/"); ++ } ++ strcat(result, arr[0]); ++ strcat(result, ".slice"); ++ *token = '-'; ++ token = strchr(token + 1, '-'); ++ } ++ ++ // Add /arr[0]/arr[1]-arr[2].scope, 3 include two slashes and one dash ++ if (strlen(cgroup_path) > PATH_MAX || strlen(result) + 3 + strlen(".scope") > ++ PATH_MAX - strlen(arr[0] - strlen(arr[1]) - strlen(arr[2]))) { ++ ERROR("Invalid systemd cgroup parent: exceeds max length of path"); ++ return NULL; ++ } ++ ++ (void)strcat(result, "/"); ++ (void)strcat(result, arr[0]); ++ (void)strcat(result, "/"); ++ (void)strcat(result, arr[1]); ++ (void)strcat(result, "-"); ++ (void)strcat(result, arr[2]); ++ (void)strcat(result, ".scope"); ++ ++ return util_strdup_s(result); ++} ++ ++cgroup_oom_handler_info_t *common_get_cgroup_oom_handler(int fd, const char *name, const char *cgroup_path, const char *exit_fifo) ++{ ++ if (g_cgroup_ops.get_cgroup_oom_handler == NULL) { ++ ERROR("Unimplmented get_cgroup_oom_handler op"); ++ return NULL; ++ } ++ ++ return g_cgroup_ops.get_cgroup_oom_handler(fd, name, cgroup_path, exit_fifo); ++} ++ ++void common_free_cgroup_oom_handler_info(cgroup_oom_handler_info_t *info) ++{ ++ if (info == NULL) { ++ return; ++ } ++ ++ if (info->oom_event_fd >= 0) { ++ close(info->oom_event_fd); ++ } ++ if (info->cgroup_file_fd >= 0) { ++ close(info->cgroup_file_fd); ++ } ++ ++ free(info->name); ++ free(info->cgroup_memory_event_path); ++ free(info); ++} +diff --git a/src/daemon/common/cgroup/cgroup.h b/src/daemon/common/cgroup/cgroup.h +index 1efc3ca6..8c76d99d 100644 +--- a/src/daemon/common/cgroup/cgroup.h ++++ b/src/daemon/common/cgroup/cgroup.h +@@ -41,6 +41,11 @@ int common_get_cgroup_mnt_and_root_path(const char *subsystem, char **mountpoint + char *common_get_init_cgroup_path(const char *subsystem); + char *common_get_own_cgroup_path(const char *subsystem); + ++char *common_convert_cgroup_path(const char *cgroup_path); ++ ++cgroup_oom_handler_info_t *common_get_cgroup_oom_handler(int fd, const char *name, const char *cgroup_path, const char *exit_fifo); ++void common_free_cgroup_oom_handler_info(cgroup_oom_handler_info_t *info); ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/common/cgroup/cgroup_common.h b/src/daemon/common/cgroup/cgroup_common.h +index 2a0935cb..e3912bf0 100644 +--- a/src/daemon/common/cgroup/cgroup_common.h ++++ b/src/daemon/common/cgroup/cgroup_common.h +@@ -116,6 +116,17 @@ typedef struct { + cgroup_pids_metrics_t cgpids_metrics; + } cgroup_metrics_t; + ++#define CGROUP_OOM_HANDLE_CONTINUE false ++#define CGROUP_OOM_HANDLE_CLOSE true ++ ++typedef struct _cgroup_oom_handler_info_t { ++ int oom_event_fd; ++ int cgroup_file_fd; ++ char *name; ++ char *cgroup_memory_event_path; ++ bool (*oom_event_handler)(int, void *); ++} cgroup_oom_handler_info_t; ++ + typedef struct { + int (*get_cgroup_version)(void); + int (*get_cgroup_info)(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, +@@ -128,6 +139,8 @@ typedef struct { + + char *(*get_init_cgroup_path)(const char *subsystem); + char *(*get_own_cgroup_path)(const char *subsystem); ++ ++ cgroup_oom_handler_info_t *(*get_cgroup_oom_handler)(int fd, const char *name, const char *cgroup_path, const char *exit_fifo); + } cgroup_ops; + + #ifdef __cplusplus +diff --git a/src/daemon/common/cgroup/cgroup_v1.c b/src/daemon/common/cgroup/cgroup_v1.c +index 51cf7512..41f3110a 100644 +--- a/src/daemon/common/cgroup/cgroup_v1.c ++++ b/src/daemon/common/cgroup/cgroup_v1.c +@@ -12,14 +12,20 @@ + * Create: 2023-03-29 + * Description: provide cgroup v1 functions + ******************************************************************************/ ++#ifndef _GNU_SOURCE ++#define _GNU_SOURCE ++#endif ++ + #include "cgroup.h" + + #include <stdio.h> + #include <stdlib.h> ++#include <sys/eventfd.h> + + #include "utils.h" + #include "sysinfo.h" + #include "err_msg.h" ++#include "events_sender_api.h" + + #define CGROUP_HUGETLB_LIMIT "hugetlb.%s.limit_in_bytes" + #define CGROUP_MOUNT_PATH_PREFIX "/sys/fs/cgroup/" +@@ -1045,6 +1051,159 @@ static char *common_get_cgroup_path(const char *path, const char *subsystem) + return res; + } + ++static bool oom_cb_cgroup_v1(int fd, void *cbdata) ++{ ++ cgroup_oom_handler_info_t *info = (cgroup_oom_handler_info_t *)cbdata; ++ /* Try to read cgroup.event_control and known if the cgroup was removed ++ * if the cgroup was removed and only one event received, ++ * we know that it is a cgroup removal event rather than an oom event ++ */ ++ bool cgroup_removed = false; ++ if (info == NULL) { ++ ERROR("Invalide callback data"); ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ if (access(info->cgroup_memory_event_path, F_OK) < 0) { ++ DEBUG("Cgroup event path was removed"); ++ cgroup_removed = true; ++ } ++ ++ uint64_t event_count; ++ ssize_t num_read = util_read_nointr(fd, &event_count, sizeof(uint64_t)); ++ if (num_read < 0) { ++ ERROR("Failed to read oom event from eventfd"); ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ if (num_read == 0) { ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ if (num_read != sizeof(uint64_t)) { ++ ERROR("Failed to read full oom event from eventfd"); ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ if (event_count == 0) { ++ ERROR("Unexpected event count when reading for oom event"); ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ if (event_count == 1 && cgroup_removed) { ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ INFO("OOM event detected"); ++ (void)isulad_monitor_send_container_event(info->name, OOM, -1, 0, NULL, NULL); ++ ++ return CGROUP_OOM_HANDLE_CLOSE; ++} ++ ++static char *get_memory_cgroup_path_v1(const char *cgroup_path) ++{ ++ int nret = 0; ++ __isula_auto_free char *converted_cgroup_path = NULL; ++ __isula_auto_free char *mnt = NULL; ++ __isula_auto_free char *root = NULL; ++ char fpath[PATH_MAX] = { 0 }; ++ ++ converted_cgroup_path = common_convert_cgroup_path(cgroup_path); ++ if (converted_cgroup_path == NULL) { ++ ERROR("Failed to transfer cgroup path"); ++ return NULL; ++ } ++ ++ nret = get_cgroup_mnt_and_root_path_v1("memory", &mnt, &root); ++ if (nret != 0 || mnt == NULL || root == NULL) { ++ ERROR("Can not find cgroup mnt and root path for subsystem 'memory'"); ++ return NULL; ++ } ++ ++ // When iSulad is run inside docker, the root is based of the host cgroup. ++ // Replace root to "/" ++ if (strncmp(root, "/docker/", strlen("/docker/")) == 0) { ++ root[1] = '\0'; ++ } ++ ++ nret = snprintf(fpath, sizeof(fpath), "%s/%s", mnt, root); ++ if (nret < 0 || (size_t)nret >= sizeof(fpath)) { ++ ERROR("Failed to print string"); ++ return NULL; ++ } ++ ++ return util_path_join(fpath, converted_cgroup_path); ++} ++ ++static cgroup_oom_handler_info_t *get_cgroup_oom_handler_v1(int fd, const char *name, const char *cgroup_path, const char *exit_fifo) ++{ ++ __isula_auto_free char *memory_cgroup_path = NULL; ++ __isula_auto_free char *memory_cgroup_oom_control_path = NULL; ++ __isula_auto_free char *data = NULL; ++ __isula_auto_close int cgroup_event_control_fd = -1; ++ if (name == NULL || cgroup_path == NULL || exit_fifo == NULL) { ++ ERROR("Invalid arguments"); ++ return NULL; ++ } ++ ++ cgroup_oom_handler_info_t *info = util_common_calloc_s(sizeof(cgroup_oom_handler_info_t)); ++ if (info == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ info->name = util_strdup_s(name); ++ info->cgroup_file_fd = -1; ++ info->oom_event_fd = -1; ++ info->oom_event_handler = oom_cb_cgroup_v1; ++ ++ memory_cgroup_path = get_memory_cgroup_path_v1(cgroup_path); ++ if (memory_cgroup_path == NULL) { ++ ERROR("Failed to get memory cgroup path"); ++ goto cleanup; ++ } ++ ++ info->cgroup_memory_event_path = util_path_join(memory_cgroup_path, "cgroup.event_control"); ++ if (info->cgroup_memory_event_path == NULL) { ++ ERROR("Failed to join memory cgroup file path"); ++ goto cleanup; ++ } ++ ++ cgroup_event_control_fd = util_open(info->cgroup_memory_event_path, O_WRONLY | O_CLOEXEC, 0); ++ if (cgroup_event_control_fd < 0) { ++ ERROR("Failed to open %s", info->cgroup_memory_event_path); ++ goto cleanup; ++ } ++ ++ memory_cgroup_oom_control_path = util_path_join(memory_cgroup_path, "memory.oom_control"); ++ if (memory_cgroup_oom_control_path == NULL) { ++ ERROR("Failed to join memory cgroup file path"); ++ goto cleanup; ++ } ++ ++ info->cgroup_file_fd = util_open(memory_cgroup_oom_control_path, O_RDONLY | O_CLOEXEC, 0); ++ if (info->cgroup_file_fd < 0) { ++ ERROR("Failed to open %s", memory_cgroup_oom_control_path); ++ goto cleanup; ++ } ++ ++ info->oom_event_fd = eventfd(0, EFD_CLOEXEC); ++ if (info->oom_event_fd < 0) { ++ ERROR("Failed to create oom eventfd"); ++ goto cleanup; ++ } ++ ++ if (asprintf(&data, "%d %d", info->oom_event_fd, info->cgroup_file_fd) < 0 || ++ util_write_nointr(cgroup_event_control_fd, data, strlen(data)) < 0) { ++ ERROR("Failed to write to cgroup.event_control"); ++ goto cleanup; ++ } ++ ++ return info; ++cleanup: ++ common_free_cgroup_oom_handler_info(info); ++ return NULL; ++} ++ + char *get_init_cgroup_path_v1(const char *subsystem) + { + return common_get_cgroup_path("/proc/1/cgroup", subsystem); +@@ -1071,5 +1230,6 @@ int cgroup_v1_ops_init(cgroup_ops *ops) + ops->get_cgroup_mnt_and_root_path = get_cgroup_mnt_and_root_path_v1; + ops->get_init_cgroup_path = get_init_cgroup_path_v1; + ops->get_own_cgroup_path = get_own_cgroup_v1; ++ ops->get_cgroup_oom_handler = get_cgroup_oom_handler_v1; + return 0; + } +\ No newline at end of file +diff --git a/src/daemon/common/cgroup/cgroup_v2.c b/src/daemon/common/cgroup/cgroup_v2.c +index 65cf90d8..a36258f0 100644 +--- a/src/daemon/common/cgroup/cgroup_v2.c ++++ b/src/daemon/common/cgroup/cgroup_v2.c +@@ -17,12 +17,14 @@ + #include <stdio.h> + #include <stdlib.h> + #include <sys/stat.h> ++#include <sys/inotify.h> + + #include <isula_libutils/auto_cleanup.h> + + #include "utils.h" + #include "path.h" + #include "sysinfo.h" ++#include "events_sender_api.h" + + // Cgroup V2 Item Definition + #define CGROUP2_CPU_WEIGHT "cpu.weight" +@@ -408,10 +410,143 @@ static int get_cgroup_metrics_v2(const char *cgroup_path, cgroup_metrics_t *cgro + + static int get_cgroup_mnt_and_root_v2(const char *subsystem, char **mountpoint, char **root) + { +- *mountpoint = util_strdup_s(CGROUP_ISULAD_PATH); ++ if (mountpoint != NULL) { ++ *mountpoint = util_strdup_s(CGROUP_ISULAD_PATH); ++ } + return 0; + } + ++static bool oom_cb_cgroup_v2(int fd, void *cbdata) ++{ ++ const size_t events_size = sizeof(struct inotify_event) + NAME_MAX + 1; ++ char events[events_size]; ++ cgroup_oom_handler_info_t *info = (cgroup_oom_handler_info_t *)cbdata; ++ ++ if (info == NULL) { ++ ERROR("Invalid callback data"); ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ ssize_t num_read = util_read_nointr(fd, &events, events_size); ++ if (num_read < 0) { ++ ERROR("Failed to read oom event from eventfd in v2"); ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ if (((struct inotify_event *)events)->mask & ( IN_DELETE | IN_DELETE_SELF)) { ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ __isula_auto_file FILE *fp = fopen(info->cgroup_memory_event_path, "re"); ++ if (fp == NULL) { ++ ERROR("Failed to open cgroups file: %s", info->cgroup_memory_event_path); ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ __isula_auto_free char *line = NULL; ++ size_t len = 0; ++ ssize_t read; ++ while ((read = getline(&line, &len, fp)) != -1) { ++ int count; ++ const char *oom_str = "oom "; ++ const char *oom_kill_str = "oom_kill "; ++ const int oom_len = strlen(oom_str), oom_kill_len = strlen(oom_kill_str); ++ ++ if (read >= oom_kill_len + 2 && memcmp(line, oom_kill_str, oom_kill_len) == 0) { ++ len = oom_kill_len; ++ } else if (read >= oom_len + 2 && memcmp(line, oom_str, oom_len) == 0) { ++ len = oom_len; ++ } else { ++ continue; ++ } ++ ++ // to make use of util_safe_int, it requires it ends with '\0' ++ line[strcspn(line, "\n")] = '\0'; ++ if (util_safe_int(&line[len], &count) < 0) { ++ ERROR("Failed to parse: %s", &line[len]); ++ continue; ++ } ++ ++ if (count == 0) { ++ continue; ++ } ++ ++ INFO("OOM event detected in cgroup v2"); ++ (void)isulad_monitor_send_container_event(info->name, OOM, -1, 0, NULL, NULL); ++ ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ return CGROUP_OOM_HANDLE_CONTINUE; ++} ++ ++static char *get_real_cgroup_path_v2(const char *cgroup_path) ++{ ++ __isula_auto_free char *converted_cgroup_path = NULL; ++ converted_cgroup_path = common_convert_cgroup_path(cgroup_path); ++ if (converted_cgroup_path == NULL) { ++ ERROR("Failed to convert cgroup path"); ++ return NULL; ++ } ++ ++ return util_path_join(CGROUP_MOUNTPOINT, converted_cgroup_path); ++} ++ ++cgroup_oom_handler_info_t *get_cgroup_oom_handler_v2(int fd, const char *name, const char *cgroup_path, const char *exit_fifo) ++{ ++ __isula_auto_free char *real_cgroup_path = NULL; ++ if (name == NULL || cgroup_path == NULL || exit_fifo == NULL) { ++ ERROR("Invalid arguments"); ++ return NULL; ++ } ++ ++ cgroup_oom_handler_info_t *info = util_common_calloc_s(sizeof(cgroup_oom_handler_info_t)); ++ if (info == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ info->name = util_strdup_s(name); ++ info->oom_event_fd = -1; ++ info->cgroup_file_fd = -1; ++ info->oom_event_handler = oom_cb_cgroup_v2; ++ ++ real_cgroup_path = get_real_cgroup_path_v2(cgroup_path); ++ if (real_cgroup_path == NULL) { ++ ERROR("Failed to transfer cgroup path: %s", cgroup_path); ++ goto cleanup; ++ } ++ ++ info->cgroup_memory_event_path = util_path_join(real_cgroup_path, "memory.events"); ++ if (info->cgroup_memory_event_path == NULL) { ++ ERROR("Failed to join path"); ++ goto cleanup; ++ } ++ ++ if ((info->oom_event_fd = inotify_init()) < 0) { ++ ERROR("Failed to init inotify fd"); ++ goto cleanup; ++ } ++ ++ if (inotify_add_watch(info->oom_event_fd, info->cgroup_memory_event_path, IN_MODIFY) < 0) { ++ ERROR("Failed to watch inotify fd for %s", info->cgroup_memory_event_path); ++ goto cleanup; ++ } ++ ++ // watch exit fifo for container exit, so we can close the inotify fd ++ // because inotify cannot watch cgroup file delete event ++ if (inotify_add_watch(info->oom_event_fd, exit_fifo, IN_DELETE | IN_DELETE_SELF) < 0) { ++ ERROR("Failed to watch inotify fd for %s", exit_fifo); ++ goto cleanup; ++ } ++ ++ return info; ++ ++cleanup: ++ common_free_cgroup_oom_handler_info(info); ++ return NULL; ++} ++ + int get_cgroup_version_v2() + { + return CGROUP_VERSION_2; +@@ -426,5 +561,6 @@ int cgroup_v2_ops_init(cgroup_ops *ops) + ops->get_cgroup_info = get_cgroup_info_v2; + ops->get_cgroup_metrics = get_cgroup_metrics_v2; + ops->get_cgroup_mnt_and_root_path = get_cgroup_mnt_and_root_v2; ++ ops->get_cgroup_oom_handler = get_cgroup_oom_handler_v2; + return 0; + } +\ No newline at end of file +diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +index 47a33c2c..cac5c0ba 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +@@ -1055,6 +1055,9 @@ void ContainerManagerService::UpdateBaseStatusFromInspect( + } else { // Case 3 + state = runtime::v1::CONTAINER_CREATED; + } ++ if (inspect->state->oom_killed) { ++ reason = "OOMKilled"; ++ } + if (inspect->state->error != nullptr) { + message = inspect->state->error; + } +diff --git a/src/daemon/modules/api/container_api.h b/src/daemon/modules/api/container_api.h +index 43d66d64..830fd696 100644 +--- a/src/daemon/modules/api/container_api.h ++++ b/src/daemon/modules/api/container_api.h +@@ -221,6 +221,8 @@ void container_state_set_restarting(container_state_t *s, int exit_code); + void container_state_set_paused(container_state_t *s); + void container_state_reset_paused(container_state_t *s); + ++void container_state_set_oom_killed(container_state_t *s); ++ + void container_state_set_dead(container_state_t *s); + + void container_state_increase_restart_count(container_state_t *s); +@@ -269,8 +271,7 @@ bool container_is_valid_state_string(const char *state); + + void container_update_health_monitor(const char *container_id); + +-extern int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_info, const char *name, +- const char *runtime, bool sandbox_container); ++extern int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const pid_ppid_info_t *pid_info, const container_t *cont); + + extern char *container_exit_fifo_create(const char *cont_state_path); + +diff --git a/src/daemon/modules/container/container_events_handler.c b/src/daemon/modules/container/container_events_handler.c +index b84f1ad5..109a628c 100644 +--- a/src/daemon/modules/container/container_events_handler.c ++++ b/src/daemon/modules/container/container_events_handler.c +@@ -114,7 +114,7 @@ static int container_state_changed(container_t *cont, const struct isulad_events + bool has_been_manually_stopped = false; + + /* only handle Exit event */ +- if (events->type != EVENTS_TYPE_STOPPED1) { ++ if (events->type != EVENTS_TYPE_STOPPED1 && events->type != EVENTS_TYPE_OOM) { + return 0; + } + +@@ -187,6 +187,16 @@ static int container_state_changed(container_t *cont, const struct isulad_events + } + + break; ++ ++ case EVENTS_TYPE_OOM: { ++ container_lock(cont); ++ container_state_set_oom_killed(cont->state); ++ if (container_state_to_disk(cont)) { ++ WARN("Failed to save container \"%s\" to disk", id); ++ } ++ container_unlock(cont); ++ break; ++ } + default: + /* ignore garbage */ + break; +diff --git a/src/daemon/modules/container/container_state.c b/src/daemon/modules/container/container_state.c +index f31959fa..452a2b26 100644 +--- a/src/daemon/modules/container/container_state.c ++++ b/src/daemon/modules/container/container_state.c +@@ -154,6 +154,7 @@ void container_state_set_running(container_state_t *s, const pid_ppid_info_t *pi + state->paused = false; + } + state->exit_code = 0; ++ state->oom_killed = false; + + if (pid_info != NULL) { + state->pid = pid_info->pid; +@@ -222,6 +223,19 @@ void container_state_set_paused(container_state_t *s) + container_state_unlock(s); + } + ++void container_state_set_oom_killed(container_state_t *s) ++{ ++ if (s == NULL || s->state == NULL) { ++ return; ++ } ++ ++ container_state_lock(s); ++ ++ s->state->oom_killed = true; ++ ++ container_state_unlock(s); ++} ++ + /* state reset paused */ + void container_state_reset_paused(container_state_t *s) + { +@@ -573,6 +587,7 @@ container_inspect_state *container_state_to_inspect_state(container_state_t *s) + state->running = s->state->running; + state->paused = s->state->paused; + state->restarting = s->state->restarting; ++ state->oom_killed = s->state->oom_killed; + state->pid = s->state->pid; + + state->exit_code = s->state->exit_code; +diff --git a/src/daemon/modules/container/restore/restore.c b/src/daemon/modules/container/restore/restore.c +index 76868e28..52f68d21 100644 +--- a/src/daemon/modules/container/restore/restore.c ++++ b/src/daemon/modules/container/restore/restore.c +@@ -24,6 +24,7 @@ + #include <isula_libutils/container_config_v2.h> + #include <isula_libutils/host_config.h> + #include <isula_libutils/log.h> ++#include <isula_libutils/auto_cleanup.h> + + #include "isulad_config.h" + +@@ -44,6 +45,8 @@ + #include "utils_file.h" + #include "utils_timestamp.h" + #include "id_name_manager.h" ++#include "cgroup.h" ++#include "specs_api.h" + + /* restore supervisor */ + static int restore_supervisor(const container_t *cont) +@@ -55,9 +58,7 @@ static int restore_supervisor(const container_t *cont) + char *exit_fifo = NULL; + char *id = cont->common_config->id; + char *statepath = cont->state_path; +- char *runtime = cont->runtime; + pid_ppid_info_t pid_info = { 0 }; +- bool sandbox_container = false; + + nret = snprintf(container_state, sizeof(container_state), "%s/%s", statepath, id); + if (nret < 0 || (size_t)nret >= sizeof(container_state)) { +@@ -91,11 +92,8 @@ static int restore_supervisor(const container_t *cont) + pid_info.ppid = cont->state->state->p_pid; + pid_info.start_time = cont->state->state->start_time; + pid_info.pstart_time = cont->state->state->p_start_time; +-#ifdef ENABLE_CRI_API_V1 +- sandbox_container = is_sandbox_container(cont->common_config->sandbox_info); +-#endif + +- if (container_supervisor_add_exit_monitor(exit_fifo_fd, &pid_info, id, runtime, sandbox_container)) { ++ if (container_supervisor_add_exit_monitor(exit_fifo_fd, exit_fifo, &pid_info, cont)) { + ERROR("Failed to add exit monitor to supervisor"); + ret = -1; + goto out; +diff --git a/src/daemon/modules/container/supervisor/supervisor.c b/src/daemon/modules/container/supervisor/supervisor.c +index 63289283..1b7da383 100644 +--- a/src/daemon/modules/container/supervisor/supervisor.c ++++ b/src/daemon/modules/container/supervisor/supervisor.c +@@ -41,6 +41,8 @@ + #ifdef ENABLE_CRI_API_V1 + #include "sandbox_ops.h" + #endif ++#include "cgroup.h" ++#include "specs_api.h" + + pthread_mutex_t g_supervisor_lock = PTHREAD_MUTEX_INITIALIZER; + struct epoll_descr g_supervisor_descr; +@@ -269,24 +271,52 @@ static int supervisor_exit_cb(int fd, uint32_t events, void *cbdata, struct epol + return EPOLL_LOOP_HANDLE_CONTINUE; + } + ++static int oom_handle_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr) ++{ ++ cgroup_oom_handler_info_t *oom_handler_info = (cgroup_oom_handler_info_t *)cbdata; ++ bool close_oom_handler = CGROUP_OOM_HANDLE_CLOSE; ++ // supervisor only handle one oom event, so we remove the handler directly ++ if (oom_handler_info != NULL && oom_handler_info->oom_event_handler != NULL) { ++ close_oom_handler = oom_handler_info->oom_event_handler(fd, oom_handler_info); ++ } ++ ++ if (close_oom_handler == CGROUP_OOM_HANDLE_CLOSE) { ++ supervisor_handler_lock(); ++ epoll_loop_del_handler(&g_supervisor_descr, fd); ++ supervisor_handler_unlock(); ++ ++ common_free_cgroup_oom_handler_info(oom_handler_info); ++ } ++ ++ return EPOLL_LOOP_HANDLE_CONTINUE; ++} ++ + /* supervisor add exit monitor */ +-int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_info, const char *name, +- const char *runtime, bool sandbox_container) ++int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const pid_ppid_info_t *pid_info, const container_t *cont) + { + int ret = 0; + struct supervisor_handler_data *data = NULL; ++ cgroup_oom_handler_info_t *oom_handler_info = NULL; ++ __isula_auto_free char *cgroup_path = NULL; + + if (fd < 0) { + ERROR("Invalid exit fifo fd"); + return -1; + } + +- if (pid_info == NULL || name == NULL || runtime == NULL) { ++ if (pid_info == NULL || cont == NULL || cont->common_config == NULL) { + ERROR("Invalid input arguments"); + close(fd); + return -1; + } + ++ cgroup_path = merge_container_cgroups_path(cont->common_config->id, cont->hostconfig); ++ if (cgroup_path == NULL) { ++ ERROR("Failed to get cgroup path"); ++ close(fd); ++ return -1; ++ } ++ + data = util_common_calloc_s(sizeof(struct supervisor_handler_data)); + if (data == NULL) { + ERROR("Memory out"); +@@ -295,15 +325,26 @@ int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_inf + } + + data->fd = fd; +- data->name = util_strdup_s(name); +- data->runtime = util_strdup_s(runtime); +- data->is_sandbox_container = sandbox_container; ++ data->name = util_strdup_s(cont->common_config->id); ++ data->runtime = util_strdup_s(cont->runtime); ++#ifdef ENABLE_CRI_API_V1 ++ data->is_sandbox_container = is_sandbox_container(cont->common_config->sandbox_info); ++#endif + data->pid_info.pid = pid_info->pid; + data->pid_info.start_time = pid_info->start_time; + data->pid_info.ppid = pid_info->ppid; + data->pid_info.pstart_time = pid_info->pstart_time; ++ oom_handler_info = common_get_cgroup_oom_handler(fd, cont->common_config->id, cgroup_path, exit_fifo); + + supervisor_handler_lock(); ++ if (oom_handler_info != NULL) { ++ ret = epoll_loop_add_handler(&g_supervisor_descr, oom_handler_info->oom_event_fd, oom_handle_cb, oom_handler_info); ++ if (ret != 0) { ++ ERROR("Failed to add handler for oom event"); ++ goto err; ++ } ++ } ++ + ret = epoll_loop_add_handler(&g_supervisor_descr, fd, supervisor_exit_cb, data); + if (ret != 0) { + ERROR("Failed to add handler for exit fifo"); +@@ -314,6 +355,7 @@ int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_inf + + err: + supervisor_handler_data_free(data); ++ common_free_cgroup_oom_handler_info(oom_handler_info); + out: + supervisor_handler_unlock(); + return ret; +diff --git a/src/daemon/modules/events/collector.c b/src/daemon/modules/events/collector.c +index fb4a7fea..af688742 100644 +--- a/src/daemon/modules/events/collector.c ++++ b/src/daemon/modules/events/collector.c +@@ -133,6 +133,9 @@ static container_events_type_t lcrsta2Evetype(int value) + case THAWED: + et = EVENTS_TYPE_THAWED; + break; ++ case OOM: ++ et = EVENTS_TYPE_OOM; ++ break; + default: + et = EVENTS_TYPE_EXIT; + break; +@@ -822,8 +825,8 @@ static int post_event_to_events_hander(const struct isulad_events_format *events + return -1; + } + +- /* only post STOPPED event to events_hander */ +- if (events->type != EVENTS_TYPE_STOPPED1) { ++ /* only post STOPPED event and OOM event to events_hander */ ++ if (events->type != EVENTS_TYPE_STOPPED1 && events->type != EVENTS_TYPE_OOM) { + return 0; + } + +diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c +index a8090d5a..eb7ce4f4 100644 +--- a/src/daemon/modules/service/service_container.c ++++ b/src/daemon/modules/service/service_container.c +@@ -275,14 +275,13 @@ static void clean_resources_on_failure(const container_t *cont, const char *engi + return; + } + +-static int do_post_start_on_success(const char *id, const char *runtime, bool sandbox_container, +- const char *pidfile, int exit_fifo_fd, +- const pid_ppid_info_t *pid_info) ++static int do_post_start_on_success(container_t *cont, int exit_fifo_fd, ++ const char *exit_fifo, const pid_ppid_info_t *pid_info) + { + int ret = 0; + + // exit_fifo_fd was closed in container_supervisor_add_exit_monitor +- if (container_supervisor_add_exit_monitor(exit_fifo_fd, pid_info, id, runtime, sandbox_container)) { ++ if (container_supervisor_add_exit_monitor(exit_fifo_fd, exit_fifo, pid_info, cont)) { + ERROR("Failed to add exit monitor to supervisor"); + ret = -1; + } +@@ -750,7 +749,6 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo + oci_runtime_spec *oci_spec = NULL; + rt_create_params_t create_params = { 0 }; + rt_start_params_t start_params = { 0 }; +- bool sandbox_container = false; + + nret = snprintf(bundle, sizeof(bundle), "%s/%s", cont->root_path, id); + if (nret < 0 || (size_t)nret >= sizeof(bundle)) { +@@ -899,7 +897,6 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo + if (cont->common_config->sandbox_info != NULL) { + create_params.task_addr = cont->common_config->sandbox_info->task_address; + } +- sandbox_container = is_sandbox_container(cont->common_config->sandbox_info); + #endif + + if (runtime_create(id, runtime, &create_params) != 0) { +@@ -924,7 +921,7 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo + + ret = runtime_start(id, runtime, &start_params, pid_info); + if (ret == 0) { +- if (do_post_start_on_success(id, runtime, sandbox_container, pidfile, exit_fifo_fd, pid_info) != 0) { ++ if (do_post_start_on_success(cont, exit_fifo_fd, exit_fifo, pid_info) != 0) { + ERROR("Failed to do post start on runtime start success"); + ret = -1; + goto clean_resources; +-- +2.34.1 + diff --git a/0036-add-ci-cases-for-oomkilled-monitor.patch b/0036-add-ci-cases-for-oomkilled-monitor.patch new file mode 100644 index 0000000..8f4f67e --- /dev/null +++ b/0036-add-ci-cases-for-oomkilled-monitor.patch @@ -0,0 +1,279 @@ +From 0111a575f829b946068dcb11286f0d84363cfc3d Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Thu, 28 Mar 2024 12:51:53 +0000 +Subject: [PATCH 36/43] add ci cases for oomkilled monitor + +Signed-off-by: jikai <jikai11@huawei.com> +--- + CI/test_cases/container_cases/inspect.sh | 14 ++++++++ + test/cgroup/cpu/CMakeLists.txt | 2 ++ + .../image/oci/oci_config_merge/CMakeLists.txt | 1 + + test/image/oci/registry/CMakeLists.txt | 1 + + test/mocks/sender_mock.cc | 34 +++++++++++++++++++ + test/mocks/sender_mock.h | 31 +++++++++++++++++ + test/network/network_ns/CMakeLists.txt | 1 + + test/runtime/isula/CMakeLists.txt | 1 + + test/runtime/lcr/CMakeLists.txt | 1 + + test/specs/specs/CMakeLists.txt | 1 + + test/specs/specs_extend/CMakeLists.txt | 1 + + test/specs/verify/CMakeLists.txt | 1 + + test/volume/CMakeLists.txt | 3 +- + 13 files changed, 91 insertions(+), 1 deletion(-) + create mode 100644 test/mocks/sender_mock.cc + create mode 100644 test/mocks/sender_mock.h + +diff --git a/CI/test_cases/container_cases/inspect.sh b/CI/test_cases/container_cases/inspect.sh +index cde9ea1f..b4f4a785 100755 +--- a/CI/test_cases/container_cases/inspect.sh ++++ b/CI/test_cases/container_cases/inspect.sh +@@ -27,6 +27,7 @@ function test_inspect_spec() + { + local ret=0 + local image="busybox" ++ local ubuntu_image="ubuntu" + local test="container inspect test => (${FUNCNAME[@]})" + + msg_info "${test} starting..." +@@ -37,6 +38,12 @@ function test_inspect_spec() + isula images | grep busybox + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++)) + ++ isula pull ${ubuntu_image} ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${ubuntu_image}" && return ${FAILURE} ++ ++ isula images | grep ubuntu ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${ubuntu_image}" && ((ret++)) ++ + containername=test_inspect + + isula create --name $containername --ipc host --pid host --uts host --restart=on-failure:10 --hook-spec ${test_data_path}/test-hookspec.json --cpu-shares 100 --memory 5MB --memory-reservation 4MB --cpu-period 1000000 --cpu-quota 200000 --cpuset-cpus 1 --cpuset-mems 0 --kernel-memory 50M --pids-limit=10000 --volume /home:/root --env a=1 $image /bin/sh ls +@@ -139,6 +146,13 @@ function test_inspect_spec() + + isula rm -f $containername + ++ isula run -it -m 4m --name $containername $ubuntu_image perl -e 'for ($i = 0; $i < 100000000; $i++) { $a .= " " x 1024 }' ++ ++ isula inspect -f "{{json .State.OOMKilled}} {{.Name}}" $containername 2>&1 | sed -n '1p' | grep "true" ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container with image: ${ubuntu_image}" && ((ret++)) ++ ++ isula rm -f $containername ++ + msg_info "${test} finished with return ${ret}..." + return ${ret} + } +diff --git a/test/cgroup/cpu/CMakeLists.txt b/test/cgroup/cpu/CMakeLists.txt +index 30bfc417..9c3cfa12 100644 +--- a/test/cgroup/cpu/CMakeLists.txt ++++ b/test/cgroup/cpu/CMakeLists.txt +@@ -13,6 +13,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/command_parser.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/daemon_arguments.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/isulad_config.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc + cgroup_cpu_ut.cc) + + target_include_directories(${EXE} PUBLIC +@@ -23,6 +24,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/api + ${CMAKE_BINARY_DIR}/conf + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd +diff --git a/test/image/oci/oci_config_merge/CMakeLists.txt b/test/image/oci/oci_config_merge/CMakeLists.txt +index d13ec738..ffd3999d 100644 +--- a/test/image/oci/oci_config_merge/CMakeLists.txt ++++ b/test/image/oci/oci_config_merge/CMakeLists.txt +@@ -35,6 +35,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/containers_store_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/namespace_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/container_unix_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/sender_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/spec/parse_volume.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/spec/specs.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/spec/parse_volume.c +diff --git a/test/image/oci/registry/CMakeLists.txt b/test/image/oci/registry/CMakeLists.txt +index 5b5bc3f5..6166c2d0 100644 +--- a/test/image/oci/registry/CMakeLists.txt ++++ b/test/image/oci/registry/CMakeLists.txt +@@ -44,6 +44,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../mocks/storage_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../mocks/oci_image_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../mocks/http_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../mocks/sender_mock.cc + registry_ut.cc) + + target_include_directories(${EXE} PUBLIC +diff --git a/test/mocks/sender_mock.cc b/test/mocks/sender_mock.cc +new file mode 100644 +index 00000000..26028d7f +--- /dev/null ++++ b/test/mocks/sender_mock.cc +@@ -0,0 +1,34 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-29 ++ * Description: provide collector mock ++ ******************************************************************************/ ++ ++#include "sender_mock.h" ++ ++namespace { ++MockEventSender *g_sender_mock = nullptr; ++} ++ ++void MockEventSender_SetMock(MockEventSender *mock) ++{ ++ g_sender_mock = mock; ++} ++ ++int isulad_monitor_send_container_event(const char *name, runtime_state_t state, int pid, int exit_code, ++ const char *args, const char *extra_annations) ++{ ++ if (g_sender_mock != nullptr) { ++ return g_sender_mock->IsuladMonitorEventSendContainerEvent(name, state, pid, exit_code, args, extra_annations); ++ } ++ return 0; ++} +diff --git a/test/mocks/sender_mock.h b/test/mocks/sender_mock.h +new file mode 100644 +index 00000000..f4fe75f0 +--- /dev/null ++++ b/test/mocks/sender_mock.h +@@ -0,0 +1,31 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-30 ++ * Description: provide sender mock ++ ******************************************************************************/ ++ ++#ifndef _ISULAD_TEST_MOCKS_SENDER_MOCK_H ++#define _ISULAD_TEST_MOCKS_SENDER_MOCK_H ++ ++#include <gmock/gmock.h> ++#include "events_sender_api.h" ++ ++class MockEventSender { ++public: ++ MOCK_METHOD6(IsuladMonitorEventSendContainerEvent, int(const char *name, runtime_state_t state, int pid, int exit_code, ++ const char *args, const char *extra_annations)); ++}; ++ ++void MockEventSender_SetMock(MockEventSender *mock); ++ ++#endif ++ +diff --git a/test/network/network_ns/CMakeLists.txt b/test/network/network_ns/CMakeLists.txt +index 71b8039d..6f3f36a0 100644 +--- a/test/network/network_ns/CMakeLists.txt ++++ b/test/network/network_ns/CMakeLists.txt +@@ -43,6 +43,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/selinux_label_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/isulad_config_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc + network_ns_ut.cc) + + target_include_directories(${EXE} PUBLIC +diff --git a/test/runtime/isula/CMakeLists.txt b/test/runtime/isula/CMakeLists.txt +index c1f0a5cc..15636623 100644 +--- a/test/runtime/isula/CMakeLists.txt ++++ b/test/runtime/isula/CMakeLists.txt +@@ -31,6 +31,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/engine_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/isulad_config_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/runtime/isula/isula_rt_ops.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc + isula_rt_ops_ut.cc) + + target_include_directories(${EXE} PUBLIC +diff --git a/test/runtime/lcr/CMakeLists.txt b/test/runtime/lcr/CMakeLists.txt +index c3b93d67..5b2ed11a 100644 +--- a/test/runtime/lcr/CMakeLists.txt ++++ b/test/runtime/lcr/CMakeLists.txt +@@ -29,6 +29,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/namespace_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/container_unix_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/engine_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/isulad_config_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c + lcr_rt_ops_ut.cc) +diff --git a/test/specs/specs/CMakeLists.txt b/test/specs/specs/CMakeLists.txt +index 45f688f9..12c11f51 100644 +--- a/test/specs/specs/CMakeLists.txt ++++ b/test/specs/specs/CMakeLists.txt +@@ -43,6 +43,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/isulad_config_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/image_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc + specs_ut.cc) + + target_include_directories(${EXE} PUBLIC +diff --git a/test/specs/specs_extend/CMakeLists.txt b/test/specs/specs_extend/CMakeLists.txt +index 1b737089..2fd37e1c 100644 +--- a/test/specs/specs_extend/CMakeLists.txt ++++ b/test/specs/specs_extend/CMakeLists.txt +@@ -43,6 +43,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/isulad_config_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/image_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc + specs_extend_ut.cc) + + target_include_directories(${EXE} PUBLIC +diff --git a/test/specs/verify/CMakeLists.txt b/test/specs/verify/CMakeLists.txt +index b0602127..7f000cd1 100644 +--- a/test/specs/verify/CMakeLists.txt ++++ b/test/specs/verify/CMakeLists.txt +@@ -38,6 +38,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/image_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc + verify_ut.cc) + + target_include_directories(${EXE} PUBLIC +diff --git a/test/volume/CMakeLists.txt b/test/volume/CMakeLists.txt +index 27d07330..1f9dac03 100644 +--- a/test/volume/CMakeLists.txt ++++ b/test/volume/CMakeLists.txt +@@ -24,6 +24,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup/cgroup_v1.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup/cgroup_v2.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup/cgroup_common.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../test/mocks/sender_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/cmd/command_parser.c + volume_ut.cc) + +@@ -43,6 +44,6 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/utils/console + ) + +-target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) ++target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) + set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +-- +2.34.1 + diff --git a/0037-add-cgroup-v2-doc.patch b/0037-add-cgroup-v2-doc.patch new file mode 100644 index 0000000..bfd7530 --- /dev/null +++ b/0037-add-cgroup-v2-doc.patch @@ -0,0 +1,252 @@ +From 2cbce684b973bf4e250f41d750253b5b8abde32d Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Tue, 20 Feb 2024 19:22:11 +0800 +Subject: [PATCH 37/43] add cgroup v2 doc + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + docs/design/README_zh.md | 2 + + .../detailed/Container/cgroup_v2_design_zh.md | 193 ++++++++++++++++++ + docs/images/cgroup_v2_module.svg | 16 ++ + 3 files changed, 211 insertions(+) + create mode 100644 docs/design/detailed/Container/cgroup_v2_design_zh.md + create mode 100644 docs/images/cgroup_v2_module.svg + +diff --git a/docs/design/README_zh.md b/docs/design/README_zh.md +index f2c187a1..b7ec3ddb 100644 +--- a/docs/design/README_zh.md ++++ b/docs/design/README_zh.md +@@ -18,6 +18,8 @@ + + - 查看 restart 模块的设计文档： [restart_manager_design](./detailed/Container/restart_manager_design.md)。 + ++- 查看 cgroup v2 的设计文档： [cgroup_v2_design](./detailed/Container/cgroup_v2_design_zh.md)。 ++ + ## CRI + + - 查看 CRI的启动程序的重构文档： [cri_cni_refactor](./detailed/CRI/cri_cni_refactor_zh.md) 。 +diff --git a/docs/design/detailed/Container/cgroup_v2_design_zh.md b/docs/design/detailed/Container/cgroup_v2_design_zh.md +new file mode 100644 +index 00000000..e1ce81d0 +--- /dev/null ++++ b/docs/design/detailed/Container/cgroup_v2_design_zh.md +@@ -0,0 +1,193 @@ ++| Author | zhongtao | ++| ------ | --------------------- | ++| Date | 2024-02-19 | ++| Email | zhongtao17@huawei.com | ++# 方案目标 ++ ++cgroup是linux中用于限制进程组资源的功能。cgroup目前包括两个版本，cgroup v1和cgroup v2。cgroup v2的目标是取代cgroup v1，出于兼容性的考虑，cgroup v1并没有在内核中删除，并且大概率会长期存在。该需求的目的为使得iSulad支持cgroup v2. ++ ++## 与cgroup v1差异 ++无论是cgroup v1还是cgroup v2，iSulad提供给用户使用的接口都是一致的。不过由于有部分cgroup v1支持的功能在cgroup v2中被去掉了或者实现方式有所变化，因此部分接口在cgroup v2中不可用或者含义发生变化。iSulad支持限制如下资源： ++ ++|资源|功能|和cgroup v1的差异| ++|---|---|---| ++|devices|限制对应的设备是否可以在容器中访问以及访问权限|devcies子系统不再使用往cgroup文件里写值的方式进行限制，而是采用ebpf的方式进行限制| ++|memory|限制容器的内存资源|不支持swappiness，不支持kmem相关参数，不支持oom_control| ++|cpu/cpuset|限制容器的cpu资源|不支持rt_*相关(实时线程)的限制| ++|blkio/io|限制容器的块设备io|不仅限制块设备的IO，也能限制buffer IO| ++|hugetlb|限制大页内存的使用|无差异| ++|pids|限制容器使用的pid|无差异| ++|files|限制容器使用的fd|无差异| ++|freeze|暂停容器|无差异| ++ ++## 使用方式 ++ ++使用的示例如下： ++ ++1. 以限制内存资源为例，假设我们需要限制单个容器最多使用10M内存，则可以在运行容器时加上-m 10m参数进行限制： ++ ++ ```sh ++ [root@openEuler iSulad]# isula run -tid -m 10m busybox sh ++ 000c0c6eb609179062b19a3d2de4d7c38a42c887f55e2a7759ed9df851277163 ++ ``` ++ ++ -m 10m表示限制容器内最多只能使用10m内存，可以通过isula stats命令查看资源的限制情况： ++ ++ ```sh ++ [root@openEuler iSulad]# isula stats --no-stream 000c0c6eb6 ++ CONTAINER CPU % MEM USAGE / LIMIT MEM % BLOCK I / O PIDS ++ 000c0c6eb609 0.00 104.00 KiB / 10.00 MiB 1.02 0.00 B / 0.00 B 1 ++ ``` ++ ++ 可以动态更新资源的限制： ++ ```sh ++ [root@openEuler iSulad]# isula update -m 20m 000c0c6eb6 ++ 000c0c6eb6 ++ [root@openEuler iSulad]# isula stats --no-stream 000c0c6eb6 ++ CONTAINER CPU % MEM USAGE / LIMIT MEM % BLOCK I / O PIDS ++ 000c0c6eb609 0.00 104.00 KiB / 20.00 MiB 0.51 0.00 B / 0.00 B 1 ++ ``` ++ ++2. 假设我们要将设备/dev/sda挂载到容器中成为/dev/sdx并限制为只读设备，则可以这么配置： ++ ++```sh ++ [root@openEuler iSulad]# isula run -ti --rm --device=/dev/sda:/dev/sdx:wm busybox fdisk /dev/sdx ++ fdisk: can't open '/dev/sdx' ++ [root@openEuler iSulad]# ++``` ++ ++挂载设备到容器的语法为`--device=$host:$container:rwm $host`指定设备在主机上的绝对路径，$container指定设备在容器内的绝对路径，r表示可读，w表示可写，m表示可以创建node 上述命令中rwm三个参数缺少r参数，也就是说允许写和创建node但是不允许读(即只读)。 ++ ++3. 使用cri的PodSandboxStats接口与ContainerStats接口获取容器的资源使用状况： ++ ++```sh ++[root@openEuler ~]# crictl statsp c3 ++ POD POD ID CPU % MEM ++ test-sandbox c32556d3bb139 0.00 196.6kB ++[root@openEuler ~]# crictl statsp --output json c3 ++...... ++ "linux": { ++ "cpu": { ++ "timestamp": "1708499622485777700", ++ "usageCoreNanoSeconds": { ++ "value": "180973" ++ }, ++ "usageNanoCores": null ++ }, ++ "memory": { ++ "timestamp": "1708499622485777700", ++ "workingSetBytes": { ++ "value": "196608" ++ }, ++ "availableBytes": { ++ "value": "0" ++ }, ++ "usageBytes": { ++ "value": "4386816" ++ }, ++ "rssBytes": { ++ "value": "176128" ++ }, ++ "pageFaults": { ++ "value": "1193" ++ }, ++ "majorPageFaults": { ++ "value": "6" ++ } ++ }, ++ "network": null, ++ "process": { ++ "timestamp": "1708499622485777700", ++ "processCount": { ++ "value": "2" ++ } ++ }, ++..... ++ ++[root@openEuler ~]# crictl stats 01 ++CONTAINER CPU % MEM DISK INODES ++01a726f61c5c3 0.01 3.801MB 16.4kB 8 ++[root@openEuler ~]# ++``` ++ ++# 总体设计 ++ ++在原有只支持cgroup v1的基础上，对cgroup模块进行了重构，重构后的架构图如下： ++ ++ ++ ++主要功能为以下两种： ++ ++1. iSulad在资源控制参数设置和更新过程中，负责参数的合法校验，用于拦截非法请求，真正的cgroup操作由容器运行时完成。 ++ ++2. iSulad在获得sandbox资源使用信息时，直接读取sandbox cgroup文件信息。 ++ ++# 接口描述 ++由于无论是cgroup v1还是cgroup v2，iSulad提供给用户使用的接口都是一致的。 ++无新增接口。 ++ ++```c ++int verify_container_settings(const oci_runtime_spec *container, const sysinfo_t *sysinfo); ++ ++int verify_host_config_settings(host_config *hostconfig, const sysinfo_t *sysinfo, bool update); ++ ++ ++typedef struct { ++ int (*get_cgroup_info)(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, ++ cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, ++ cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, ++ cgroup_files_info_t *filesinfo, bool quiet); ++ int (*get_cgroup_metrics)(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics); ++ ++ int (*common_find_cgroup_mnt_and_root)(const char *subsystem, char **mountpoint, char **root); ++ ++ char *(*sysinfo_cgroup_controller_cpurt_mnt_path)(void); ++} cgroup_ops; ++ ++int cgroup_v2_ops_init(cgroup_ops *ops) ++{ ++ if (ops == NULL) { ++ return -1; ++ } ++ ops->get_cgroup_info = common_get_cgroup_info_v2; ++ ops->get_cgroup_metrics = common_get_cgroup_v2_metrics; ++ ops->common_find_cgroup_mnt_and_root = common_find_cgroup_v2_mnt_and_root; ++ return 0; ++} ++``` ++ ++# 详细设计 ++ ++```mermaid ++sequenceDiagram ++ participant isula ++ participant kubelet ++ participant isulad ++ participant runc ++ participant cgroup ++ ++ isula->>isulad: request ++ kubelet->>isulad:request ++ alt run/create/update ++ isulad->>isulad:verify request option ++ isulad->>runc:run/create/update request ++ runc ->> cgroup:write cgroup file ++ else stats ++ par container stats ++ isulad->>runc:container stats request ++ runc ->> cgroup:read cgroup file ++ runc ->> isulad:container stats info ++ and sandbox stats ++ isulad ->> cgroup: read cgroup file ++ end ++ end ++ isulad ->> isula:response ++ isulad ->> kubelet:response ++``` ++ ++# 使用限制 ++ ++1. 只支持cgroup 挂载点在/sys/fs/cgroup ++2. cgroup v1与cgrooup v2混用场景不支持 ++3. 该需求只涉及cgroup v2对runc容器运行时的支持 ++ +diff --git a/docs/images/cgroup_v2_module.svg b/docs/images/cgroup_v2_module.svg +new file mode 100644 +index 00000000..59e7939b +--- /dev/null ++++ b/docs/images/cgroup_v2_module.svg +@@ -0,0 +1,16 @@ ++<svg version="1.1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1060.0000000000007 754.5203469732791" width="1060.0000000000007" height="754.5203469732791"> ++ <!-- svg-source:excalidraw --> ++ ++ <defs> ++ <style class="style-fonts"> ++ @font-face { ++ font-family: "Virgil"; ++ src: url("https://excalidraw.com/Virgil.woff2"); ++ } ++ @font-face { ++ font-family: "Cascadia"; ++ src: url("https://excalidraw.com/Cascadia.woff2"); ++ } ++ </style> ++ </defs> ++ <rect x="0" y="0" width="1060.0000000000007" height="754.5203469732791" fill="#ffffff"></rect><g stroke-linecap="round" transform="translate(374.51277982271654 10) rotate(0 98 27)"><path d="M0 0 C76.18 0, 152.36 0, 196 0 M0 0 C53.42 0, 106.84 0, 196 0 M196 0 C196 10.9, 196 21.81, 196 54 M196 0 C196 13.47, 196 26.95, 196 54 M196 54 C120.92 54, 45.83 54, 0 54 M196 54 C128.59 54, 61.18 54, 0 54 M0 54 C0 35.38, 0 16.75, 0 0 M0 54 C0 42.69, 0 31.39, 0 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(456.95125638521654 25.5) rotate(0 15.5615234375 11.5)"><text x="15.5615234375" y="0" font-family="Helvetica, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">CLI</text></g><g stroke-linecap="round" transform="translate(644.5127798227165 11) rotate(0 98 27)"><path d="M0 0 C41.14 0, 82.28 0, 196 0 M0 0 C77.31 0, 154.62 0, 196 0 M196 0 C196 11.1, 196 22.21, 196 54 M196 0 C196 10.93, 196 21.86, 196 54 M196 54 C135.62 54, 75.24 54, 0 54 M196 54 C135.82 54, 75.63 54, 0 54 M0 54 C0 39.1, 0 24.21, 0 0 M0 54 C0 32.96, 0 11.92, 0 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(725.2911001352165 26.5) rotate(0 17.2216796875 11.5)"><text x="17.2216796875" y="0" font-family="Helvetica, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">CRI</text></g><g stroke-linecap="round"><g transform="translate(481.84615384615404 66.66668701171875) rotate(0 0 0)"><path d="M0 0 C0 0, 0 0, 0 0 M0 0 C0 0, 0 0, 0 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(481.84615384615404 66.66668701171875) rotate(0 0 0)"><path d="MNaN NaN CNaN NaN, NaN NaN, NaN NaN MNaN NaN CNaN NaN, NaN NaN, NaN NaN" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(481.84615384615404 66.66668701171875) rotate(0 0 0)"><path d="MNaN NaN CNaN NaN, NaN NaN, NaN NaN MNaN NaN CNaN NaN, NaN NaN, NaN NaN" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round"><g transform="translate(474.4873658176107 65) rotate(0 -0.1998750712791164 41.22263106766279)"><path d="M0 0 C-0.15 30.41, -0.29 60.82, -0.4 82.45 M0 0 C-0.09 19.52, -0.19 39.04, -0.4 82.45" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(474.4873658176107 65) rotate(0 -0.1998750712791164 41.22263106766279)"><path d="M-10.52 54.21 C-6.79 64.62, -3.06 75.04, -0.4 82.45 M-10.52 54.21 C-8.13 60.89, -5.73 67.58, -0.4 82.45" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(474.4873658176107 65) rotate(0 -0.1998750712791164 41.22263106766279)"><path d="M10 54.3 C6.16 64.68, 2.33 75.06, -0.4 82.45 M10 54.3 C7.54 60.97, 5.07 67.63, -0.4 82.45" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round"><g transform="translate(731.846153846154 66.66668701171875) rotate(0 0.33331298828125 39.666656494140625)"><path d="M0 0 C0.21 25.34, 0.43 50.69, 0.67 79.33 M0 0 C0.15 18.11, 0.3 36.21, 0.67 79.33" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(731.846153846154 66.66668701171875) rotate(0 0.33331298828125 39.666656494140625)"><path d="M-9.83 51.23 C-6.48 60.21, -3.12 69.18, 0.67 79.33 M-9.83 51.23 C-7.43 57.64, -5.04 64.06, 0.67 79.33" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(731.846153846154 66.66668701171875) rotate(0 0.33331298828125 39.666656494140625)"><path d="M10.69 51.06 C7.49 60.09, 4.29 69.12, 0.67 79.33 M10.69 51.06 C8.4 57.51, 6.11 63.96, 0.67 79.33" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(623.846153846154 147.83334350585938) rotate(0 114 26.5)"><path d="M0 0 C65.21 0, 130.41 0, 228 0 M0 0 C55.16 0, 110.32 0, 228 0 M228 0 C228 11.04, 228 22.08, 228 53 M228 0 C228 18.15, 228 36.3, 228 53 M228 53 C142.95 53, 57.9 53, 0 53 M228 53 C141.49 53, 54.97 53, 0 53 M0 53 C0 38.91, 0 24.82, 0 0 M0 53 C0 37.05, 0 21.09, 0 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(685.048302283654 162.83334350585938) rotate(0 52.7978515625 11.5)"><text x="52.7978515625" y="0" font-family="Helvetica, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">CRI module</text></g><g stroke-linecap="round" transform="translate(368.84615384615404 244.00006103515625) rotate(0 249 29)"><path d="M0 0 C0 0, 0 0, 0 0 M0 0 C0 0, 0 0, 0 0 M-0.26 6.4 C1.23 4.69, 2.71 2.98, 4.99 0.36 M-0.26 6.4 C1.06 4.88, 2.38 3.36, 4.99 0.36 M0.13 12.04 C2.65 9.15, 5.17 6.25, 10.63 -0.03 M0.13 12.04 C4.18 7.38, 8.23 2.73, 10.63 -0.03 M-0.13 18.44 C5.78 11.65, 11.68 4.85, 15.62 0.33 M-0.13 18.44 C4.65 12.94, 9.43 7.45, 15.62 0.33 M0.27 24.08 C6.61 16.78, 12.95 9.49, 21.26 -0.07 M0.27 24.08 C7.35 15.93, 14.44 7.78, 21.26 -0.07 M0 30.48 C10.35 18.58, 20.7 6.67, 26.25 0.29 M0 30.48 C7.06 22.36, 14.12 14.24, 26.25 0.29 M-0.26 36.88 C10.79 24.17, 21.84 11.46, 31.89 -0.1 M-0.26 36.88 C6.72 28.85, 13.7 20.82, 31.89 -0.1 M0.14 42.52 C8.28 33.16, 16.42 23.79, 36.88 0.26 M0.14 42.52 C8.6 32.78, 17.07 23.04, 36.88 0.26 M-0.12 48.92 C10 37.27, 20.13 25.62, 42.52 -0.14 M-0.12 48.92 C15.94 30.44, 32.01 11.95, 42.52 -0.14 M0.27 54.56 C19.08 32.92, 37.89 11.29, 47.51 0.22 M0.27 54.56 C16.07 36.39, 31.87 18.21, 47.51 0.22 M1.32 59.45 C15.2 43.48, 29.08 27.52, 53.15 -0.17 M1.32 59.45 C15.97 42.6, 30.61 25.76, 53.15 -0.17 M6.96 59.05 C18.55 45.73, 30.13 32.4, 58.14 0.19 M6.96 59.05 C21.89 41.88, 36.82 24.71, 58.14 0.19 M11.95 59.42 C23.87 45.71, 35.78 32, 63.78 -0.21 M11.95 59.42 C23.08 46.61, 34.21 33.81, 63.78 -0.21 M17.59 59.02 C31.37 43.17, 45.15 27.32, 68.77 0.15 M17.59 59.02 C31.77 42.72, 45.94 26.41, 68.77 0.15 M22.58 59.38 C39.68 39.72, 56.77 20.05, 74.41 -0.24 M22.58 59.38 C41.62 37.47, 60.67 15.57, 74.41 -0.24 M28.23 58.99 C41.06 44.22, 53.9 29.45, 79.4 0.12 M28.23 58.99 C39.94 45.51, 51.66 32.03, 79.4 0.12 M33.21 59.35 C47.97 42.37, 62.73 25.39, 85.04 -0.28 M33.21 59.35 C51.3 38.53, 69.4 17.72, 85.04 -0.28 M38.86 58.95 C57.09 37.97, 75.33 17, 90.03 0.08 M38.86 58.95 C56.79 38.32, 74.73 17.68, 90.03 0.08 M43.84 59.31 C55.38 46.04, 66.92 32.77, 95.67 -0.31 M43.84 59.31 C58.46 42.5, 73.08 25.68, 95.67 -0.31 M49.49 58.92 C66.46 39.39, 83.43 19.87, 100.66 0.05 M49.49 58.92 C60.16 46.63, 70.84 34.35, 100.66 0.05 M54.47 59.28 C74.29 36.48, 94.11 13.68, 106.3 -0.34 M54.47 59.28 C70.16 41.23, 85.86 23.17, 106.3 -0.34 M59.46 59.64 C79.34 36.76, 99.23 13.89, 111.29 0.02 M59.46 59.64 C71.6 45.67, 83.75 31.7, 111.29 0.02 M65.1 59.24 C76.05 46.65, 86.99 34.06, 116.27 0.38 M65.1 59.24 C78.5 43.83, 91.89 28.42, 116.27 0.38 M70.09 59.6 C84.61 42.9, 99.13 26.19, 121.92 -0.02 M70.09 59.6 C81.62 46.34, 93.15 33.07, 121.92 -0.02 M75.73 59.21 C89.96 42.85, 104.18 26.48, 126.91 0.34 M75.73 59.21 C87.18 46.04, 98.62 32.88, 126.91 0.34 M80.72 59.57 C93.68 44.66, 106.64 29.75, 132.55 -0.05 M80.72 59.57 C92.76 45.71, 104.81 31.86, 132.55 -0.05 M86.36 59.17 C104.09 38.78, 121.81 18.39, 137.54 0.31 M86.36 59.17 C105.14 37.57, 123.92 15.97, 137.54 0.31 M91.35 59.53 C106.02 42.66, 120.69 25.79, 143.18 -0.09 M91.35 59.53 C106.23 42.42, 121.11 25.3, 143.18 -0.09 M96.99 59.14 C112.44 41.37, 127.88 23.61, 148.17 0.27 M96.99 59.14 C111.55 42.39, 126.11 25.64, 148.17 0.27 M101.98 59.5 C113.88 45.81, 125.79 32.11, 153.81 -0.12 M101.98 59.5 C116.17 43.17, 130.36 26.85, 153.81 -0.12 M107.62 59.1 C119.8 45.1, 131.97 31.09, 158.8 0.24 M107.62 59.1 C122.1 42.45, 136.58 25.79, 158.8 0.24 M112.61 59.46 C128.24 41.48, 143.88 23.49, 164.44 -0.16 M112.61 59.46 C125.25 44.92, 137.89 30.38, 164.44 -0.16 M118.25 59.07 C136.64 37.92, 155.02 16.78, 169.43 0.2 M118.25 59.07 C132.75 42.4, 147.24 25.72, 169.43 0.2 M123.24 59.43 C142.49 37.28, 161.75 15.13, 175.07 -0.19 M123.24 59.43 C139.45 40.78, 155.66 22.14, 175.07 -0.19 M128.88 59.04 C143.28 42.48, 157.67 25.92, 180.06 0.17 M128.88 59.04 C145.27 40.19, 161.65 21.34, 180.06 0.17 M133.87 59.4 C148 43.14, 162.13 26.88, 185.7 -0.23 M133.87 59.4 C152.51 37.95, 171.15 16.5, 185.7 -0.23 M139.51 59 C156.42 39.55, 173.33 20.1, 190.69 0.13 M139.51 59 C151.93 44.71, 164.35 30.43, 190.69 0.13 M144.5 59.36 C160.02 41.5, 175.55 23.64, 196.33 -0.26 M144.5 59.36 C161.51 39.79, 178.53 20.22, 196.33 -0.26 M150.14 58.97 C162.64 44.59, 175.13 30.22, 201.32 0.1 M150.14 58.97 C161.09 46.38, 172.03 33.79, 201.32 0.1 M155.13 59.33 C171.07 40.99, 187.01 22.66, 206.96 -0.3 M155.13 59.33 C172.03 39.89, 188.92 20.46, 206.96 -0.3 M160.77 58.93 C174.63 42.99, 188.49 27.05, 211.95 0.06 M160.77 58.93 C171.56 46.53, 182.34 34.12, 211.95 0.06 M165.76 59.29 C176.79 46.6, 187.82 33.91, 217.59 -0.33 M165.76 59.29 C182.58 39.95, 199.39 20.6, 217.59 -0.33 M170.75 59.65 C184.71 43.59, 198.68 27.52, 222.58 0.03 M170.75 59.65 C187.56 40.31, 204.38 20.97, 222.58 0.03 M176.39 59.26 C191.56 41.8, 206.74 24.35, 228.22 -0.36 M176.39 59.26 C189.83 43.79, 203.28 28.33, 228.22 -0.36 M181.38 59.62 C192.29 47.07, 203.19 34.52, 233.21 0 M181.38 59.62 C194.86 44.11, 208.35 28.59, 233.21 0 M187.02 59.22 C201.63 42.42, 216.23 25.62, 238.19 0.36 M187.02 59.22 C203.93 39.77, 220.84 20.32, 238.19 0.36 M192.01 59.58 C206.25 43.19, 220.5 26.81, 243.84 -0.04 M192.01 59.58 C204.3 45.44, 216.6 31.29, 243.84 -0.04 M197.65 59.19 C212.49 42.12, 227.32 25.06, 248.82 0.32 M197.65 59.19 C213.08 41.44, 228.51 23.69, 248.82 0.32 M202.64 59.55 C221.03 38.4, 239.41 17.24, 254.47 -0.07 M202.64 59.55 C217.61 42.32, 232.59 25.1, 254.47 -0.07 M208.28 59.15 C221.91 43.48, 235.53 27.8, 259.45 0.29 M208.28 59.15 C218.53 47.36, 228.79 35.57, 259.45 0.29 M213.27 59.51 C231.57 38.46, 249.88 17.4, 265.1 -0.11 M213.27 59.51 C232.06 37.9, 250.85 16.28, 265.1 -0.11 M218.91 59.12 C236.08 39.37, 253.25 19.62, 270.08 0.25 M218.91 59.12 C230.11 46.23, 241.32 33.34, 270.08 0.25 M223.9 59.48 C238.62 42.55, 253.34 25.61, 275.73 -0.14 M223.9 59.48 C235.29 46.38, 246.68 33.27, 275.73 -0.14 M229.54 59.09 C243.84 42.63, 258.14 26.18, 280.71 0.22 M229.54 59.09 C240.45 46.53, 251.36 33.98, 280.71 0.22 M234.53 59.45 C253.82 37.26, 273.1 15.07, 286.36 -0.18 M234.53 59.45 C251.94 39.42, 269.35 19.39, 286.36 -0.18 M240.17 59.05 C252.51 44.86, 264.84 30.67, 291.34 0.18 M240.17 59.05 C250.57 47.08, 260.98 35.12, 291.34 0.18 M245.16 59.41 C263.48 38.34, 281.79 17.27, 296.99 -0.21 M245.16 59.41 C255.55 47.45, 265.95 35.49, 296.99 -0.21 M250.8 59.02 C262.26 45.83, 273.73 32.64, 301.97 0.15 M250.8 59.02 C264.78 42.93, 278.76 26.85, 301.97 0.15 M255.79 59.38 C275.2 37.04, 294.62 14.71, 307.62 -0.25 M255.79 59.38 C267.06 46.41, 278.33 33.44, 307.62 -0.25 M261.43 58.98 C281.55 35.84, 301.66 12.7, 312.6 0.11 M261.43 58.98 C275.31 43.02, 289.18 27.05, 312.6 0.11 M266.42 59.34 C281.62 41.86, 296.82 24.37, 318.25 -0.28 M266.42 59.34 C277 47.16, 287.59 34.99, 318.25 -0.28 M272.06 58.95 C291.2 36.93, 310.33 14.92, 323.23 0.08 M272.06 58.95 C287.89 40.73, 303.73 22.52, 323.23 0.08 M277.05 59.31 C296.65 36.76, 316.25 14.21, 328.88 -0.31 M277.05 59.31 C292.22 41.86, 307.39 24.41, 328.88 -0.31 M282.69 58.91 C300.76 38.12, 318.83 17.34, 333.86 0.05 M282.69 58.91 C296.21 43.36, 309.73 27.81, 333.86 0.05 M287.68 59.27 C298.51 46.82, 309.33 34.36, 339.51 -0.35 M287.68 59.27 C299.53 45.64, 311.37 32.01, 339.51 -0.35 M292.67 59.63 C312.27 37.08, 331.87 14.53, 344.49 0.01 M292.67 59.63 C306.96 43.18, 321.26 26.74, 344.49 0.01 M298.31 59.24 C317.54 37.11, 336.77 14.99, 349.48 0.37 M298.31 59.24 C312.22 43.23, 326.13 27.23, 349.48 0.37 M303.3 59.6 C316.65 44.23, 330.01 28.87, 355.12 -0.02 M303.3 59.6 C316.2 44.76, 329.1 29.92, 355.12 -0.02 M308.94 59.2 C328.41 36.8, 347.88 14.41, 360.11 0.34 M308.94 59.2 C324.11 41.75, 339.28 24.3, 360.11 0.34 M313.93 59.56 C327.45 44, 340.98 28.45, 365.75 -0.06 M313.93 59.56 C332.16 38.59, 350.39 17.62, 365.75 -0.06 M319.57 59.17 C336.72 39.43, 353.88 19.7, 370.74 0.3 M319.57 59.17 C334.43 42.07, 349.29 24.98, 370.74 0.3 M324.56 59.53 C335.87 46.51, 347.19 33.49, 376.38 -0.09 M324.56 59.53 C340.98 40.63, 357.41 21.73, 376.38 -0.09 M330.2 59.13 C343.68 43.63, 357.16 28.12, 381.37 0.27 M330.2 59.13 C343.28 44.08, 356.37 29.03, 381.37 0.27 M335.19 59.5 C346.65 46.31, 358.11 33.13, 387.01 -0.13 M335.19 59.5 C352.87 39.16, 370.55 18.82, 387.01 -0.13 M340.83 59.1 C361.26 35.6, 381.68 12.1, 392 0.23 M340.83 59.1 C356.62 40.93, 372.41 22.77, 392 0.23 M345.82 59.46 C362.48 40.29, 379.15 21.12, 397.64 -0.16 M345.82 59.46 C365.77 36.51, 385.72 13.56, 397.64 -0.16 M351.46 59.07 C366.19 42.12, 380.92 25.18, 402.63 0.2 M351.46 59.07 C370.86 36.74, 390.27 14.42, 402.63 0.2 M356.45 59.43 C368.51 45.55, 380.57 31.67, 408.27 -0.2 M356.45 59.43 C370.6 43.14, 384.76 26.86, 408.27 -0.2 M362.09 59.03 C376.7 42.23, 391.3 25.43, 413.26 0.16 M362.09 59.03 C378.05 40.67, 394 22.32, 413.26 0.16 M367.08 59.39 C378.94 45.75, 390.8 32.1, 418.9 -0.23 M367.08 59.39 C387.52 35.87, 407.97 12.35, 418.9 -0.23 M372.72 59 C385.98 43.75, 399.24 28.49, 423.89 0.13 M372.72 59 C391.39 37.52, 410.05 16.05, 423.89 0.13 M377.71 59.36 C390.4 44.76, 403.09 30.16, 429.53 -0.26 M377.71 59.36 C388.17 47.32, 398.63 35.29, 429.53 -0.26 M383.35 58.96 C398.02 42.08, 412.7 25.2, 434.52 0.1 M383.35 58.96 C399.24 40.69, 415.12 22.41, 434.52 0.1 M388.34 59.32 C407.86 36.86, 427.39 14.39, 440.16 -0.3 M388.34 59.32 C404.51 40.71, 420.69 22.1, 440.16 -0.3 M393.98 58.93 C412.27 37.89, 430.56 16.84, 445.15 0.06 M393.98 58.93 C411.14 39.19, 428.3 19.45, 445.15 0.06 M398.97 59.29 C418.74 36.54, 438.51 13.8, 450.79 -0.33 M398.97 59.29 C409.66 46.99, 420.35 34.69, 450.79 -0.33 M403.95 59.65 C418 43.49, 432.04 27.34, 455.78 0.03 M403.95 59.65 C421.57 39.39, 439.18 19.13, 455.78 0.03 M409.6 59.25 C429.13 36.79, 448.66 14.32, 461.42 -0.37 M409.6 59.25 C426.67 39.61, 443.75 19.96, 461.42 -0.37 M414.58 59.61 C432.11 39.45, 449.64 19.29, 466.41 -0.01 M414.58 59.61 C426.95 45.39, 439.32 31.16, 466.41 -0.01 M420.23 59.22 C437.71 39.11, 455.19 19, 471.4 0.35 M420.23 59.22 C440.22 36.22, 460.22 13.22, 471.4 0.35 M425.21 59.58 C439.77 42.84, 454.32 26.09, 477.04 -0.04 M425.21 59.58 C438.7 44.06, 452.19 28.54, 477.04 -0.04 M430.86 59.18 C441.89 46.49, 452.92 33.8, 482.03 0.32 M430.86 59.18 C444.33 43.69, 457.79 28.2, 482.03 0.32 M435.84 59.54 C453.03 39.78, 470.21 20.01, 487.67 -0.08 M435.84 59.54 C448.67 44.78, 461.5 30.03, 487.67 -0.08 M441.49 59.15 C461.92 35.64, 482.36 12.13, 492.66 0.28 M441.49 59.15 C456.03 42.42, 470.57 25.7, 492.66 0.28 M446.47 59.51 C463.79 39.59, 481.1 19.68, 498.3 -0.11 M446.47 59.51 C464.99 38.21, 483.5 16.92, 498.3 -0.11 M452.12 59.12 C463.81 45.67, 475.49 32.22, 498.04 6.29 M452.12 59.12 C468.41 40.37, 484.71 21.62, 498.04 6.29 M457.1 59.48 C467.14 47.93, 477.18 36.38, 498.44 11.93 M457.1 59.48 C471.98 42.36, 486.86 25.24, 498.44 11.93 M462.75 59.08 C469.96 50.78, 477.18 42.48, 498.17 18.33 M462.75 59.08 C471.27 49.27, 479.8 39.46, 498.17 18.33 M467.73 59.44 C474.48 51.68, 481.22 43.93, 497.91 24.72 M467.73 59.44 C478.02 47.61, 488.31 35.77, 497.91 24.72 M473.38 59.05 C479.12 52.45, 484.85 45.84, 498.31 30.37 M473.38 59.05 C480.13 51.27, 486.89 43.5, 498.31 30.37 M478.36 59.41 C485.3 51.43, 492.23 43.45, 498.05 36.77 M478.36 59.41 C486.21 50.38, 494.06 41.35, 498.05 36.77 M484.01 59.01 C489.47 52.73, 494.93 46.44, 498.44 42.41 M484.01 59.01 C487.46 55.04, 490.92 51.06, 498.44 42.41 M488.99 59.37 C491.9 56.02, 494.81 52.68, 498.18 48.81 M488.99 59.37 C492.18 55.71, 495.36 52.05, 498.18 48.81 M494.64 58.98 C495.47 58.02, 496.3 57.06, 497.92 55.2 M494.64 58.98 C495.63 57.83, 496.63 56.69, 497.92 55.2" stroke="#b2f2bb" stroke-width="0.5" fill="none"></path><path d="M0 0 C128.04 0, 256.08 0, 498 0 M0 0 C193.36 0, 386.71 0, 498 0 M498 0 C498 15.46, 498 30.92, 498 58 M498 0 C498 16.09, 498 32.18, 498 58 M498 58 C391.86 58, 285.72 58, 0 58 M498 58 C384.43 58, 270.86 58, 0 58 M0 58 C0 46.4, 0 34.79, 0 0 M0 58 C0 35.14, 0 12.29, 0 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(558.378380408654 261.50006103515625) rotate(0 59.4677734375 11.5)"><text x="59.4677734375" y="0" font-family="Helvetica, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">verify module</text></g><g stroke-linecap="round"><g transform="translate(731.1794668344353 200.66668701171875) rotate(0 -0.6252581550315597 20.66668701171875)"><path d="M0 0 C-0.26 8.47, -0.51 16.94, -1.25 41.33 M0 0 C-0.41 13.51, -0.82 27.03, -1.25 41.33" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(731.1794668344353 200.66668701171875) rotate(0 -0.6252581550315597 20.66668701171875)"><path d="M-7.73 21.7 C-6.4 25.72, -5.08 29.74, -1.25 41.33 M-7.73 21.7 C-5.61 28.12, -3.49 34.54, -1.25 41.33" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(731.1794668344353 200.66668701171875) rotate(0 -0.6252581550315597 20.66668701171875)"><path d="M6.41 22.13 C4.84 26.06, 3.27 30, -1.25 41.33 M6.41 22.13 C3.9 28.41, 1.4 34.68, -1.25 41.33" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(71.64100529597363 395.3846670297477) rotate(0 284.48724130483777 93.35896888146038)"><path d="M0 0 C0 0, 0 0, 0 0 M0 0 C0 0, 0 0, 0 0 M-0.26 6.4 C1.58 4.28, 3.42 2.16, 4.99 0.36 M-0.26 6.4 C1.6 4.26, 3.45 2.12, 4.99 0.36 M0.13 12.04 C2.5 9.32, 4.86 6.6, 10.63 -0.03 M0.13 12.04 C3.79 7.83, 7.45 3.62, 10.63 -0.03 M-0.13 18.44 C3.77 13.95, 7.67 9.46, 15.62 0.33 M-0.13 18.44 C5.26 12.24, 10.66 6.03, 15.62 0.33 M0.27 24.08 C7.63 15.61, 14.99 7.15, 21.26 -0.07 M0.27 24.08 C5.42 18.15, 10.58 12.22, 21.26 -0.07 M0 30.48 C7.43 21.94, 14.86 13.39, 26.25 0.29 M0 30.48 C6.83 22.63, 13.65 14.78, 26.25 0.29 M-0.26 36.88 C11.62 23.22, 23.49 9.56, 31.89 -0.1 M-0.26 36.88 C11.63 23.21, 23.51 9.53, 31.89 -0.1 M0.14 42.52 C13.72 26.9, 27.3 11.28, 36.88 0.26 M0.14 42.52 C12.31 28.52, 24.49 14.51, 36.88 0.26 M-0.12 48.92 C10.02 37.25, 20.16 25.59, 42.52 -0.14 M-0.12 48.92 C15.45 31.01, 31.02 13.09, 42.52 -0.14 M0.27 54.56 C18.31 33.82, 36.34 13.07, 47.51 0.22 M0.27 54.56 C16.58 35.8, 32.89 17.04, 47.51 0.22 M0.01 60.96 C11.85 47.34, 23.69 33.72, 53.15 -0.17 M0.01 60.96 C15.44 43.21, 30.87 25.46, 53.15 -0.17 M-0.25 67.36 C14.61 50.26, 29.47 33.17, 58.14 0.19 M-0.25 67.36 C20.45 43.55, 41.14 19.74, 58.14 0.19 M0.14 73 C13.03 58.18, 25.92 43.35, 63.78 -0.21 M0.14 73 C20.35 49.75, 40.56 26.5, 63.78 -0.21 M-0.12 79.4 C19.43 56.91, 38.97 34.43, 68.77 0.15 M-0.12 79.4 C25.79 49.6, 51.69 19.8, 68.77 0.15 M0.28 85.04 C27.39 53.85, 54.51 22.65, 74.41 -0.24 M0.28 85.04 C17.58 65.14, 34.88 45.23, 74.41 -0.24 M0.01 91.44 C17.98 70.77, 35.95 50.09, 79.4 0.12 M0.01 91.44 C23.11 64.87, 46.2 38.31, 79.4 0.12 M-0.25 97.84 C26.41 67.17, 53.07 36.5, 85.04 -0.28 M-0.25 97.84 C28.6 64.65, 57.45 31.46, 85.04 -0.28 M0.15 103.48 C18.15 82.77, 36.15 62.06, 90.03 0.08 M0.15 103.48 C31.08 67.9, 62.01 32.32, 90.03 0.08 M-0.11 109.88 C33.76 70.91, 67.63 31.94, 95.67 -0.31 M-0.11 109.88 C33.1 71.67, 66.31 33.46, 95.67 -0.31 M0.28 115.52 C38.85 71.15, 77.43 26.77, 100.66 0.05 M0.28 115.52 C24.85 87.26, 49.42 59, 100.66 0.05 M0.02 121.92 C22.42 96.15, 44.82 70.38, 106.3 -0.34 M0.02 121.92 C30.58 86.76, 61.14 51.61, 106.3 -0.34 M-0.24 128.32 C35.23 87.51, 70.7 46.71, 111.29 0.02 M-0.24 128.32 C29.43 94.19, 59.09 60.06, 111.29 0.02 M0.15 133.96 C36.92 91.66, 73.69 49.36, 116.27 0.38 M0.15 133.96 C23.76 106.81, 47.36 79.65, 116.27 0.38 M-0.11 140.36 C24.57 111.97, 49.25 83.58, 121.92 -0.02 M-0.11 140.36 C42 91.91, 84.12 43.47, 121.92 -0.02 M0.29 146 C40.16 100.13, 80.03 54.26, 126.91 0.34 M0.29 146 C32.03 109.48, 63.78 72.96, 126.91 0.34 M0.02 152.4 C52.83 91.65, 105.64 30.9, 132.55 -0.05 M0.02 152.4 C50.83 93.96, 101.63 35.52, 132.55 -0.05 M-0.24 158.8 C49.37 101.73, 98.98 44.66, 137.54 0.31 M-0.24 158.8 C27.42 126.98, 55.08 95.16, 137.54 0.31 M0.16 164.44 C47.25 110.26, 94.35 56.08, 143.18 -0.09 M0.16 164.44 C49.65 107.5, 99.15 50.56, 143.18 -0.09 M-0.1 170.84 C32.17 133.71, 64.45 96.57, 148.17 0.27 M-0.1 170.84 C50.05 113.15, 100.2 55.45, 148.17 0.27 M0.29 176.48 C58.34 109.7, 116.39 42.92, 153.81 -0.12 M0.29 176.48 C55.97 112.42, 111.66 48.37, 153.81 -0.12 M0.03 182.88 C57.85 116.36, 115.67 49.85, 158.8 0.24 M0.03 182.88 C57.52 116.74, 115.01 50.61, 158.8 0.24 M1.08 187.77 C64.7 114.58, 128.32 41.39, 164.44 -0.16 M1.08 187.77 C38.12 145.15, 75.16 102.54, 164.44 -0.16 M6.72 187.37 C53.41 133.66, 100.1 79.96, 169.43 0.2 M6.72 187.37 C53.15 133.96, 99.59 80.54, 169.43 0.2 M11.71 187.73 C71.91 118.47, 132.12 49.22, 175.07 -0.19 M11.71 187.73 C60.02 132.16, 108.33 76.58, 175.07 -0.19 M17.35 187.34 C79.87 115.41, 142.4 43.49, 180.06 0.17 M17.35 187.34 C71.88 124.61, 126.41 61.88, 180.06 0.17 M22.34 187.7 C84.7 115.96, 147.06 44.23, 185.7 -0.23 M22.34 187.7 C86.25 114.17, 150.17 40.65, 185.7 -0.23 M27.33 188.06 C70.69 138.18, 114.05 88.3, 190.69 0.13 M27.33 188.06 C68.58 140.6, 109.83 93.15, 190.69 0.13 M32.97 187.66 C74.82 139.51, 116.68 91.37, 196.33 -0.26 M32.97 187.66 C81.81 131.48, 130.65 75.29, 196.33 -0.26 M37.96 188.02 C78.89 140.93, 119.83 93.84, 201.32 0.1 M37.96 188.02 C102.11 114.22, 166.27 40.42, 201.32 0.1 M43.6 187.63 C89.37 134.98, 135.13 82.33, 206.96 -0.3 M43.6 187.63 C87.49 137.14, 131.37 86.66, 206.96 -0.3 M48.59 187.99 C92.84 137.08, 137.1 86.17, 211.95 0.06 M48.59 187.99 C85.34 145.7, 122.1 103.42, 211.95 0.06 M54.23 187.59 C103.67 130.72, 153.11 73.84, 217.59 -0.33 M54.23 187.59 C105.54 128.57, 156.84 69.55, 217.59 -0.33 M59.22 187.95 C119.4 118.72, 179.59 49.48, 222.58 0.03 M59.22 187.95 C97.23 144.22, 135.25 100.49, 222.58 0.03 M64.86 187.56 C103.05 143.63, 141.23 99.7, 228.22 -0.36 M64.86 187.56 C116 128.72, 167.15 69.89, 228.22 -0.36 M69.85 187.92 C134.56 113.47, 199.28 39.03, 233.21 0 M69.85 187.92 C104.68 147.85, 139.5 107.79, 233.21 0 M75.49 187.52 C131.53 123.06, 187.57 58.6, 238.19 0.36 M75.49 187.52 C125.54 129.95, 175.59 72.38, 238.19 0.36 M80.48 187.88 C132.43 128.12, 184.39 68.35, 243.84 -0.04 M80.48 187.88 C142.22 116.86, 203.96 45.84, 243.84 -0.04 M86.12 187.49 C145.76 118.88, 205.4 50.27, 248.82 0.32 M86.12 187.49 C119.25 149.38, 152.38 111.27, 248.82 0.32 M91.11 187.85 C141.58 129.79, 192.05 71.73, 254.47 -0.07 M91.11 187.85 C132.35 140.41, 173.58 92.97, 254.47 -0.07 M96.75 187.45 C160.69 113.9, 224.64 40.34, 259.45 0.29 M96.75 187.45 C140.87 136.7, 185 85.94, 259.45 0.29 M101.74 187.82 C143.35 139.95, 184.96 92.08, 265.1 -0.11 M101.74 187.82 C163.21 117.1, 224.67 46.39, 265.1 -0.11 M107.38 187.42 C168.81 116.75, 230.25 46.08, 270.08 0.25 M107.38 187.42 C147.14 141.68, 186.91 95.94, 270.08 0.25 M112.37 187.78 C153.55 140.41, 194.72 93.04, 275.73 -0.14 M112.37 187.78 C145.54 149.62, 178.72 111.45, 275.73 -0.14 M118.01 187.39 C156.24 143.41, 194.47 99.43, 280.71 0.22 M118.01 187.39 C154.04 145.94, 190.06 104.5, 280.71 0.22 M123 187.75 C173.43 129.73, 223.86 71.71, 286.36 -0.18 M123 187.75 C155.76 150.06, 188.53 112.36, 286.36 -0.18 M128.64 187.35 C184.1 123.56, 239.55 59.76, 291.34 0.18 M128.64 187.35 C187.94 119.13, 247.25 50.91, 291.34 0.18 M133.63 187.71 C173.37 141.99, 213.11 96.27, 296.99 -0.21 M133.63 187.71 C190.88 121.85, 248.14 55.98, 296.99 -0.21 M138.61 188.07 C176.48 144.51, 214.35 100.95, 301.97 0.15 M138.61 188.07 C203.94 112.92, 269.27 37.77, 301.97 0.15 M144.26 187.68 C207.21 115.26, 270.15 42.85, 307.62 -0.25 M144.26 187.68 C184.88 140.95, 225.5 94.22, 307.62 -0.25 M149.24 188.04 C199.13 130.65, 249.02 73.26, 312.6 0.11 M149.24 188.04 C201.19 128.28, 253.14 68.52, 312.6 0.11 M154.89 187.64 C199.87 135.89, 244.86 84.15, 318.25 -0.28 M154.89 187.64 C188.37 149.12, 221.86 110.6, 318.25 -0.28 M159.87 188 C212.16 127.86, 264.44 67.72, 323.23 0.08 M159.87 188 C204.51 136.66, 249.14 85.31, 323.23 0.08 M165.52 187.61 C225.12 119.05, 284.71 50.49, 328.88 -0.31 M165.52 187.61 C221.62 123.07, 277.72 58.54, 328.88 -0.31 M170.51 187.97 C218.73 132.49, 266.95 77.02, 333.86 0.05 M170.51 187.97 C216.91 134.59, 263.31 81.21, 333.86 0.05 M176.15 187.57 C227.78 128.17, 279.42 68.77, 339.51 -0.35 M176.15 187.57 C225.47 130.83, 274.8 74.09, 339.51 -0.35 M181.14 187.93 C215.64 148.24, 250.14 108.55, 344.49 0.01 M181.14 187.93 C235.53 125.36, 289.93 62.78, 344.49 0.01 M186.78 187.54 C235.99 130.93, 285.2 74.31, 349.48 0.37 M186.78 187.54 C240.9 125.27, 295.03 63.01, 349.48 0.37 M191.77 187.9 C242.3 129.77, 292.83 71.63, 355.12 -0.02 M191.77 187.9 C235.33 137.78, 278.9 87.66, 355.12 -0.02 M197.41 187.5 C247.56 129.81, 297.71 72.12, 360.11 0.34 M197.41 187.5 C230.23 149.75, 263.05 111.99, 360.11 0.34 M202.4 187.86 C266.4 114.23, 330.41 40.6, 365.75 -0.06 M202.4 187.86 C250.72 132.27, 299.05 76.68, 365.75 -0.06 M208.04 187.47 C245.22 144.69, 282.41 101.92, 370.74 0.3 M208.04 187.47 C255.11 133.32, 302.18 79.18, 370.74 0.3 M213.03 187.83 C265.6 127.35, 318.17 66.87, 376.38 -0.09 M213.03 187.83 C261.23 132.37, 309.44 76.92, 376.38 -0.09 M218.67 187.44 C278.8 118.27, 338.92 49.1, 381.37 0.27 M218.67 187.44 C265.8 133.22, 312.92 79.01, 381.37 0.27 M223.66 187.8 C276.49 127.02, 329.32 66.24, 387.01 -0.13 M223.66 187.8 C273.21 130.79, 322.76 73.79, 387.01 -0.13 M229.3 187.4 C285.97 122.21, 342.64 57.01, 392 0.23 M229.3 187.4 C279.92 129.17, 330.54 70.94, 392 0.23 M234.29 187.76 C280.3 134.83, 326.31 81.9, 397.64 -0.16 M234.29 187.76 C270.77 145.79, 307.25 103.82, 397.64 -0.16 M239.93 187.37 C303.84 113.84, 367.76 40.31, 402.63 0.2 M239.93 187.37 C301.12 116.97, 362.31 46.58, 402.63 0.2 M244.92 187.73 C284.53 142.15, 324.15 96.58, 408.27 -0.2 M244.92 187.73 C297.53 127.2, 350.14 66.68, 408.27 -0.2 M250.56 187.33 C289.94 142.03, 329.33 96.72, 413.26 0.16 M250.56 187.33 C302.79 127.25, 355.02 67.16, 413.26 0.16 M255.55 187.69 C306.66 128.89, 357.77 70.09, 418.9 -0.23 M255.55 187.69 C309.53 125.58, 363.52 63.48, 418.9 -0.23 M260.53 188.05 C300.7 141.85, 340.86 95.65, 423.89 0.13 M260.53 188.05 C308.95 132.35, 357.37 76.65, 423.89 0.13 M266.18 187.66 C325.09 119.89, 384 52.12, 429.53 -0.26 M266.18 187.66 C322.32 123.07, 378.46 58.49, 429.53 -0.26 M271.16 188.02 C314.46 138.21, 357.76 88.4, 434.52 0.1 M271.16 188.02 C314.75 137.87, 358.34 87.73, 434.52 0.1 M276.81 187.62 C328.19 128.52, 379.57 69.41, 440.16 -0.3 M276.81 187.62 C329.19 127.36, 381.58 67.09, 440.16 -0.3 M281.79 187.98 C342.46 118.19, 403.13 48.4, 445.15 0.06 M281.79 187.98 C332.59 129.55, 383.38 71.12, 445.15 0.06 M287.44 187.59 C330.61 137.92, 373.79 88.25, 450.79 -0.33 M287.44 187.59 C350.28 115.29, 413.13 42.99, 450.79 -0.33 M292.42 187.95 C349.2 122.63, 405.98 57.32, 455.78 0.03 M292.42 187.95 C344.93 127.55, 397.43 67.15, 455.78 0.03 M298.07 187.55 C356.97 119.79, 415.88 52.03, 461.42 -0.37 M298.07 187.55 C350.83 126.85, 403.6 66.15, 461.42 -0.37 M303.05 187.91 C366.31 115.15, 429.56 42.39, 466.41 -0.01 M303.05 187.91 C352.88 130.59, 402.71 73.27, 466.41 -0.01 M308.7 187.52 C359.95 128.56, 411.21 69.6, 471.4 0.35 M308.7 187.52 C355.37 133.83, 402.04 80.14, 471.4 0.35 M313.68 187.88 C378.19 113.67, 442.69 39.47, 477.04 -0.04 M313.68 187.88 C359.85 134.77, 406.01 81.67, 477.04 -0.04 M319.33 187.49 C368.16 131.31, 416.99 75.14, 482.03 0.32 M319.33 187.49 C366.77 132.91, 414.21 78.33, 482.03 0.32 M324.31 187.85 C371.05 134.09, 417.78 80.33, 487.67 -0.08 M324.31 187.85 C387.86 114.74, 451.42 41.63, 487.67 -0.08 M329.96 187.45 C370.53 140.78, 411.09 94.11, 492.66 0.28 M329.96 187.45 C381.47 128.19, 432.99 68.92, 492.66 0.28 M334.94 187.81 C381.87 133.83, 428.79 79.85, 498.3 -0.11 M334.94 187.81 C395.92 117.67, 456.89 47.52, 498.3 -0.11 M340.59 187.42 C398.69 120.58, 456.79 53.73, 503.29 0.25 M340.59 187.42 C374.02 148.96, 407.45 110.49, 503.29 0.25 M345.57 187.78 C384.09 143.47, 422.6 99.17, 508.93 -0.15 M345.57 187.78 C394.3 131.72, 443.04 75.66, 508.93 -0.15 M351.22 187.38 C412.85 116.49, 474.47 45.59, 513.92 0.21 M351.22 187.38 C410.71 118.94, 470.21 50.49, 513.92 0.21 M356.2 187.74 C401.94 135.13, 447.67 82.52, 519.56 -0.18 M356.2 187.74 C395.61 142.41, 435.02 97.07, 519.56 -0.18 M361.85 187.35 C415.64 125.47, 469.43 63.58, 524.55 0.18 M361.85 187.35 C405.59 137.03, 449.33 86.71, 524.55 0.18 M366.83 187.71 C428.76 116.47, 490.68 45.24, 530.19 -0.22 M366.83 187.71 C431.07 113.82, 495.3 39.92, 530.19 -0.22 M371.82 188.07 C435.51 114.8, 499.19 41.54, 535.18 0.14 M371.82 188.07 C429.09 122.18, 486.37 56.3, 535.18 0.14 M377.46 187.67 C410.67 149.47, 443.88 111.27, 540.82 -0.25 M377.46 187.67 C436.17 120.13, 494.89 52.6, 540.82 -0.25 M382.45 188.03 C446.92 113.87, 511.39 39.71, 545.81 0.11 M382.45 188.03 C433.21 129.64, 483.97 71.25, 545.81 0.11 M388.09 187.64 C426.28 143.71, 464.46 99.79, 551.45 -0.28 M388.09 187.64 C430.42 138.95, 472.74 90.26, 551.45 -0.28 M393.08 188 C449.29 123.33, 505.5 58.67, 556.44 0.08 M393.08 188 C427.87 147.97, 462.67 107.95, 556.44 0.08 M398.72 187.6 C441.6 138.28, 484.48 88.95, 562.08 -0.32 M398.72 187.6 C435.57 145.21, 472.42 102.82, 562.08 -0.32 M403.71 187.96 C452.43 131.92, 501.15 75.88, 567.07 0.04 M403.71 187.96 C466.64 115.57, 529.58 43.17, 567.07 0.04 M409.35 187.57 C456.57 133.25, 503.79 78.94, 570.74 1.91 M409.35 187.57 C458.78 130.71, 508.21 73.84, 570.74 1.91 M414.34 187.93 C457.93 137.78, 501.53 87.63, 571.14 7.55 M414.34 187.93 C464.43 130.31, 514.52 72.69, 571.14 7.55 M419.98 187.53 C455.88 146.25, 491.77 104.96, 570.88 13.95 M419.98 187.53 C474.7 124.59, 529.42 61.64, 570.88 13.95 M424.97 187.89 C478.19 126.68, 531.4 65.46, 570.62 20.35 M424.97 187.89 C479.08 125.65, 533.19 63.4, 570.62 20.35 M430.61 187.5 C461.96 151.44, 493.32 115.37, 571.01 25.99 M430.61 187.5 C482.58 127.72, 534.55 67.94, 571.01 25.99 M435.6 187.86 C463.06 156.28, 490.51 124.69, 570.75 32.39 M435.6 187.86 C470.95 147.2, 506.3 106.53, 570.75 32.39 M441.24 187.47 C478.22 144.93, 515.2 102.39, 571.14 38.03 M441.24 187.47 C482.61 139.88, 523.97 92.3, 571.14 38.03 M446.23 187.83 C483.02 145.51, 519.81 103.19, 570.88 44.43 M446.23 187.83 C492.81 134.25, 539.38 80.67, 570.88 44.43 M451.87 187.43 C483.79 150.72, 515.7 114.01, 570.62 50.83 M451.87 187.43 C488.81 144.94, 525.75 102.45, 570.62 50.83 M456.86 187.79 C491.8 147.6, 526.73 107.42, 571.02 56.47 M456.86 187.79 C486.08 154.18, 515.3 120.57, 571.02 56.47 M462.5 187.4 C498.57 145.91, 534.64 104.42, 570.75 62.87 M462.5 187.4 C491.27 154.3, 520.04 121.2, 570.75 62.87 M467.49 187.76 C504.96 144.65, 542.43 101.54, 571.15 68.51 M467.49 187.76 C500.74 149.51, 533.98 111.27, 571.15 68.51 M473.13 187.36 C497.25 159.63, 521.36 131.89, 570.89 74.91 M473.13 187.36 C494.09 163.26, 515.04 139.16, 570.89 74.91 M478.12 187.72 C510.33 150.67, 542.53 113.63, 570.63 81.31 M478.12 187.72 C509.96 151.09, 541.81 114.46, 570.63 81.31 M483.76 187.33 C508.95 158.35, 534.14 129.37, 571.02 86.95 M483.76 187.33 C501.44 167, 519.11 146.66, 571.02 86.95 M488.75 187.69 C506.26 167.55, 523.77 147.4, 570.76 93.35 M488.75 187.69 C518.83 153.08, 548.92 118.48, 570.76 93.35 M493.74 188.05 C520.82 156.9, 547.9 125.75, 571.15 98.99 M493.74 188.05 C523.96 153.29, 554.18 118.52, 571.15 98.99 M499.38 187.65 C518.41 165.77, 537.43 143.88, 570.89 105.39 M499.38 187.65 C527.48 155.33, 555.58 123, 570.89 105.39 M504.37 188.01 C519.7 170.37, 535.04 152.73, 570.63 111.79 M504.37 188.01 C527.67 161.21, 550.97 134.41, 570.63 111.79 M510.01 187.62 C526.8 168.3, 543.59 148.99, 571.03 117.43 M510.01 187.62 C527.77 167.19, 545.53 146.76, 571.03 117.43 M515 187.98 C534.29 165.79, 553.58 143.59, 570.76 123.83 M515 187.98 C535.88 163.96, 556.76 139.94, 570.76 123.83 M520.64 187.58 C539.4 166.01, 558.15 144.44, 571.16 129.47 M520.64 187.58 C535.48 170.51, 550.32 153.44, 571.16 129.47 M525.63 187.94 C535.44 176.66, 545.25 165.37, 570.9 135.87 M525.63 187.94 C541.96 169.15, 558.3 150.36, 570.9 135.87 M531.27 187.55 C544.87 171.9, 558.48 156.26, 570.64 142.27 M531.27 187.55 C542.99 174.07, 554.7 160.6, 570.64 142.27 M536.26 187.91 C549.4 172.79, 562.54 157.68, 571.03 147.91 M536.26 187.91 C547.63 174.83, 559 161.75, 571.03 147.91 M541.9 187.52 C552.01 175.88, 562.13 164.25, 570.77 154.31 M541.9 187.52 C552.39 175.45, 562.88 163.39, 570.77 154.31 M546.89 187.88 C553.56 180.2, 560.23 172.53, 571.16 159.95 M546.89 187.88 C554.25 179.41, 561.6 170.95, 571.16 159.95 M552.53 187.48 C559.85 179.06, 567.18 170.63, 570.9 166.35 M552.53 187.48 C556.49 182.93, 560.45 178.37, 570.9 166.35 M557.52 187.84 C562.29 182.36, 567.06 176.87, 570.64 172.75 M557.52 187.84 C560.58 184.32, 563.64 180.8, 570.64 172.75 M563.16 187.45 C565.25 185.05, 567.33 182.65, 571.04 178.39 M563.16 187.45 C565.86 184.34, 568.56 181.23, 571.04 178.39" stroke="#b2f2bb" stroke-width="0.5" fill="none"></path><path d="M0 0 C121.22 0, 242.44 0, 568.97 0 M0 0 C164.3 0, 328.6 0, 568.97 0 M568.97 0 C568.97 60.43, 568.97 120.86, 568.97 186.72 M568.97 0 C568.97 53.24, 568.97 106.47, 568.97 186.72 M568.97 186.72 C349.73 186.72, 130.49 186.72, 0 186.72 M568.97 186.72 C381.6 186.72, 194.22 186.72, 0 186.72 M0 186.72 C0 123.26, 0 59.81, 0 0 M0 186.72 C0 124.43, 0 62.13, 0 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(91.08545878266341 410.62101028311974) rotate(0 66.15234375 11.5)"><text x="0" y="0" font-family="Helvetica, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">cgroup module</text></g><g stroke-linecap="round" transform="translate(109.21805279071555 154.46154315655042) rotate(0 102.5 30)"><path d="M0 0 C76.31 0, 152.62 0, 205 0 M0 0 C60.02 0, 120.04 0, 205 0 M205 0 C205 17.63, 205 35.25, 205 60 M205 0 C205 17.11, 205 34.22, 205 60 M205 60 C150.16 60, 95.33 60, 0 60 M205 60 C142.29 60, 79.57 60, 0 60 M0 60 C0 47.33, 0 34.67, 0 0 M0 60 C0 42.71, 0 25.43, 0 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(144.18875591571555 172.96154315655042) rotate(0 67.529296875 11.5)"><text x="67.529296875" y="0" font-family="Helvetica, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">isulad-> main.c</text></g><g mask="url(#mask-Z2IlJidp5B5mEt1LY22zf)" stroke-linecap="round"><g transform="translate(208.85173473177133 216.65387197641223) rotate(0 -0.33793531837079627 90.40968941641856)"><path d="M0 0 C-0.14 36.52, -0.27 73.03, -0.68 180.82 M0 0 C-0.17 45.12, -0.34 90.24, -0.68 180.82" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(208.85173473177133 216.65387197641223) rotate(0 -0.33793531837079627 90.40968941641856)"><path d="M-10.83 152.59 C-8.78 158.29, -6.73 163.99, -0.68 180.82 M-10.83 152.59 C-8.3 159.63, -5.76 166.68, -0.68 180.82" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(208.85173473177133 216.65387197641223) rotate(0 -0.33793531837079627 90.40968941641856)"><path d="M9.69 152.67 C7.6 158.35, 5.5 164.04, -0.68 180.82 M9.69 152.67 C7.1 159.69, 4.52 166.72, -0.68 180.82" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask id="mask-Z2IlJidp5B5mEt1LY22zf"><rect x="0" y="0" fill="#fff" width="309.5276053685129" height="497.47325080924935"></rect><rect x="137.90833066340053" y="295.5635613928308" fill="#000" width="141.2109375" height="23" opacity="1"></rect></mask><g transform="translate(137.90833066340053 295.5635613928308) rotate(0 70.60546875 11.5)"><text x="70.60546875" y="0" font-family="Helvetica, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">cgroup_ops_init</text></g><g stroke-linecap="round" transform="translate(101.82307753274779 508.3165004206021) rotate(0 98 29)"><path d="M0 0 C40.3 0, 80.6 0, 196 0 M0 0 C39.68 0, 79.35 0, 196 0 M196 0 C196 17.87, 196 35.74, 196 58 M196 0 C196 17.81, 196 35.62, 196 58 M196 58 C141.93 58, 87.86 58, 0 58 M196 58 C119.63 58, 43.26 58, 0 58 M0 58 C0 38.27, 0 18.53, 0 0 M0 58 C0 42.53, 0 27.07, 0 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(117.54768690774779 525.8165004206021) rotate(0 82.275390625 11.5)"><text x="82.275390625" y="0" font-family="Helvetica, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">cgroup_v1 module</text></g><g stroke-linecap="round" transform="translate(401.20519080528857 506.23076923076917) rotate(0 98 25)"><path d="M0 0 C48.4 0, 96.8 0, 196 0 M0 0 C52.22 0, 104.43 0, 196 0 M196 0 C196 18.17, 196 36.34, 196 50 M196 0 C196 15.98, 196 31.95, 196 50 M196 50 C119.73 50, 43.45 50, 0 50 M196 50 C123.78 50, 51.57 50, 0 50 M0 50 C0 34.31, 0 18.63, 0 0 M0 50 C0 37.76, 0 25.51, 0 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(416.92980018028857 519.7307692307692) rotate(0 82.275390625 11.5)"><text x="82.275390625" y="0" font-family="Helvetica, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">cgroup_v2 module</text></g><g stroke-linecap="round"><g transform="translate(566.2947696692333 305.8498554667045) rotate(0 -98.53129481428007 54.070998441748415)"><path d="M0 0 C-72.92 40.02, -145.84 80.03, -197.06 108.14 M0 0 C-70.73 38.81, -141.45 77.62, -197.06 108.14" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(566.2947696692333 305.8498554667045) rotate(0 -98.53129481428007 54.070998441748415)"><path d="M-177.28 85.58 C-184.6 93.93, -191.92 102.28, -197.06 108.14 M-177.28 85.58 C-184.38 93.68, -191.48 101.78, -197.06 108.14" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(566.2947696692333 305.8498554667045) rotate(0 -98.53129481428007 54.070998441748415)"><path d="M-167.41 103.57 C-178.38 105.26, -189.35 106.95, -197.06 108.14 M-167.41 103.57 C-178.05 105.21, -188.7 106.85, -197.06 108.14" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round"><g transform="translate(688.0098845775594 305.6465550352888) rotate(0 65.41268960241374 43.93665881792563)"><path d="M0 0 C29.75 19.98, 59.5 39.97, 130.83 87.87 M0 0 C42.07 28.26, 84.15 56.52, 130.83 87.87" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(688.0098845775594 305.6465550352888) rotate(0 65.41268960241374 43.93665881792563)"><path d="M101.7 80.67 C108.33 82.31, 114.95 83.95, 130.83 87.87 M101.7 80.67 C111.07 82.99, 120.43 85.3, 130.83 87.87" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(688.0098845775594 305.6465550352888) rotate(0 65.41268960241374 43.93665881792563)"><path d="M113.14 63.64 C117.17 69.15, 121.19 74.66, 130.83 87.87 M113.14 63.64 C118.83 71.43, 124.52 79.23, 130.83 87.87" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(734.6924015925484 398.69230769230774) rotate(0 123 30)"><path d="M0 0 C51.23 0, 102.47 0, 246 0 M0 0 C84.24 0, 168.48 0, 246 0 M246 0 C246 22.38, 246 44.76, 246 60 M246 0 C246 22.77, 246 45.53, 246 60 M246 60 C180.6 60, 115.2 60, 0 60 M246 60 C171.95 60, 97.91 60, 0 60 M0 60 C0 38.56, 0 17.11, 0 0 M0 60 C0 43.32, 0 26.65, 0 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(788.7715031550484 417.19230769230774) rotate(0 68.9208984375 11.5)"><text x="68.9208984375" y="0" font-family="Helvetica, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">runtime module</text></g><g stroke-linecap="round"><g transform="translate(847.9580239569957 463.60901384450085) rotate(0 3.7916014027637033 104.80790931965728)"><path d="M0 0 C2.02 55.88, 4.04 111.75, 7.58 209.62 M0 0 C2.1 58.15, 4.21 116.29, 7.58 209.62" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(847.9580239569957 463.60901384450085) rotate(0 3.7916014027637033 104.80790931965728)"><path d="M-3.69 181.81 C-0.68 189.23, 2.32 196.64, 7.58 209.62 M-3.69 181.81 C-0.56 189.53, 2.56 197.24, 7.58 209.62" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(847.9580239569957 463.60901384450085) rotate(0 3.7916014027637033 104.80790931965728)"><path d="M16.82 181.07 C14.36 188.68, 11.89 196.29, 7.58 209.62 M16.82 181.07 C14.26 188.99, 11.69 196.91, 7.58 209.62" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(741.141995638666 683.0577882873695) rotate(0 123 29.5)"><path d="M0 0 C61.28 0, 122.55 0, 246 0 M0 0 C67 0, 134.01 0, 246 0 M246 0 C246 21.57, 246 43.14, 246 59 M246 0 C246 13.82, 246 27.63, 246 59 M246 59 C169.83 59, 93.66 59, 0 59 M246 59 C150.65 59, 55.31 59, 0 59 M0 59 C0 42.6, 0 26.21, 0 0 M0 59 C0 36.92, 0 14.83, 0 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(844.688870638666 701.0577882873695) rotate(0 19.453125 11.5)"><text x="19.453125" y="0" font-family="Helvetica, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">runc</text></g><g stroke-linecap="round" transform="translate(360.61538461538487 150.39745154747584) rotate(0 114 26.5)"><path d="M0 0 C73.79 0, 147.58 0, 228 0 M0 0 C65.01 0, 130.01 0, 228 0 M228 0 C228 20.42, 228 40.85, 228 53 M228 0 C228 17.45, 228 34.91, 228 53 M228 53 C150.52 53, 73.03 53, 0 53 M228 53 C151.94 53, 75.87 53, 0 53 M0 53 C0 39.84, 0 26.68, 0 0 M0 53 C0 34.52, 0 16.05, 0 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(418.47280649038487 165.39745154747584) rotate(0 56.142578125 11.5)"><text x="56.142578125" y="0" font-family="Helvetica, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">isula module</text></g><g stroke-linecap="round" transform="translate(10 107.66668231670661) rotate(0 520.0000000000003 250.25641808143044)"><path d="M0 0 C319.44 0, 638.88 0, 1040 0 M1040 0 C1040 153.69, 1040 307.37, 1040 500.51 M1040 500.51 C742.63 500.51, 445.26 500.51, 0 500.51 M0 500.51 C0 340.73, 0 180.94, 0 0" stroke="#1e1e1e" stroke-width="1.5" fill="none" stroke-dasharray="8 9"></path></g><g stroke-linecap="round"><g transform="translate(472.56413386418296 205.10259540264417) rotate(0 0 20.512812687800476)"><path d="M0 0 C0 12.05, 0 24.11, 0 41.03 M0 0 C0 11.7, 0 23.4, 0 41.03" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(472.56413386418296 205.10259540264417) rotate(0 0 20.512812687800476)"><path d="M-7.02 21.75 C-4.95 27.41, -2.89 33.08, 0 41.03 M-7.02 21.75 C-5.01 27.25, -3.01 32.74, 0 41.03" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(472.56413386418296 205.10259540264417) rotate(0 0 20.512812687800476)"><path d="M7.02 21.75 C4.95 27.41, 2.89 33.08, 0 41.03 M7.02 21.75 C5.01 27.25, 3.01 32.74, 0 41.03" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(257.0740559895836 417.0356324459415) rotate(0 98 29)"><path d="M0 0 C65.48 0, 130.97 0, 196 0 M0 0 C63.94 0, 127.88 0, 196 0 M196 0 C196 15.77, 196 31.53, 196 58 M196 0 C196 12.32, 196 24.64, 196 58 M196 58 C147.67 58, 99.34 58, 0 58 M196 58 C123.95 58, 51.9 58, 0 58 M0 58 C0 36.42, 0 14.84, 0 0 M0 58 C0 35.7, 0 13.39, 0 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(267.2517903645836 434.5356324459415) rotate(0 87.822265625 11.5)"><text x="87.822265625" y="0" font-family="Helvetica, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">common cgroup api</text></g><g stroke-linecap="round"><g transform="translate(302.04162786372365 476.17288734790225) rotate(0 -36.3238555368913 12.234823288057385)"><path d="M0 0 C-21.71 7.31, -43.41 14.62, -72.65 24.47 M0 0 C-24.16 8.14, -48.31 16.27, -72.65 24.47" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(302.04162786372365 476.17288734790225) rotate(0 -36.3238555368913 12.234823288057385)"><path d="M-49.21 5.75 C-56.21 11.34, -63.21 16.94, -72.65 24.47 M-49.21 5.75 C-57 11.97, -64.8 18.2, -72.65 24.47" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(302.04162786372365 476.17288734790225) rotate(0 -36.3238555368913 12.234823288057385)"><path d="M-42.66 25.19 C-51.62 24.98, -60.58 24.76, -72.65 24.47 M-42.66 25.19 C-52.63 24.95, -62.6 24.71, -72.65 24.47" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round"><g transform="translate(396.4073802806717 477.44310314906653) rotate(0 37.92595757378467 10.962953920717581)"><path d="M0 0 C18.86 5.45, 37.71 10.9, 75.85 21.93 M0 0 C16.62 4.8, 33.23 9.61, 75.85 21.93" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(396.4073802806717 477.44310314906653) rotate(0 37.92595757378467 10.962953920717581)"><path d="M45.92 23.95 C53.36 23.45, 60.8 22.95, 75.85 21.93 M45.92 23.95 C52.48 23.51, 59.03 23.07, 75.85 21.93" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(396.4073802806717 477.44310314906653) rotate(0 37.92595757378467 10.962953920717581)"><path d="M51.62 4.24 C57.64 8.64, 63.67 13.03, 75.85 21.93 M51.62 4.24 C56.93 8.11, 62.24 11.99, 75.85 21.93" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(279.5661335676041 685.5203469732791) rotate(0 123 29.5)"><path d="M0 0 C51.97 0, 103.94 0, 246 0 M0 0 C79.29 0, 158.58 0, 246 0 M246 0 C246 14.61, 246 29.22, 246 59 M246 0 C246 14.91, 246 29.81, 246 59 M246 59 C161.48 59, 76.96 59, 0 59 M246 59 C190.09 59, 134.19 59, 0 59 M0 59 C0 43.43, 0 27.85, 0 0 M0 59 C0 38.7, 0 18.4, 0 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(356.4284382551041 703.5203469732791) rotate(0 46.1376953125 11.5)"><text x="46.1376953125" y="0" font-family="Helvetica, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">cgroup file</text></g><g mask="url(#mask-EYY2S4Db98XZJRcvHx1vQ)" stroke-linecap="round"><g transform="translate(371.8052665912037 581.8665635115902) rotate(0 0.8604642600295591 49.47556254594039)"><path d="M0 0 C0.53 30.21, 1.05 60.42, 1.72 98.95 M0 0 C0.61 35.09, 1.22 70.19, 1.72 98.95" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(371.8052665912037 581.8665635115902) rotate(0 0.8604642600295591 49.47556254594039)"><path d="M-9.03 70.94 C-5.75 79.49, -2.47 88.04, 1.72 98.95 M-9.03 70.94 C-5.22 80.88, -1.4 90.81, 1.72 98.95" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(371.8052665912037 581.8665635115902) rotate(0 0.8604642600295591 49.47556254594039)"><path d="M11.49 70.59 C8.51 79.25, 5.53 87.91, 1.72 98.95 M11.49 70.59 C8.03 80.65, 4.56 90.71, 1.72 98.95" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask id="mask-EYY2S4Db98XZJRcvHx1vQ"><rect x="0" y="0" fill="#fff" width="473.52619511126284" height="780.817688603471"></rect><rect x="328.75948085123326" y="619.8421260575307" fill="#000" width="87.8125" height="23" opacity="1"></rect></mask><g transform="translate(328.75948085123326 619.8421260575305) rotate(0 43.906249999999986 11.499999999999972)"><text x="43.90625" y="0" font-family="Helvetica, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">read/write</text></g><g mask="url(#mask-sovtxKH081DyUEeId5jis)" stroke-linecap="round"><g transform="translate(740.141995638666 718.2353652610261) rotate(0 -106.78793103553097 0.6813763689311543)"><path d="M0 0 C-54.68 0.35, -109.36 0.7, -213.58 1.36 M0 0 C-56.56 0.36, -113.12 0.72, -213.58 1.36" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(740.141995638666 718.2353652610261) rotate(0 -106.78793103553097 0.6813763689311543)"><path d="M-185.45 -9.08 C-192.65 -6.4, -199.85 -3.73, -213.58 1.36 M-185.45 -9.08 C-192.9 -6.31, -200.35 -3.55, -213.58 1.36" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(740.141995638666 718.2353652610261) rotate(0 -106.78793103553097 0.6813763689311543)"><path d="M-185.32 11.44 C-192.55 8.86, -199.79 6.28, -213.58 1.36 M-185.32 11.44 C-192.8 8.77, -200.29 6.1, -213.58 1.36" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask id="mask-sovtxKH081DyUEeId5jis"><rect x="0" y="0" fill="#fff" width="1053.717857709728" height="819.5981179988885"></rect><rect x="589.447814603135" y="707.4167416299572" fill="#000" width="87.8125" height="23" opacity="1"></rect></mask><g transform="translate(589.447814603135 707.4167416299572) rotate(0 43.90625 11.500000000000028)"><text x="43.90625" y="0" font-family="Helvetica, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">read/write</text></g></svg> +\ No newline at end of file +-- +2.34.1 + diff --git a/0038-add-modify-for-cgroup-v2-ci-test.patch b/0038-add-modify-for-cgroup-v2-ci-test.patch new file mode 100644 index 0000000..843652a --- /dev/null +++ b/0038-add-modify-for-cgroup-v2-ci-test.patch @@ -0,0 +1,608 @@ +From 33b26f27dd897574d73ce8654620a13edbeb947e Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Tue, 2 Apr 2024 02:31:58 +1400 +Subject: [PATCH 38/69] add modify for cgroup v2 ci test + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + CI/test_cases/manual_cases/cgroupv2.sh | 276 +++++++++++++++---------- + 1 file changed, 165 insertions(+), 111 deletions(-) + +diff --git a/CI/test_cases/manual_cases/cgroupv2.sh b/CI/test_cases/manual_cases/cgroupv2.sh +index f8982f08..8e431688 100755 +--- a/CI/test_cases/manual_cases/cgroupv2.sh ++++ b/CI/test_cases/manual_cases/cgroupv2.sh +@@ -31,59 +31,59 @@ function test_cgroup2_cpu() + + if [[ -f /sys/fs/cgroup/isulad/cpu.weight ]];then + # min value +- isula run -ti --rm --cpu-shares 2 busybox cat /sys/fs/cgroup/cpu.weight | grep ^1$'\r' ++ isula run --runtime $1 -ti --rm --cpu-shares 2 busybox cat /sys/fs/cgroup/cpu.weight | grep ^1$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.weight min value failed" && ((ret++)) + + # max value +- isula run -ti --rm --cpu-shares 262144 busybox cat /sys/fs/cgroup/cpu.weight | grep ^10000$'\r' ++ isula run --runtime $1 -ti --rm --cpu-shares 262144 busybox cat /sys/fs/cgroup/cpu.weight | grep ^10000$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.weight max value failed" && ((ret++)) + + # invalid value +- isula run -ti --rm --cpu-shares -1 busybox echo hello ++ isula run --runtime $1 -ti --rm --cpu-shares -1 busybox echo hello + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.weight -1 failed" && ((ret++)) + + # default value +- isula run -ti --rm --cpu-shares 0 busybox cat /sys/fs/cgroup/cpu.weight | grep ^100$'\r' ++ isula run --runtime $1 -ti --rm --cpu-shares 0 busybox cat /sys/fs/cgroup/cpu.weight | grep ^100$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.weight default value failed" && ((ret++)) + fi + + if [[ -f /sys/fs/cgroup/isulad/cpu.max ]];then + # normal value +- isula run -ti --rm --cpu-quota 50000 --cpu-period 12345 busybox cat /sys/fs/cgroup/cpu.max | grep ^"50000 12345"$'\r' ++ isula run --runtime $1 -ti --rm --cpu-quota 50000 --cpu-period 12345 busybox cat /sys/fs/cgroup/cpu.max | grep ^"50000 12345"$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.max normal value failed" && ((ret++)) + + # invalid min period +- isula run -ti --rm --cpu-quota 50000 --cpu-period 999 busybox echo hello ++ isula run --runtime $1 -ti --rm --cpu-quota 50000 --cpu-period 999 busybox echo hello + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.max invalid min period failed" && ((ret++)) + + # invalid max period +- isula run -ti --rm --cpu-quota 50000 --cpu-period 1000001 busybox echo hello ++ isula run --runtime $1 -ti --rm --cpu-quota 50000 --cpu-period 1000001 busybox echo hello + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.max invalid max period failed" && ((ret++)) + + # invalid quota +- isula run -ti --rm --cpu-quota 999 --cpu-period 1000000 busybox echo hello ++ isula run --runtime $1 -ti --rm --cpu-quota 999 --cpu-period 1000000 busybox echo hello + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.max invalid quota failed" && ((ret++)) + + # default 0 quota +- isula run -ti --rm --cpu-quota 0 --cpu-period 1000000 busybox cat /sys/fs/cgroup/cpu.max | grep ^"max 1000000"$'\r' ++ isula run --runtime $1 -ti --rm --cpu-quota 0 --cpu-period 1000000 busybox cat /sys/fs/cgroup/cpu.max | grep ^"max 1000000"$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.max default 0 quota failed" && ((ret++)) + + # default -1 quota +- isula run -ti --rm --cpu-quota -1 --cpu-period 1000000 busybox cat /sys/fs/cgroup/cpu.max | grep ^"max 1000000"$'\r' ++ isula run --runtime $1 -ti --rm --cpu-quota -1 --cpu-period 1000000 busybox cat /sys/fs/cgroup/cpu.max | grep ^"max 1000000"$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.max default -1 quota failed" && ((ret++)) + + # cpus 1 +- isula run -ti --rm --cpus 1 busybox cat /sys/fs/cgroup/cpu.max | grep ^"100000 100000"$'\r' ++ isula run --runtime $1 -ti --rm --cpus 1 busybox cat /sys/fs/cgroup/cpu.max | grep ^"100000 100000"$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.max cpus 1 failed" && ((ret++)) + + # cpus 0 +- isula run -ti --rm --cpus 0 busybox cat /sys/fs/cgroup/cpu.max | grep ^"max 100000"$'\r' ++ isula run --runtime $1 -ti --rm --cpus 0 busybox cat /sys/fs/cgroup/cpu.max | grep ^"max 100000"$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.max cpus 0 failed" && ((ret++)) + fi + + if [[ -f /sys/fs/cgroup/isulad/cpuset.cpus.effective ]];then + # normal value +- isula run -tid -n cpuset --cpuset-cpus 0 --cpuset-mems 0 busybox sh ++ isula run --runtime $1 -tid -n cpuset --cpuset-cpus 0 --cpuset-mems 0 busybox sh + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpuset run container failed" && ((ret++)) + + isula exec -ti cpuset cat /sys/fs/cgroup/cpuset.cpus | grep ^0$'\r' +@@ -96,19 +96,19 @@ function test_cgroup2_cpu() + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpuset remove container failed" && ((ret++)) + + # invalid cpus -1 value +- isula run -tid -n cpuset --cpuset-cpus -1 busybox sh ++ isula run --runtime $1 -tid -n cpuset --cpuset-cpus -1 busybox sh + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpuset cpus invalid -1 failed" && ((ret++)) + + # invalid cpus 100000 value +- isula run -tid -n cpuset --cpuset-cpus 100000 busybox sh ++ isula run --runtime $1 -tid -n cpuset --cpuset-cpus 100000 busybox sh + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpuset cpus invalid 100000 failed" && ((ret++)) + + # invalid mems -1 value +- isula run -tid -n cpuset --cpuset-mems -1 busybox sh ++ isula run --runtime $1 -tid -n cpuset --cpuset-mems -1 busybox sh + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpuset mems invalid -1 failed" && ((ret++)) + + # invalid mems 100000 value +- isula run -tid -n cpuset --cpuset-mems 100000 busybox sh ++ isula run --runtime $1 -tid -n cpuset --cpuset-mems 100000 busybox sh + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpuset mems invalid 100000 failed" && ((ret++)) + fi + +@@ -121,33 +121,38 @@ function test_cgroup2_io() + + if [[ -f "/sys/fs/cgroup/isulad/io.bfq.weight" ]];then + # min value +- isula run -ti --rm --blkio-weight 10 busybox cat "/sys/fs/cgroup/io.bfq.weight" | grep 1$'\r' +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.bfq.weight min value failed" && ((ret++)) ++ if [ $1 == "lcr" ]; then ++ isula run --runtime $1 -ti --rm --blkio-weight 10 busybox cat "/sys/fs/cgroup/io.bfq.weight" | grep 1$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.bfq.weight min value failed" && ((ret++)) ++ else ++ isula run --runtime $1 -ti --rm --blkio-weight 10 busybox cat "/sys/fs/cgroup/io.bfq.weight" | grep 10$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.bfq.weight min value failed" && ((ret++)) ++ fi + + # max value +- isula run -ti --rm --blkio-weight 1000 busybox cat "/sys/fs/cgroup/io.bfq.weight" | grep 1000$'\r' ++ isula run --runtime $1 -ti --rm --blkio-weight 1000 busybox cat "/sys/fs/cgroup/io.bfq.weight" | grep 1000$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.bfq.weight max value failed" && ((ret++)) + + # default value +- isula run -ti --rm --blkio-weight 0 busybox cat "/sys/fs/cgroup/io.bfq.weight" | grep 100$'\r' ++ isula run --runtime $1 -ti --rm --blkio-weight 0 busybox cat "/sys/fs/cgroup/io.bfq.weight" | grep 100$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.bfq.weight default value failed" && ((ret++)) + + # invalid value +- isula run -ti --rm --blkio-weight -1 busybox echo hello ++ isula run --runtime $1 -ti --rm --blkio-weight -1 busybox echo hello + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.bfq.weight -1 failed" && ((ret++)) + fi + + if [[ -f "/sys/fs/cgroup/isulad/io.bfq.weight_device" ]];then + # min value +- isula run -ti --rm --blkio-weight-device /dev/null:10 busybox cat "/sys/fs/cgroup/io.bfq.weight_device" | grep ^"1:3 10"$'\r' ++ isula run --runtime $1 -ti --rm --blkio-weight-device /dev/null:10 busybox cat "/sys/fs/cgroup/io.bfq.weight_device" | grep ^"1:3 10"$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.bfq.weight_device max value failed" && ((ret++)) + + # max value +- isula run -ti --rm --blkio-weight-device /dev/null:1000 busybox cat "/sys/fs/cgroup/io.bfq.weight_device" | grep ^"1:3 10000"$'\r' ++ isula run --runtime $1 -ti --rm --blkio-weight-device /dev/null:1000 busybox cat "/sys/fs/cgroup/io.bfq.weight_device" | grep ^"1:3 10000"$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.bfq.weight_device max value failed" && ((ret++)) + + # disable weight device +- isula run -tid -n weight_device --rm --blkio-weight-device /dev/null:0 busybox sh ++ isula run --runtime $1 -tid -n weight_device --rm --blkio-weight-device /dev/null:0 busybox sh + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.bfq.weight_device failed" && ((ret++)) + + isula exec -ti weight_device cat "/sys/fs/cgroup/io.bfq.weight_device" | grep "1:3" +@@ -159,33 +164,43 @@ function test_cgroup2_io() + + if [[ -f "/sys/fs/cgroup/isulad/io.weight" ]];then + # min value +- isula run -ti --rm --blkio-weight 10 busybox cat "/sys/fs/cgroup/io.weight" | grep ^"default 1"$'\r' +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight min value failed" && ((ret++)) ++ if [ $1 == "lcr" ]; then ++ isula run --runtime $1 -ti --rm --blkio-weight 10 busybox cat "/sys/fs/cgroup/io.weight" | grep ^"default 1"$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight min value failed" && ((ret++)) ++ else ++ isula run --runtime $1 -ti --rm --blkio-weight 10 busybox cat "/sys/fs/cgroup/io.bfq.weight" | grep ^"default 10"$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight min value failed" && ((ret++)) ++ fi + + # max value +- isula run -ti --rm --blkio-weight 1000 busybox cat "/sys/fs/cgroup/io.weight" | grep ^"default 10000"$'\r' +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight max value failed" && ((ret++)) ++ if [ $1 == "lcr" ]; then ++ isula run --runtime $1 -ti --rm --blkio-weight 1000 busybox cat "/sys/fs/cgroup/io.weight" | grep ^"default 10000"$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight max value failed" && ((ret++)) ++ else ++ isula run --runtime $1 -ti --rm --blkio-weight 1000 busybox cat "/sys/fs/cgroup/io.bfq.weight" | grep ^"default 1000"$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight max value failed" && ((ret++)) ++ fi + + # default value +- isula run -ti --rm --blkio-weight 0 busybox cat "/sys/fs/cgroup/io.weight" | grep ^"default 100"$'\r' ++ isula run --runtime $1 -ti --rm --blkio-weight 0 busybox cat "/sys/fs/cgroup/io.weight" | grep ^"default 100"$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight default value failed" && ((ret++)) + + # invalid value +- isula run -ti --rm --blkio-weight -1 busybox echo hello ++ isula run --runtime $1 -ti --rm --blkio-weight -1 busybox echo hello + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight -1 failed" && ((ret++)) + fi + + if [[ -f "/sys/fs/cgroup/isulad/io.weight_device" ]];then + # min value +- isula run -ti --rm --blkio-weight-device /dev/null:10 busybox cat "/sys/fs/cgroup/io.weight_device" | grep ^"1:3 10"$'\r' ++ isula run --runtime $1 -ti --rm --blkio-weight-device /dev/null:10 busybox cat "/sys/fs/cgroup/io.weight_device" | grep ^"1:3 10"$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight max value failed" && ((ret++)) + + # max value +- isula run -ti --rm --blkio-weight-device /dev/null:1000 busybox cat "/sys/fs/cgroup/io.weight_device" | grep ^"1:3 10000"$'\r' ++ isula run --runtime $1 -ti --rm --blkio-weight-device /dev/null:1000 busybox cat "/sys/fs/cgroup/io.weight_device" | grep ^"1:3 10000"$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight max value failed" && ((ret++)) + + # disable weight device +- isula run -tid -n weight_device --rm --blkio-weight-device /dev/null:0 busybox sh ++ isula run --runtime $1 -tid -n weight_device --rm --blkio-weight-device /dev/null:0 busybox sh + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight failed" && ((ret++)) + + isula exec -ti weight_device cat "/sys/fs/cgroup/io.weight_device" | grep ^"1:3"$'\r' +@@ -197,16 +212,22 @@ function test_cgroup2_io() + + if [[ -f /sys/fs/cgroup/isulad/io.max ]];then + # normal value +- isula run -ti --rm --device-read-bps /dev/null:1g --device-read-iops /dev/null:1000 --device-write-bps /dev/null:2g --device-write-iops /dev/null:2000 busybox cat /sys/fs/cgroup/io.max | grep ^"1:3 rbps=1073741824 wbps=2147483648 riops=1000 wiops=2000"$'\r' ++ isula run --runtime $1 -ti --rm --device-read-bps /dev/null:1g --device-read-iops /dev/null:1000 --device-write-bps /dev/null:2g --device-write-iops /dev/null:2000 busybox cat /sys/fs/cgroup/io.max | grep ^"1:3 rbps=1073741824 wbps=2147483648 riops=1000 wiops=2000"$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.max failed" && ((ret++)) + + # invalid +- isula run -ti --rm --device-read-bps /dev/null:-1 busybox echo hello ++ isula run --runtime $1 -ti --rm --device-read-bps /dev/null:-1 busybox echo hello + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.max -1 failed" && ((ret++)) + +- # 0 is no limit +- isula run -ti --rm --device-read-bps /dev/null:0 --device-read-iops /dev/null:0 --device-write-bps /dev/null:0 --device-write-iops /dev/null:0 busybox echo hello +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.max 0 failed" && ((ret++)) ++ if [ $1 == "lcr" ]; then ++ # 0 is no limit ++ isula run --runtime $1 -ti --rm --device-read-bps /dev/null:0 --device-read-iops /dev/null:0 --device-write-bps /dev/null:0 --device-write-iops /dev/null:0 busybox echo hello ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.max 0 failed" && ((ret++)) ++ else ++ # 0 is limit ++ isula run --runtime $1 -ti --rm --device-read-bps /dev/null:0 --device-read-iops /dev/null:0 --device-write-bps /dev/null:0 --device-write-iops /dev/null:0 busybox echo hello ++ [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.max 0 success" && ((ret++)) ++ fi + fi + + return ${ret} +@@ -218,51 +239,51 @@ function test_cgroup2_memory() + + if [[ -f /sys/fs/cgroup/isulad/memory.max ]];then + # normal value +- isula run -ti --rm -m 10m busybox cat /sys/fs/cgroup/memory.max | grep ^10485760$'\r' ++ isula run --runtime $1 -ti --rm -m 10m busybox cat /sys/fs/cgroup/memory.max | grep ^10485760$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.max run container failed" && ((ret++)) + + # 0 is max +- isula run -ti --rm -m 0 busybox cat /sys/fs/cgroup/memory.max | grep ^max$'\r' ++ isula run --runtime $1 -ti --rm -m 0 busybox cat /sys/fs/cgroup/memory.max | grep ^max$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.max 0 failed" && ((ret++)) + + # invalid +- isula run -ti --rm -m -1 busybox echo hello ++ isula run --runtime $1 -ti --rm -m -1 busybox echo hello + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.max -1 failed" && ((ret++)) + fi + + if [[ -f /sys/fs/cgroup/isulad/memory.low ]];then + # normal value +- isula run -ti --rm --memory-reservation 10m busybox cat /sys/fs/cgroup/memory.low | grep ^10485760$'\r' ++ isula run --runtime $1 -ti --rm --memory-reservation 10m busybox cat /sys/fs/cgroup/memory.low | grep ^10485760$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.low normal value failed" && ((ret++)) + + # -1 is invalid +- isula run -ti --rm --memory-reservation -1 busybox echo hello ++ isula run --runtime $1 -ti --rm --memory-reservation -1 busybox echo hello + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.low invalid failed" && ((ret++)) + + # 0 +- isula run -ti --rm --memory-reservation 0 busybox cat /sys/fs/cgroup/memory.low | grep ^0$'\r' ++ isula run --runtime $1 -ti --rm --memory-reservation 0 busybox cat /sys/fs/cgroup/memory.low | grep ^0$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.low 0 failed" && ((ret++)) + fi + + if [[ -f /sys/fs/cgroup/isulad/memory.swap.max ]];then + # normal value +- isula run -ti --rm --memory 10m --memory-swap 20m busybox cat /sys/fs/cgroup/memory.swap.max | grep ^10485760$'\r' ++ isula run --runtime $1 -ti --rm --memory 10m --memory-swap 20m busybox cat /sys/fs/cgroup/memory.swap.max | grep ^10485760$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.swap.max normal value failed" && ((ret++)) + + # invalid +- isula run -ti --rm --memory 10m --memory-swap 5m busybox echo hello ++ isula run --runtime $1 -ti --rm --memory 10m --memory-swap 5m busybox echo hello + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.swap.max invalid failed" && ((ret++)) + + # 0 is the same as memory +- isula run -ti --rm --memory 10m --memory-swap 0 busybox cat /sys/fs/cgroup/memory.swap.max | grep ^10485760$'\r' ++ isula run --runtime $1 -ti --rm --memory 10m --memory-swap 0 busybox cat /sys/fs/cgroup/memory.swap.max | grep ^10485760$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.swap.max 0 failed" && ((ret++)) + + # -1 is max +- isula run -ti --rm --memory 10m --memory-swap -1 busybox cat /sys/fs/cgroup/memory.swap.max | grep ^max$'\r' ++ isula run --runtime $1 -ti --rm --memory 10m --memory-swap -1 busybox cat /sys/fs/cgroup/memory.swap.max | grep ^max$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.swap.max -1 failed" && ((ret++)) + + # disable swap +- isula run -ti --rm --memory 10m --memory-swap 10m busybox cat /sys/fs/cgroup/memory.swap.max | grep ^0$'\r' ++ isula run --runtime $1 -ti --rm --memory 100m --memory-swap 100m busybox cat /sys/fs/cgroup/memory.swap.max | grep ^0$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.swap.max disable swap failed" && ((ret++)) + fi + +@@ -275,15 +296,15 @@ function test_cgroup2_pids() + + if [[ -f /sys/fs/cgroup/isulad/pids.max ]];then + # normal value +- isula run -ti --rm --pids-limit 123456 busybox cat /sys/fs/cgroup/pids.max | grep ^123456$'\r' ++ isula run --runtime $1 -ti --rm --pids-limit 123456 busybox cat /sys/fs/cgroup/pids.max | grep ^123456$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 pids.max run container failed" && ((ret++)) + + # -1 is max +- isula run -ti --rm --pids-limit -1 busybox cat /sys/fs/cgroup/pids.max | grep ^max$'\r' ++ isula run --runtime $1 -ti --rm --pids-limit -1 busybox cat /sys/fs/cgroup/pids.max | grep ^max$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 pids.max run container failed" && ((ret++)) + + # 0 is max +- isula run -ti --rm --pids-limit 0 busybox cat /sys/fs/cgroup/pids.max | grep ^max$'\r' ++ isula run --runtime $1 -ti --rm --pids-limit 0 busybox cat /sys/fs/cgroup/pids.max | grep ^max$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 pids.max run container failed" && ((ret++)) + fi + +@@ -295,7 +316,7 @@ function test_cgroup2_hugetlb() + local ret=0 + + if [[ -f /sys/fs/cgroup/isulad/hugetlb.2MB.max ]];then +- isula run -ti --rm --hugetlb-limit 2M:32M busybox cat /sys/fs/cgroup/hugetlb.2MB.max | grep ^33554432$'\r' ++ isula run --runtime $1 -ti --rm --hugetlb-limit 2M:32M busybox cat /sys/fs/cgroup/hugetlb.2MB.max | grep ^33554432$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 hugetlb.2M.max run container failed" && ((ret++)) + fi + +@@ -307,7 +328,7 @@ function test_cgroup2_freeze() + local ret=0 + + if [[ -f /sys/fs/cgroup/isulad/cgroup.freeze ]];then +- isula run -tid -n freeze busybox sh ++ isula run --runtime $1 -tid -n freeze busybox sh + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 freeze run container failed" && ((ret++)) + + isula pause freeze +@@ -335,15 +356,15 @@ function test_cgroup2_files() + + if [[ -f /sys/fs/cgroup/isulad/files.limit ]];then + # normal value +- isula run -ti --rm --files-limit 123 busybox cat /sys/fs/cgroup/files.limit | grep ^123$'\r' ++ isula run --runtime $1 -ti --rm --files-limit 123 busybox cat /sys/fs/cgroup/files.limit | grep ^123$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 files.limit run container failed" && ((ret++)) + + # -1 is max +- isula run -ti --rm --files-limit -1 busybox cat /sys/fs/cgroup/files.limit | grep ^max$'\r' ++ isula run --runtime $1 -ti --rm --files-limit -1 busybox cat /sys/fs/cgroup/files.limit | grep ^max$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 files.limit run container failed" && ((ret++)) + + # 0 is max +- isula run -ti --rm --files-limit 0 busybox cat /sys/fs/cgroup/files.limit | grep ^max$'\r' ++ isula run --runtime $1 -ti --rm --files-limit 0 busybox cat /sys/fs/cgroup/files.limit | grep ^max$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 files.limit run container failed" && ((ret++)) + fi + +@@ -405,8 +426,13 @@ function test_cgroup2_cpu_update() + isula update --cpu-quota 0 --cpu-period 1000000 $cgroup2_update + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update cpu.max 0 quota failed" && ((ret++)) + +- isula exec -ti $cgroup2_update cat /sys/fs/cgroup/cpu.max | grep ^"max 1000000"$'\r' +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update cpu.max 0 quota value not right" && ((ret++)) ++ if [ $1 == "lcr" ]; then ++ isula exec -ti $cgroup2_update cat /sys/fs/cgroup/cpu.max | grep ^"max 1000000"$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update cpu.max 0 quota value not right" && ((ret++)) ++ else ++ isula exec -ti $cgroup2_update cat /sys/fs/cgroup/cpu.max | grep ^"50000 1000000"$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update cpu.max 0 quota value not right" && ((ret++)) ++ fi + + # default -1 quota + isula update --cpu-quota -1 --cpu-period 1000000 $cgroup2_update +@@ -416,7 +442,7 @@ function test_cgroup2_cpu_update() + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update cpu.max -1 quota value not right" && ((ret++)) + + # cpus 1 +- isula run -tid -n cpu_update busybox sh ++ isula run --runtime $1 -tid -n cpu_update busybox sh + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 run cpu_update failed" && ((ret++)) + + isula update --cpus 1 cpu_update +@@ -476,8 +502,13 @@ function test_cgroup2_io_update() + isula update --blkio-weight 10 $cgroup2_update + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.bfq.weight min value failed" && ((ret++)) + +- isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.bfq.weight" | grep 1$'\r' +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.bfq.weight min value not right" && ((ret++)) ++ if [ $1 == "lcr" ]; then ++ isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.bfq.weight" | grep 1$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.bfq.weight min value not right" && ((ret++)) ++ else ++ isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.bfq.weight" | grep 10$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.bfq.weight min value not right" && ((ret++)) ++ fi + + # max value + isula update --blkio-weight 1000 $cgroup2_update +@@ -503,22 +534,38 @@ function test_cgroup2_io_update() + isula update --blkio-weight 10 $cgroup2_update + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight min value failed" && ((ret++)) + +- isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.weight" | grep ^"default 1"$'\r' +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight min value not right" && ((ret++)) ++ if [ $1 == "lcr" ]; then ++ isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.weight" | grep ^"default 1"$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight min value not right" && ((ret++)) ++ else ++ isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.bfq.weight" | grep ^"default 10"$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight min value not right" && ((ret++)) ++ fi + + # max value + isula update --blkio-weight 1000 $cgroup2_update + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight max value failed" && ((ret++)) + +- isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.weight" | grep ^"default 10000"$'\r' +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight max value not right" && ((ret++)) ++ if [ $1 == "lcr" ]; then ++ isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.weight" | grep ^"default 10000"$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight max value not right" && ((ret++)) ++ else ++ isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.bfq.weight" | grep ^"default 1000"$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight max value not right" && ((ret++)) ++ fi + + # 0 means value not change + isula update --blkio-weight 0 $cgroup2_update + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight 0 failed" && ((ret++)) + +- isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.weight" | grep ^"default 10000"$'\r' +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight 0 not right" && ((ret++)) ++ ++ if [ $1 == "lcr" ]; then ++ isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.weight" | grep ^"default 10000"$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight max value not right" && ((ret++)) ++ else ++ isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.bfq.weight" | grep ^"default 1000"$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight max value not right" && ((ret++)) ++ fi + + # invalid value + isula update --blkio-weight -1 $cgroup2_update echo hello +@@ -591,12 +638,13 @@ function test_cgroup2_memory_update() + isula exec -ti $cgroup2_update cat /sys/fs/cgroup/memory.swap.max | grep ^10485760$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update memory.swap.max 0 value not right" && ((ret++)) + +- # -1 is max +- isula update --memory 10m --memory-swap -1 $cgroup2_update +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update memory.swap.max -1 failed" && ((ret++)) +- +- isula exec -ti $cgroup2_update cat /sys/fs/cgroup/memory.swap.max | grep ^max$'\r' +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update memory.swap.max -1 value not right" && ((ret++)) ++ if [ $1 == "lcr" ]; then ++ # -1 is max ++ isula update --memory 10m --memory-swap -1 $cgroup2_update ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update memory.swap.max -1 failed" && ((ret++)) ++ isula exec -ti $cgroup2_update cat /sys/fs/cgroup/memory.swap.max | grep ^max$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update memory.swap.max -1 value not right" && ((ret++)) ++ fi + + # disable swap + isula update --memory 10m --memory-swap 10m $cgroup2_update +@@ -613,16 +661,16 @@ function test_cgroup2_unsupported() + { + local ret=0 + +- isula run -ti --rm --cpu-rt-period 1000000 --cpu-rt-runtime 1000000 busybox echo hello ++ isula run --runtime $1 -ti --rm --cpu-rt-period 1000000 --cpu-rt-runtime 1000000 busybox echo hello + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --cpu-rt-period and --cpu-rt-runtime should failed" && ((ret++)) + +- isula run -ti --rm --kernel-memory 100m busybox echo hello ++ isula run --runtime $1 -ti --rm --kernel-memory 100m busybox echo hello + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --kernel-memory should failed" && ((ret++)) + +- isula run -ti --rm --memory-swappiness 50 busybox echo hello ++ isula run --runtime $1 -ti --rm --memory-swappiness 50 busybox echo hello + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --memory-swappiness should failed" && ((ret++)) + +- isula run -ti --rm --oom-kill-disable busybox echo hello ++ isula run --runtime $1 -ti --rm --oom-kill-disable busybox echo hello + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --oom-kill-disable should failed" && ((ret++)) + + isula update --cpu-rt-period 1000000 --cpu-rt-runtime 1000000 $cgroup2_update +@@ -641,7 +689,7 @@ function test_cgroup2_parent() + rmdir /sys/fs/cgroup/isulad + rmdir /sys/fs/cgroup/abc + +- id=`isula run -tid --cgroup-parent /abc -m 10m busybox sh` ++ id=`isula run --runtime $1 -tid --cgroup-parent /abc -m 10m busybox sh` + cat /sys/fs/cgroup/abc/$id/memory.max | grep ^10485760$ + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --cgroup-parent cannot work" && ((ret++)) + +@@ -657,39 +705,39 @@ function test_cgroup2_device() + mknod_num=$(echo $dev_num | sed 's/:/ /g') + + # read only +- isula run -ti --rm --device=$dev_name:/dev/sdx:r busybox sh -c 'echo q | fdisk /dev/sdx | grep "read only"' ++ isula run --runtime $1 -ti --rm --device=$dev_name:/dev/sdx:r busybox sh -c 'echo q | fdisk /dev/sdx | grep "read only"' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device r failed" && ((ret++)) + +- isula run -ti --rm --device=$dev_name:/dev/sdx:rm busybox sh -c 'echo q | fdisk /dev/sdx | grep "read only"' ++ isula run --runtime $1 -ti --rm --device=$dev_name:/dev/sdx:rm busybox sh -c 'echo q | fdisk /dev/sdx | grep "read only"' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device rm failed" && ((ret++)) + +- isula run -ti --rm --device-cgroup-rule="b $dev_num r" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx | grep 'read only'" ++ isula run --runtime $1 -ti --rm --device-cgroup-rule="b $dev_num r" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx | grep 'read only'" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device r failed" && ((ret++)) + +- isula run -ti --rm --device-cgroup-rule="b $dev_num rm" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx | grep 'read only'" ++ isula run --runtime $1 -ti --rm --device-cgroup-rule="b $dev_num rm" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx | grep 'read only'" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device rm failed" && ((ret++)) + + # can't read +- isula run -ti --rm --device=$dev_name:/dev/sdx:w busybox sh -c 'echo q | fdisk /dev/sdx 2>&1 | grep "t open"' ++ isula run --runtime $1 -ti --rm --device=$dev_name:/dev/sdx:w busybox sh -c 'echo q | fdisk /dev/sdx 2>&1 | grep "t open"' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device w failed" && ((ret++)) + +- isula run -ti --rm --device=$dev_name:/dev/sdx:wm busybox sh -c 'echo q | fdisk /dev/sdx 2>&1 | grep "t open"' ++ isula run --runtime $1 -ti --rm --device=$dev_name:/dev/sdx:wm busybox sh -c 'echo q | fdisk /dev/sdx 2>&1 | grep "t open"' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device wm failed" && ((ret++)) + +- isula run -ti --rm --device-cgroup-rule="b $dev_num w" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx 2>&1 | grep 't open'" ++ isula run --runtime $1 -ti --rm --device-cgroup-rule="b $dev_num w" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx 2>&1 | grep 't open'" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device w failed" && ((ret++)) + +- isula run -ti --rm --device-cgroup-rule="b $dev_num wm" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx 2>&1 | grep 't open'" ++ isula run --runtime $1 -ti --rm --device-cgroup-rule="b $dev_num wm" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx 2>&1 | grep 't open'" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device wm failed" && ((ret++)) + + # can't read write +- isula run -ti --rm --device=$dev_name:/dev/sdx:m busybox sh -c 'echo q | fdisk /dev/sdx 2>&1 | grep "t open"' ++ isula run --runtime $1 -ti --rm --device=$dev_name:/dev/sdx:m busybox sh -c 'echo q | fdisk /dev/sdx 2>&1 | grep "t open"' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device m" && ((ret++)) + +- isula run -ti --rm --device-cgroup-rule="b $dev_num m" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx 2>&1 | grep 't open'" ++ isula run --runtime $1 -ti --rm --device-cgroup-rule="b $dev_num m" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx 2>&1 | grep 't open'" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device wm failed" && ((ret++)) + +- isula run -ti --rm --device-cgroup-rule="b *:* m" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx 2>&1 | grep 't open'" ++ isula run --runtime $1 -ti --rm --device-cgroup-rule="b *:* m" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx 2>&1 | grep 't open'" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device wm failed" && ((ret++)) + + return ${ret} +@@ -723,7 +771,7 @@ function prepare_test_cgroupv2() + + isula rm -f `isula ps -a -q` + +- isula run -tid -n $cgroup2_update busybox sh ++ isula run --runtime $1 -tid -n $cgroup2_update busybox sh + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 run container failed" && ((ret++)) + + return ${ret} +@@ -740,25 +788,31 @@ declare -i ans=0 + msg_info "${test} starting..." + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start isulad failed" && ((ret++)) + +-prepare_test_cgroupv2 || ((ans++)) +-if [ "$cgroupv2" == "1" ];then +- test_cgroup2_cpu || ((ans++)) +- test_cgroup2_io || ((ans++)) +- test_cgroup2_memory || ((ans++)) +- test_cgroup2_pids || ((ans++)) +- test_cgroup2_hugetlb || ((ans++)) +- test_cgroup2_freeze || ((ans++)) +- test_cgroup2_files || ((ans++)) +- test_cgroup2_cpu_update || ((ans++)) +- test_cgroup2_io_update || ((ans++)) +- test_cgroup2_memory_update || ((ans++)) +- test_cgroup2_unsupported || ((ans++)) +- test_cgroup2_parent || ((ans++)) +- test_cgroup2_device || ((ans++)) +-else +- msg_info "${test} not cgroup v2 enviorment, ignore test..." +-fi +-post_test_cgroupv2 ++for element in ${RUNTIME_LIST[@]}; ++do ++ prepare_test_cgroupv2 $element || ((ans++)) ++ if [ "$cgroupv2" == "1" ];then ++ local test="cgroup v2 test => (${element})" ++ msg_info "${test} starting..." ++ test_cgroup2_cpu $element || ((ans++)) ++ test_cgroup2_io $element || ((ans++)) ++ test_cgroup2_memory $element || ((ans++)) ++ test_cgroup2_pids $element || ((ans++)) ++ test_cgroup2_hugetlb $element || ((ans++)) ++ test_cgroup2_freeze $element || ((ans++)) ++ test_cgroup2_files $element || ((ans++)) ++ test_cgroup2_cpu_update $element || ((ans++)) ++ test_cgroup2_io_update $element || ((ans++)) ++ test_cgroup2_memory_update $element || ((ans++)) ++ test_cgroup2_unsupported $element || ((ans++)) ++ test_cgroup2_parent $element || ((ans++)) ++ test_cgroup2_device $element || ((ans++)) ++ msg_info "${test} finished with return ${ans}..." ++ else ++ msg_info "${test} not cgroup v2 enviorment, ignore test..." ++ fi ++ post_test_cgroupv2 $element ++done + + msg_info "${test} finished with return ${ans}..." + +-- +2.34.1 + diff --git a/0039-fix-run-ubuntu-container-bug-in-inspect.sh.patch b/0039-fix-run-ubuntu-container-bug-in-inspect.sh.patch new file mode 100644 index 0000000..5501261 --- /dev/null +++ b/0039-fix-run-ubuntu-container-bug-in-inspect.sh.patch @@ -0,0 +1,27 @@ +From 8e1fe0302bf1a871f66a296e456811e878b1fa3b Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Tue, 2 Apr 2024 10:06:18 +0800 +Subject: [PATCH 39/69] fix run ubuntu container bug in inspect.sh + +Signed-off-by: jikai <jikai11@huawei.com> +--- + CI/test_cases/container_cases/inspect.sh | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/CI/test_cases/container_cases/inspect.sh b/CI/test_cases/container_cases/inspect.sh +index b4f4a785..86aed3d8 100755 +--- a/CI/test_cases/container_cases/inspect.sh ++++ b/CI/test_cases/container_cases/inspect.sh +@@ -146,7 +146,8 @@ function test_inspect_spec() + + isula rm -f $containername + +- isula run -it -m 4m --name $containername $ubuntu_image perl -e 'for ($i = 0; $i < 100000000; $i++) { $a .= " " x 1024 }' ++ # use more than 10m memory limit, otherwise it might fail to run ++ isula run -it -m 10m --name $containername $ubuntu_image perl -e 'for ($i = 0; $i < 100000000; $i++) { $a .= " " x 1024 }' + + isula inspect -f "{{json .State.OOMKilled}} {{.Name}}" $containername 2>&1 | sed -n '1p' | grep "true" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container with image: ${ubuntu_image}" && ((ret++)) +-- +2.34.1 + diff --git a/0040-add-support-for-GetContainerEvents.patch b/0040-add-support-for-GetContainerEvents.patch new file mode 100644 index 0000000..397a792 --- /dev/null +++ b/0040-add-support-for-GetContainerEvents.patch @@ -0,0 +1,2601 @@ +From 745497bdc5c5192709ecc7b3edc91a5170f5b30e Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Fri, 29 Mar 2024 09:33:38 +0000 +Subject: [PATCH 40/69] add support for GetContainerEvents + +Signed-off-by: jikai <jikai11@huawei.com> +--- + src/daemon/CMakeLists.txt | 3 + + src/daemon/common/cri/v1/v1_cri_helpers.cc | 205 +++++++++++++++ + src/daemon/common/cri/v1/v1_cri_helpers.h | 5 + + .../{entry => common}/cri/v1/v1_naming.cc | 0 + .../{entry => common}/cri/v1/v1_naming.h | 0 + src/daemon/config/isulad_config.c | 1 + + .../entry/connect/grpc/cri/cri_service.cc | 3 +- + .../entry/connect/grpc/cri/cri_service.h | 1 + + .../cri/v1/cri_v1_runtime_runtime_service.cc | 147 ++++++++++- + .../cri/v1/cri_v1_runtime_runtime_service.h | 11 +- + src/daemon/entry/connect/grpc/grpc_service.cc | 6 +- + .../v1/v1_cri_container_manager_service.cc | 203 +-------------- + .../cri/v1/v1_cri_container_manager_service.h | 13 - + .../v1/v1_cri_pod_sandbox_manager_service.cc | 92 ++++++- + .../v1/v1_cri_pod_sandbox_manager_service.h | 10 +- + .../entry/cri/v1/v1_cri_runtime_service.h | 4 +- + .../cri/v1/v1_cri_runtime_service_impl.cc | 10 +- + .../cri/v1/v1_cri_runtime_service_impl.h | 7 +- + src/daemon/executor/container_cb/execution.c | 25 ++ + .../executor/container_cb/execution_create.c | 12 + + src/daemon/mailbox/CMakeLists.txt | 11 + + src/daemon/mailbox/mailbox.c | 167 +++++++++++++ + src/daemon/mailbox/mailbox.h | 82 ++++++ + src/daemon/mailbox/mailbox_message.c | 94 +++++++ + src/daemon/mailbox/mailbox_message.h | 50 ++++ + src/daemon/mailbox/message_queue.c | 234 ++++++++++++++++++ + src/daemon/mailbox/message_queue.h | 57 +++++ + src/daemon/mailbox/message_subscriber.c | 85 +++++++ + src/daemon/mailbox/message_subscriber.h | 41 +++ + src/daemon/modules/api/container_api.h | 5 + + .../modules/container/supervisor/supervisor.c | 18 ++ + src/daemon/sandbox/sandbox.cc | 9 + + src/utils/cutils/blocking_queue.c | 185 ++++++++++++++ + src/utils/cutils/blocking_queue.h | 66 +++++ + test/mocks/mailbox_mock.cc | 30 +++ + test/mocks/mailbox_mock.h | 30 +++ + test/sandbox/controller/shim/CMakeLists.txt | 1 + + test/sandbox/sandbox/CMakeLists.txt | 2 + + 38 files changed, 1681 insertions(+), 244 deletions(-) + rename src/daemon/{entry => common}/cri/v1/v1_naming.cc (100%) + rename src/daemon/{entry => common}/cri/v1/v1_naming.h (100%) + create mode 100644 src/daemon/mailbox/CMakeLists.txt + create mode 100644 src/daemon/mailbox/mailbox.c + create mode 100644 src/daemon/mailbox/mailbox.h + create mode 100644 src/daemon/mailbox/mailbox_message.c + create mode 100644 src/daemon/mailbox/mailbox_message.h + create mode 100644 src/daemon/mailbox/message_queue.c + create mode 100644 src/daemon/mailbox/message_queue.h + create mode 100644 src/daemon/mailbox/message_subscriber.c + create mode 100644 src/daemon/mailbox/message_subscriber.h + create mode 100644 src/utils/cutils/blocking_queue.c + create mode 100644 src/utils/cutils/blocking_queue.h + create mode 100644 test/mocks/mailbox_mock.cc + create mode 100644 test/mocks/mailbox_mock.h + +diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt +index d5280c88..29af3dca 100644 +--- a/src/daemon/CMakeLists.txt ++++ b/src/daemon/CMakeLists.txt +@@ -3,6 +3,7 @@ + aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} daemon_top_srcs) + add_subdirectory(executor) + add_subdirectory(entry) ++add_subdirectory(mailbox) + add_subdirectory(modules) + add_subdirectory(config) + add_subdirectory(common) +@@ -11,6 +12,7 @@ set(local_daemon_srcs + ${daemon_top_srcs} + ${EXECUTOR_SRCS} + ${ENTRY_SRCS} ++ ${MAILBOX_SRCS} + ${MODULES_SRCS} + ${CONFIG_SRCS} + ${DAEMON_COMMON_SRCS} +@@ -20,6 +22,7 @@ set(local_daemon_incs + ${CMAKE_CURRENT_SOURCE_DIR} + ${EXECUTOR_INCS} + ${ENTRY_INCS} ++ ${MAILBOX_INCS} + ${MODULES_INCS} + ${CONFIG_INCS} + ${DAEMON_COMMON_INCS} +diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.cc b/src/daemon/common/cri/v1/v1_cri_helpers.cc +index c57301ce..a3488894 100644 +--- a/src/daemon/common/cri/v1/v1_cri_helpers.cc ++++ b/src/daemon/common/cri/v1/v1_cri_helpers.cc +@@ -32,6 +32,7 @@ + #include "service_container_api.h" + #include "isulad_config.h" + #include "sha256.h" ++#include "v1_naming.h" + + namespace CRIHelpersV1 { + +@@ -458,4 +459,208 @@ void ApplySandboxSecurityContextToHostConfig(const runtime::v1::LinuxSandboxSecu + } + } + ++void PackContainerImageToStatus( ++ container_inspect *inspect, std::unique_ptr<runtime::v1::ContainerStatus> &contStatus, Errors &error) ++{ ++ if (inspect->config == nullptr) { ++ return; ++ } ++ ++ if (inspect->config->image != nullptr) { ++ contStatus->mutable_image()->set_image(inspect->config->image); ++ } ++ ++ contStatus->set_image_ref(CRIHelpers::ToPullableImageID(inspect->config->image, inspect->config->image_ref)); ++} ++ ++void UpdateBaseStatusFromInspect( ++ container_inspect *inspect, int64_t &createdAt, int64_t &startedAt, int64_t &finishedAt, ++ std::unique_ptr<runtime::v1::ContainerStatus> &contStatus) ++{ ++ runtime::v1::ContainerState state { runtime::v1::CONTAINER_UNKNOWN }; ++ std::string reason; ++ std::string message; ++ int32_t exitCode { 0 }; ++ ++ if (inspect->state == nullptr) { ++ goto pack_status; ++ } ++ ++ if (inspect->state->running) { ++ // Container is running ++ state = runtime::v1::CONTAINER_RUNNING; ++ } else { ++ // Container is not running. ++ if (finishedAt != 0) { // Case 1 ++ state = runtime::v1::CONTAINER_EXITED; ++ if (inspect->state->exit_code == 0) { ++ reason = "Completed"; ++ } else { ++ reason = "Error"; ++ } ++ } else if (inspect->state->exit_code != 0) { // Case 2 ++ state = runtime::v1::CONTAINER_EXITED; ++ finishedAt = createdAt; ++ startedAt = createdAt; ++ reason = "ContainerCannotRun"; ++ } else { // Case 3 ++ state = runtime::v1::CONTAINER_CREATED; ++ } ++ if (inspect->state->error != nullptr) { ++ message = inspect->state->error; ++ } ++ exitCode = (int32_t)inspect->state->exit_code; ++ } ++ ++pack_status: ++ contStatus->set_exit_code(exitCode); ++ contStatus->set_state(state); ++ contStatus->set_created_at(createdAt); ++ contStatus->set_started_at(startedAt); ++ contStatus->set_finished_at(finishedAt); ++ contStatus->set_reason(reason); ++ contStatus->set_message(message); ++} ++ ++void PackLabelsToStatus(container_inspect *inspect, ++ std::unique_ptr<runtime::v1::ContainerStatus> &contStatus) ++{ ++ if (inspect->config == nullptr || inspect->config->labels == nullptr) { ++ return; ++ } ++ CRIHelpers::ExtractLabels(inspect->config->labels, *contStatus->mutable_labels()); ++ CRIHelpers::ExtractAnnotations(inspect->config->annotations, *contStatus->mutable_annotations()); ++ for (size_t i = 0; i < inspect->config->labels->len; i++) { ++ if (strcmp(inspect->config->labels->keys[i], CRIHelpers::Constants::CONTAINER_LOGPATH_LABEL_KEY.c_str()) == 0) { ++ contStatus->set_log_path(inspect->config->labels->values[i]); ++ break; ++ } ++ } ++} ++ ++void ConvertMountsToStatus(container_inspect *inspect, ++ std::unique_ptr<runtime::v1::ContainerStatus> &contStatus) ++{ ++ for (size_t i = 0; i < inspect->mounts_len; i++) { ++ runtime::v1::Mount *mount = contStatus->add_mounts(); ++ mount->set_host_path(inspect->mounts[i]->source); ++ mount->set_container_path(inspect->mounts[i]->destination); ++ mount->set_readonly(!inspect->mounts[i]->rw); ++ if (inspect->mounts[i]->propagation == nullptr || strcmp(inspect->mounts[i]->propagation, "rprivate") == 0) { ++ mount->set_propagation(runtime::v1::PROPAGATION_PRIVATE); ++ } else if (strcmp(inspect->mounts[i]->propagation, "rslave") == 0) { ++ mount->set_propagation(runtime::v1::PROPAGATION_HOST_TO_CONTAINER); ++ } else if (strcmp(inspect->mounts[i]->propagation, "rshared") == 0) { ++ mount->set_propagation(runtime::v1::PROPAGATION_BIDIRECTIONAL); ++ } ++ // Note: Can't set SeLinuxRelabel ++ } ++} ++ ++void ConvertResourcesToStatus(container_inspect *inspect, ++ std::unique_ptr<runtime::v1::ContainerStatus> &contStatus) ++{ ++ if (inspect->resources == nullptr) { ++ return; ++ } ++ runtime::v1::LinuxContainerResources *resources = contStatus->mutable_resources()->mutable_linux(); ++ if (inspect->resources->cpu_shares != 0) { ++ resources->set_cpu_shares(inspect->resources->cpu_shares); ++ } ++ if (inspect->resources->cpu_period != 0) { ++ resources->set_cpu_period(inspect->resources->cpu_period); ++ } ++ if (inspect->resources->cpu_quota != 0) { ++ resources->set_cpu_quota(inspect->resources->cpu_quota); ++ } ++ if (inspect->resources->memory != 0) { ++ resources->set_memory_limit_in_bytes(inspect->resources->memory); ++ } ++ if (inspect->resources->memory_swap != 0) { ++ resources->set_memory_swap_limit_in_bytes(inspect->resources->memory_swap); ++ } ++ for (size_t i = 0; i < inspect->resources->hugetlbs_len; i++) { ++ runtime::v1::HugepageLimit *hugepage = resources->add_hugepage_limits(); ++ hugepage->set_page_size(inspect->resources->hugetlbs[i]->page_size); ++ hugepage->set_limit(inspect->resources->hugetlbs[i]->limit); ++ } ++ if (inspect->resources->unified != nullptr) { ++ for (size_t i = 0; i < inspect->resources->unified->len; i++) { ++ auto &resUnified = *(resources->mutable_unified()); ++ resUnified[inspect->resources->unified->keys[i]] = inspect->resources->unified->values[i]; ++ } ++ } ++} ++ ++void ContainerStatusToGRPC(container_inspect *inspect, ++ std::unique_ptr<runtime::v1::ContainerStatus> &contStatus, ++ Errors &error) ++{ ++ if (inspect->id != nullptr) { ++ contStatus->set_id(inspect->id); ++ } ++ ++ int64_t createdAt {}; ++ int64_t startedAt {}; ++ int64_t finishedAt {}; ++ CRIHelpers::GetContainerTimeStamps(inspect, &createdAt, &startedAt, &finishedAt, error); ++ if (error.NotEmpty()) { ++ return; ++ } ++ contStatus->set_created_at(createdAt); ++ contStatus->set_started_at(startedAt); ++ contStatus->set_finished_at(finishedAt); ++ ++ PackContainerImageToStatus(inspect, contStatus, error); ++ UpdateBaseStatusFromInspect(inspect, createdAt, startedAt, finishedAt, contStatus); ++ PackLabelsToStatus(inspect, contStatus); ++ CRINamingV1::ParseContainerName(contStatus->annotations(), contStatus->mutable_metadata(), error); ++ if (error.NotEmpty()) { ++ return; ++ } ++ ConvertMountsToStatus(inspect, contStatus); ++ ConvertResourcesToStatus(inspect, contStatus); ++} ++ ++std::unique_ptr<runtime::v1::ContainerStatus> GetContainerStatus(service_executor_t *m_cb, const std::string &containerID, Errors &error) ++{ ++ if (m_cb == nullptr) { ++ error.SetError("Invalid input arguments: empty service executor"); ++ return nullptr; ++ } ++ ++ if (containerID.empty()) { ++ error.SetError("Empty container id"); ++ return nullptr; ++ } ++ ++ std::string realContainerID = CRIHelpers::GetRealContainerOrSandboxID(m_cb, containerID, false, error); ++ if (error.NotEmpty()) { ++ ERROR("Failed to find container id %s: %s", containerID.c_str(), error.GetCMessage()); ++ error.Errorf("Failed to find container id %s: %s", containerID.c_str(), error.GetCMessage()); ++ return nullptr; ++ } ++ ++ container_inspect *inspect = CRIHelpers::InspectContainer(realContainerID, error, false); ++ if (error.NotEmpty()) { ++ return nullptr; ++ } ++ if (inspect == nullptr) { ++ error.SetError("Get null inspect"); ++ return nullptr; ++ } ++ using ContainerStatusPtr = std::unique_ptr<runtime::v1::ContainerStatus>; ++ ContainerStatusPtr contStatus(new (std::nothrow) runtime::v1::ContainerStatus); ++ if (contStatus == nullptr) { ++ error.SetError("Out of memory"); ++ free_container_inspect(inspect); ++ return nullptr; ++ } ++ ++ ContainerStatusToGRPC(inspect, contStatus, error); ++ ++ free_container_inspect(inspect); ++ return contStatus; ++} ++ + } // v1 namespace CRIHelpers +diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.h b/src/daemon/common/cri/v1/v1_cri_helpers.h +index b6e6aec6..1578c428 100644 +--- a/src/daemon/common/cri/v1/v1_cri_helpers.h ++++ b/src/daemon/common/cri/v1/v1_cri_helpers.h +@@ -27,6 +27,8 @@ + #include "checkpoint_handler.h" + #include "constants.h" + #include "errors.h" ++#include "callback.h" ++#include "cstruct_wrapper.h" + + namespace CRIHelpersV1 { + +@@ -78,6 +80,9 @@ std::string CRISandboxerConvert(const std::string &runtime); + void ApplySandboxSecurityContextToHostConfig(const runtime::v1::LinuxSandboxSecurityContext &context, host_config *hc, + Errors &error); + ++auto GetContainerStatus(service_executor_t *m_cb, const std::string &containerID, Errors &error) ++-> std::unique_ptr<runtime::v1::ContainerStatus>; ++ + }; // namespace CRIHelpers + + #endif // DAEMON_ENTRY_CRI_V1ALPHA_CRI_HELPERS_H +diff --git a/src/daemon/entry/cri/v1/v1_naming.cc b/src/daemon/common/cri/v1/v1_naming.cc +similarity index 100% +rename from src/daemon/entry/cri/v1/v1_naming.cc +rename to src/daemon/common/cri/v1/v1_naming.cc +diff --git a/src/daemon/entry/cri/v1/v1_naming.h b/src/daemon/common/cri/v1/v1_naming.h +similarity index 100% +rename from src/daemon/entry/cri/v1/v1_naming.h +rename to src/daemon/common/cri/v1/v1_naming.h +diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c +index 8179558e..778ff921 100644 +--- a/src/daemon/config/isulad_config.c ++++ b/src/daemon/config/isulad_config.c +@@ -1760,6 +1760,7 @@ int merge_json_confs_into_global(struct service_arguments *args) + args->json_confs->cri_sandboxers = tmp_json_confs->cri_sandboxers; + tmp_json_confs->cri_sandboxers = NULL; + args->json_confs->enable_cri_v1 = tmp_json_confs->enable_cri_v1; ++ args->json_confs->enable_pod_events = tmp_json_confs->enable_pod_events; + #endif + + args->json_confs->systemd_cgroup = tmp_json_confs->systemd_cgroup; +diff --git a/src/daemon/entry/connect/grpc/cri/cri_service.cc b/src/daemon/entry/connect/grpc/cri/cri_service.cc +index c1986c44..d10a60b5 100644 +--- a/src/daemon/entry/connect/grpc/cri/cri_service.cc ++++ b/src/daemon/entry/connect/grpc/cri/cri_service.cc +@@ -89,8 +89,9 @@ int CRIService::Init(const isulad_daemon_configs *config) + + #ifdef ENABLE_CRI_API_V1 + m_enableCRIV1 = config->enable_cri_v1; ++ m_enablePodEvents = config->enable_pod_events; + if (m_enableCRIV1) { +- m_runtimeV1RuntimeService.Init(m_podSandboxImage, m_pluginManager, err); ++ m_runtimeV1RuntimeService.Init(m_podSandboxImage, m_pluginManager, m_enablePodEvents, err); + if (err.NotEmpty()) { + ERROR("Init CRI v1 runtime service failed: %s", err.GetCMessage()); + return -1; +diff --git a/src/daemon/entry/connect/grpc/cri/cri_service.h b/src/daemon/entry/connect/grpc/cri/cri_service.h +index 77b2eb72..041c7c63 100644 +--- a/src/daemon/entry/connect/grpc/cri/cri_service.h ++++ b/src/daemon/entry/connect/grpc/cri/cri_service.h +@@ -56,6 +56,7 @@ private: + std::string m_podSandboxImage; + std::shared_ptr<Network::PluginManager> m_pluginManager; + bool m_enableCRIV1; ++ bool m_enablePodEvents; + }; + + } +diff --git a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc +index 76e393f3..bc5ab591 100644 +--- a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc ++++ b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc +@@ -22,11 +22,37 @@ + #include "callback.h" + #include "network_plugin.h" + #include "v1_cri_runtime_service_impl.h" ++#include "mailbox.h" ++#include "mailbox_message.h" + + using namespace CRIV1; + ++static void *cri_container_topic_handler(void *context, void *arg) ++{ ++ if (context == nullptr || arg == nullptr) { ++ ERROR("Invalid input arguments"); ++ return nullptr; ++ } ++ ++ auto v1runtimeService = static_cast<RuntimeV1RuntimeServiceImpl *>(context); ++ auto msg = static_cast<cri_container_message_t *>(arg); ++ return v1runtimeService->GenerateCRIContainerEvent(msg->container_id, msg->sandbox_id, ++ static_cast<runtime::v1::ContainerEventType>(msg->type)); ++} ++ ++static void cri_container_topic_release(void *arg) ++{ ++ if (arg == nullptr) { ++ return; ++ } ++ ++ auto resp = static_cast<runtime::v1::ContainerEventResponse *>(arg); ++ delete resp; ++} ++ + void RuntimeV1RuntimeServiceImpl::Init(std::string &podSandboxImage, +- std::shared_ptr<Network::PluginManager> networkPlugin, Errors &err) ++ std::shared_ptr<Network::PluginManager> networkPlugin, ++ bool enablePodEvents, Errors &err) + { + // Assembly implementation for CRIRuntimeServiceImpl + service_executor_t *cb = get_service_executor(); +@@ -36,7 +62,18 @@ void RuntimeV1RuntimeServiceImpl::Init(std::string &podSandboxImage, + return; + } + +- m_rService = std::unique_ptr<CRIV1::CRIRuntimeService>(new CRIRuntimeServiceImpl(podSandboxImage, cb, networkPlugin)); ++ if (enablePodEvents) { ++ if (mailbox_register_topic_handler(MAILBOX_TOPIC_CRI_CONTAINER, cri_container_topic_handler, ++ this, cri_container_topic_release, true) != 0) { ++ ERROR("Failed to register container topic handler"); ++ err.SetError("Failed to register container topic handler"); ++ return; ++ } ++ m_enablePodEvents = enablePodEvents; ++ } ++ ++ ++ m_rService = std::unique_ptr<CRIV1::CRIRuntimeService>(new CRIRuntimeServiceImpl(podSandboxImage, cb, networkPlugin, m_enablePodEvents)); + } + + void RuntimeV1RuntimeServiceImpl::Wait() +@@ -45,6 +82,54 @@ void RuntimeV1RuntimeServiceImpl::Wait() + + void RuntimeV1RuntimeServiceImpl::Shutdown() + { ++ mailbox_unregister_topic_handler(MAILBOX_TOPIC_CRI_CONTAINER); ++} ++ ++auto RuntimeV1RuntimeServiceImpl::GenerateCRIContainerEvent(const char *container_id, const char *sandbox_id, ++ runtime::v1::ContainerEventType type) -> runtime::v1::ContainerEventResponse * ++{ ++ if (container_id == nullptr || sandbox_id == nullptr) { ++ ERROR("Invalid input arguments"); ++ return nullptr; ++ } ++ ++ if (type < runtime::v1::ContainerEventType::CONTAINER_CREATED_EVENT || ++ type > runtime::v1::ContainerEventType::CONTAINER_DELETED_EVENT) { ++ ERROR("Invalid container event type %d", type); ++ return nullptr; ++ } ++ ++ std::string containerID(container_id), sandboxID(sandbox_id); ++ Errors error; ++ runtime::v1::ContainerEventResponse *response = new (std::nothrow) runtime::v1::ContainerEventResponse(); ++ if (response == nullptr) { ++ ERROR("Out of memory"); ++ return nullptr; ++ } ++ ++ runtime::v1::PodSandboxStatusResponse *statusReply = new (std::nothrow) runtime::v1::PodSandboxStatusResponse(); ++ if (statusReply == nullptr) { ++ ERROR("Out of memory"); ++ delete response; ++ return nullptr; ++ } ++ ++ m_rService->PodSandboxStatus(sandboxID, statusReply, error); ++ if (!error.Empty()) { ++ WARN("Object: CRI, Type: Failed to status pod:%s due to %s", sandboxID.c_str(), ++ error.GetMessage().c_str()); ++ } else { ++ *(response->mutable_pod_sandbox_status()) = *(statusReply->mutable_status()); ++ for (auto &containerStatus : statusReply->containers_statuses()) { ++ *(response->add_containers_statuses()) = containerStatus; ++ } ++ } ++ ++ response->set_container_event_type((runtime::v1::ContainerEventType)type); ++ response->set_container_id(containerID); ++ response->set_created_at(util_get_now_time_nanos()); ++ ++ return response; + } + + grpc::Status RuntimeV1RuntimeServiceImpl::Version(grpc::ServerContext *context, +@@ -398,14 +483,12 @@ grpc::Status RuntimeV1RuntimeServiceImpl::PodSandboxStatus(grpc::ServerContext * + + INFO("Event: {Object: CRI, Type: Status Pod: %s}", request->pod_sandbox_id().c_str()); + +- std::unique_ptr<runtime::v1::PodSandboxStatus> podStatus; +- podStatus = m_rService->PodSandboxStatus(request->pod_sandbox_id(), error); +- if (!error.Empty() || !podStatus) { ++ m_rService->PodSandboxStatus(request->pod_sandbox_id(), reply, error); ++ if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to status pod:%s due to %s", request->pod_sandbox_id().c_str(), + error.GetMessage().c_str()); + return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); + } +- *(reply->mutable_status()) = *podStatus; + + INFO("Event: {Object: CRI, Type: Statused Pod: %s}", request->pod_sandbox_id().c_str()); + +@@ -657,3 +740,55 @@ RuntimeV1RuntimeServiceImpl::RuntimeConfig(grpc::ServerContext *context, + + return grpc::Status::OK; + } ++ ++grpc::Status RuntimeV1RuntimeServiceImpl::GetContainerEvents(grpc::ServerContext *context, ++ const runtime::v1::GetEventsRequest *request, ++ grpc::ServerWriter<runtime::v1::ContainerEventResponse> *writer) ++{ ++ Errors error; ++ ++ if (context == nullptr || request == nullptr || writer == nullptr) { ++ ERROR("Invalid input arguments"); ++ return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, "Invalid input arguments"); ++ } ++ ++ if (!m_enablePodEvents) { ++ ERROR("Pod events is not enabled"); ++ return grpc::Status(grpc::StatusCode::UNIMPLEMENTED, "Pod events is not enabled"); ++ } ++ ++ INFO("Event: {Object: CRI, Type: Getting Container Events}"); ++ ++ __isula_auto_subscriber auto sub = mailbox_subscribe(MAILBOX_TOPIC_CRI_CONTAINER); ++ if (sub == nullptr) { ++ ERROR("Object: CRI, Type: Failed to subscribe container events"); ++ return grpc::Status(grpc::StatusCode::UNKNOWN, "Failed to subscribe container events"); ++ } ++ ++ for (;;) { ++ __isula_auto_mailbox_message mailbox_message *msg = NULL; ++ int ret = message_subscriber_pop(sub, &msg); ++ if (ret == 0) { ++ if (msg == nullptr) { ++ // nullptr response indicates eventqueue being shutdown, not need to unscribe now ++ return grpc::Status(grpc::StatusCode::UNKNOWN, "Event queue is shutdown"); ++ } ++ auto *response = static_cast<runtime::v1::ContainerEventResponse *>(msg->data); ++ if (!writer->Write(*response)) { ++ break; ++ } ++ } else if (ret != ETIMEDOUT) { ++ ERROR("Failed to pop message from subscriber"); ++ break; ++ } ++ if (context->IsCancelled()) { ++ INFO("Object: CRI, Type: GetContainerEvents is cancelled"); ++ break; ++ } ++ } ++ ++ mailbox_unsubscribe(MAILBOX_TOPIC_CRI_CONTAINER, sub); ++ INFO("Event: {Object: CRI, Type: Got Container Events}"); ++ ++ return grpc::Status::OK; ++} +diff --git a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h +index 52cc6b99..842d1811 100644 +--- a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h ++++ b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h +@@ -26,9 +26,13 @@ + // Implement of runtime RuntimeService + class RuntimeV1RuntimeServiceImpl : public runtime::v1::RuntimeService::Service { + public: +- void Init(std::string &podSandboxImage, std::shared_ptr<Network::PluginManager> networkPlugin, Errors &err); ++ void Init(std::string &podSandboxImage, std::shared_ptr<Network::PluginManager> networkPlugin, ++ bool enablePodEvents, Errors &err); + void Wait(); + void Shutdown(); ++ auto GenerateCRIContainerEvent(const char *container_id, const char *sandbox_id, runtime::v1::ContainerEventType type) ++ -> runtime::v1::ContainerEventResponse *; ++ + grpc::Status Version(grpc::ServerContext *context, const runtime::v1::VersionRequest *request, + runtime::v1::VersionResponse *reply) override; + +@@ -105,8 +109,13 @@ public: + const runtime::v1::RuntimeConfigRequest *request, + runtime::v1::RuntimeConfigResponse *reply) override; + ++ grpc::Status GetContainerEvents(grpc::ServerContext *context, ++ const runtime::v1::GetEventsRequest *request, ++ grpc::ServerWriter<runtime::v1::ContainerEventResponse> *writer) override; ++ + private: + std::unique_ptr<CRIV1::CRIRuntimeService> m_rService; ++ bool m_enablePodEvents; + }; + + #endif // DAEMON_ENTRY_CONNECT_GRPC_CRI_V1_RUNTIME_RUNTIME_SERVICE_H +diff --git a/src/daemon/entry/connect/grpc/grpc_service.cc b/src/daemon/entry/connect/grpc/grpc_service.cc +index 61e284f3..1d8de922 100644 +--- a/src/daemon/entry/connect/grpc/grpc_service.cc ++++ b/src/daemon/entry/connect/grpc/grpc_service.cc +@@ -108,11 +108,11 @@ public: + + void Shutdown(void) + { +- m_server->Shutdown(); +- +- // call CRI to shutdown stream server ++ // call CRI to shutdown stream server, shutdown cri first to notify events thread to exit + m_criService.Shutdown(); + ++ m_server->Shutdown(); ++ + // Shutdown daemon, this operation should remove socket file. + for (const auto &address : m_socketPath) { + if (address.find(UNIX_SOCKET_PREFIX) == 0) { +diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +index cac5c0ba..e86dafae 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +@@ -1007,208 +1007,9 @@ cleanup: + return contStats; + } + +-void ContainerManagerService::PackContainerImageToStatus( +- container_inspect *inspect, std::unique_ptr<runtime::v1::ContainerStatus> &contStatus, Errors &error) ++std::unique_ptr<runtime::v1::ContainerStatus> ContainerManagerService::ContainerStatus(const std::string &containerID, Errors &error) + { +- if (inspect->config == nullptr) { +- return; +- } +- +- if (inspect->config->image != nullptr) { +- contStatus->mutable_image()->set_image(inspect->config->image); +- } +- +- contStatus->set_image_ref(CRIHelpers::ToPullableImageID(inspect->config->image, inspect->config->image_ref)); +- return; +-} +- +-void ContainerManagerService::UpdateBaseStatusFromInspect( +- container_inspect *inspect, int64_t &createdAt, int64_t &startedAt, int64_t &finishedAt, +- std::unique_ptr<runtime::v1::ContainerStatus> &contStatus) +-{ +- runtime::v1::ContainerState state { runtime::v1::CONTAINER_UNKNOWN }; +- std::string reason; +- std::string message; +- int32_t exitCode { 0 }; +- +- if (inspect->state == nullptr) { +- goto pack_status; +- } +- +- if (inspect->state->running) { +- // Container is running +- state = runtime::v1::CONTAINER_RUNNING; +- } else { +- // Container is not running. +- if (finishedAt != 0) { // Case 1 +- state = runtime::v1::CONTAINER_EXITED; +- if (inspect->state->exit_code == 0) { +- reason = "Completed"; +- } else { +- reason = "Error"; +- } +- } else if (inspect->state->exit_code != 0) { // Case 2 +- state = runtime::v1::CONTAINER_EXITED; +- finishedAt = createdAt; +- startedAt = createdAt; +- reason = "ContainerCannotRun"; +- } else { // Case 3 +- state = runtime::v1::CONTAINER_CREATED; +- } +- if (inspect->state->oom_killed) { +- reason = "OOMKilled"; +- } +- if (inspect->state->error != nullptr) { +- message = inspect->state->error; +- } +- exitCode = (int32_t)inspect->state->exit_code; +- } +- +-pack_status: +- contStatus->set_exit_code(exitCode); +- contStatus->set_state(state); +- contStatus->set_created_at(createdAt); +- contStatus->set_started_at(startedAt); +- contStatus->set_finished_at(finishedAt); +- contStatus->set_reason(reason); +- contStatus->set_message(message); +-} +- +-void ContainerManagerService::PackLabelsToStatus(container_inspect *inspect, +- std::unique_ptr<runtime::v1::ContainerStatus> &contStatus) +-{ +- if (inspect->config == nullptr || inspect->config->labels == nullptr) { +- return; +- } +- CRIHelpers::ExtractLabels(inspect->config->labels, *contStatus->mutable_labels()); +- CRIHelpers::ExtractAnnotations(inspect->config->annotations, *contStatus->mutable_annotations()); +- for (size_t i = 0; i < inspect->config->labels->len; i++) { +- if (strcmp(inspect->config->labels->keys[i], CRIHelpers::Constants::CONTAINER_LOGPATH_LABEL_KEY.c_str()) == 0) { +- contStatus->set_log_path(inspect->config->labels->values[i]); +- break; +- } +- } +-} +- +-void ContainerManagerService::ConvertMountsToStatus(container_inspect *inspect, +- std::unique_ptr<runtime::v1::ContainerStatus> &contStatus) +-{ +- for (size_t i = 0; i < inspect->mounts_len; i++) { +- runtime::v1::Mount *mount = contStatus->add_mounts(); +- mount->set_host_path(inspect->mounts[i]->source); +- mount->set_container_path(inspect->mounts[i]->destination); +- mount->set_readonly(!inspect->mounts[i]->rw); +- if (inspect->mounts[i]->propagation == nullptr || strcmp(inspect->mounts[i]->propagation, "rprivate") == 0) { +- mount->set_propagation(runtime::v1::PROPAGATION_PRIVATE); +- } else if (strcmp(inspect->mounts[i]->propagation, "rslave") == 0) { +- mount->set_propagation(runtime::v1::PROPAGATION_HOST_TO_CONTAINER); +- } else if (strcmp(inspect->mounts[i]->propagation, "rshared") == 0) { +- mount->set_propagation(runtime::v1::PROPAGATION_BIDIRECTIONAL); +- } +- // Note: Can't set SeLinuxRelabel +- } +-} +- +-void ContainerManagerService::ConvertResourcesToStatus(container_inspect *inspect, +- std::unique_ptr<runtime::v1::ContainerStatus> &contStatus) +-{ +- if (inspect->resources == nullptr) { +- return; +- } +- runtime::v1::LinuxContainerResources *resources = contStatus->mutable_resources()->mutable_linux(); +- if (inspect->resources->cpu_shares != 0) { +- resources->set_cpu_shares(inspect->resources->cpu_shares); +- } +- if (inspect->resources->cpu_period != 0) { +- resources->set_cpu_period(inspect->resources->cpu_period); +- } +- if (inspect->resources->cpu_quota != 0) { +- resources->set_cpu_quota(inspect->resources->cpu_quota); +- } +- if (inspect->resources->memory != 0) { +- resources->set_memory_limit_in_bytes(inspect->resources->memory); +- } +- if (inspect->resources->memory_swap != 0) { +- resources->set_memory_swap_limit_in_bytes(inspect->resources->memory_swap); +- } +- for (size_t i = 0; i < inspect->resources->hugetlbs_len; i++) { +- runtime::v1::HugepageLimit *hugepage = resources->add_hugepage_limits(); +- hugepage->set_page_size(inspect->resources->hugetlbs[i]->page_size); +- hugepage->set_limit(inspect->resources->hugetlbs[i]->limit); +- } +- if (inspect->resources->unified != nullptr) { +- for (size_t i = 0; i < inspect->resources->unified->len; i++) { +- auto &resUnified = *(resources->mutable_unified()); +- resUnified[inspect->resources->unified->keys[i]] = inspect->resources->unified->values[i]; +- } +- } +-} +- +-void ContainerManagerService::ContainerStatusToGRPC(container_inspect *inspect, +- std::unique_ptr<runtime::v1::ContainerStatus> &contStatus, +- Errors &error) +-{ +- if (inspect->id != nullptr) { +- contStatus->set_id(inspect->id); +- } +- +- int64_t createdAt {}; +- int64_t startedAt {}; +- int64_t finishedAt {}; +- CRIHelpers::GetContainerTimeStamps(inspect, &createdAt, &startedAt, &finishedAt, error); +- if (error.NotEmpty()) { +- return; +- } +- contStatus->set_created_at(createdAt); +- contStatus->set_started_at(startedAt); +- contStatus->set_finished_at(finishedAt); +- +- PackContainerImageToStatus(inspect, contStatus, error); +- UpdateBaseStatusFromInspect(inspect, createdAt, startedAt, finishedAt, contStatus); +- PackLabelsToStatus(inspect, contStatus); +- CRINamingV1::ParseContainerName(contStatus->annotations(), contStatus->mutable_metadata(), error); +- if (error.NotEmpty()) { +- return; +- } +- ConvertMountsToStatus(inspect, contStatus); +- ConvertResourcesToStatus(inspect, contStatus); +-} +- +-std::unique_ptr<runtime::v1::ContainerStatus> +-ContainerManagerService::ContainerStatus(const std::string &containerID, Errors &error) +-{ +- if (containerID.empty()) { +- error.SetError("Empty container id"); +- return nullptr; +- } +- +- std::string realContainerID = CRIHelpers::GetRealContainerOrSandboxID(m_cb, containerID, false, error); +- if (error.NotEmpty()) { +- ERROR("Failed to find container id %s: %s", containerID.c_str(), error.GetCMessage()); +- error.Errorf("Failed to find container id %s: %s", containerID.c_str(), error.GetCMessage()); +- return nullptr; +- } +- +- container_inspect *inspect = CRIHelpers::InspectContainer(realContainerID, error, false); +- if (error.NotEmpty()) { +- return nullptr; +- } +- if (inspect == nullptr) { +- error.SetError("Get null inspect"); +- return nullptr; +- } +- using ContainerStatusPtr = std::unique_ptr<runtime::v1::ContainerStatus>; +- ContainerStatusPtr contStatus(new (std::nothrow) runtime::v1::ContainerStatus); +- if (contStatus == nullptr) { +- error.SetError("Out of memory"); +- free_container_inspect(inspect); +- return nullptr; +- } +- +- ContainerStatusToGRPC(inspect, contStatus, error); +- +- free_container_inspect(inspect); +- return contStatus; ++ return CRIHelpersV1::GetContainerStatus(m_cb, containerID, error); + } + + void ContainerManagerService::UpdateContainerResources(const std::string &containerID, +diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.h b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.h +index 4e772bda..50f5ed69 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.h ++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.h +@@ -111,19 +111,6 @@ private: + std::unique_ptr<runtime::v1::ContainerStats> &container); + void SetFsUsage(const imagetool_fs_info *fs_usage, int64_t timestamp, + std::unique_ptr<runtime::v1::ContainerStats> &container); +- void ContainerStatusToGRPC(container_inspect *inspect, +- std::unique_ptr<runtime::v1::ContainerStatus> &contStatus, Errors &error); +- void PackContainerImageToStatus(container_inspect *inspect, +- std::unique_ptr<runtime::v1::ContainerStatus> &contStatus, Errors &error); +- void UpdateBaseStatusFromInspect(container_inspect *inspect, int64_t &createdAt, int64_t &startedAt, +- int64_t &finishedAt, +- std::unique_ptr<runtime::v1::ContainerStatus> &contStatus); +- void PackLabelsToStatus(container_inspect *inspect, +- std::unique_ptr<runtime::v1::ContainerStatus> &contStatus); +- void ConvertMountsToStatus(container_inspect *inspect, +- std::unique_ptr<runtime::v1::ContainerStatus> &contStatus); +- void ConvertResourcesToStatus(container_inspect *inspect, +- std::unique_ptr<runtime::v1::ContainerStatus> &contStatus); + void ExecSyncFromGRPC(const std::string &containerID, const google::protobuf::RepeatedPtrField<std::string> &cmd, + int64_t timeout, container_exec_request **request, Errors &error); + auto ValidateExecRequest(const runtime::v1::ExecRequest &req, Errors &error) -> int; +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +index f125e714..4291d8a0 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +@@ -36,6 +36,7 @@ + #include "sandbox_manager.h" + #include "transform.h" + #include "isulad_config.h" ++#include "mailbox.h" + + namespace CRIV1 { + void PodSandboxManagerService::PrepareSandboxData(const runtime::v1::PodSandboxConfig &config, +@@ -302,6 +303,7 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1::PodSandboxConfig + std::string jsonCheckpoint; + std::string network_setting_json; + runtime::v1::PodSandboxConfig copyConfig = config; ++ cri_container_message_t msg = { 0 }; + + // Step 1: Parepare sandbox name, runtime and networkMode + PrepareSandboxData(config, runtimeHandler, sandboxName, runtimeInfo, networkMode, error); +@@ -372,6 +374,11 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1::PodSandboxConfig + goto cleanup_network; + } + ++ msg.container_id = sandbox->GetId().c_str(); ++ msg.sandbox_id = sandbox->GetId().c_str(); ++ msg.type = CRI_CONTAINER_MESSAGE_TYPE_CREATED; ++ mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &msg); ++ + // Step 10: Save network settings json to disk + // Update network settings before start sandbox since sandbox container will use the sandbox key + if (namespace_is_cni(networkMode.c_str())) { +@@ -391,6 +398,9 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1::PodSandboxConfig + return response_id; + } + ++ msg.type = CRI_CONTAINER_MESSAGE_TYPE_STARTED; ++ mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &msg); ++ + return sandbox->GetId(); + + cleanup_network: +@@ -700,6 +710,13 @@ void PodSandboxManagerService::RemovePodSandbox(const std::string &podSandboxID, + ERROR("Failed to delete sandbox %s: %s", podSandboxID.c_str(), error.GetCMessage()); + } + ++ if (error.Empty()) { ++ cri_container_message_t msg = { 0 }; ++ msg.container_id = sandbox->GetId().c_str(); ++ msg.sandbox_id = sandbox->GetId().c_str(); ++ msg.type = CRI_CONTAINER_MESSAGE_TYPE_DELETED; ++ mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &msg); ++ } + } + + auto PodSandboxManagerService::SharesHostNetwork(const container_inspect *inspect) -> runtime::v1::NamespaceMode +@@ -800,10 +817,29 @@ void PodSandboxManagerService::SetSandboxStatusNetwork(std::shared_ptr<sandbox:: + } + } + +-std::unique_ptr<runtime::v1::PodSandboxStatus> +-PodSandboxManagerService::PodSandboxStatus(const std::string &podSandboxID, Errors &error) ++void PodSandboxManagerService::GetContainerStatuses(const std::string &podSandboxID, ++ std::vector<std::unique_ptr<runtime::v1::ContainerStatus>> &containerStatuses, ++ std::vector<std::string> &errors) { ++ auto list_response_wrapper = GetContainerListResponse(podSandboxID, errors); ++ if (list_response_wrapper == nullptr) { ++ return; ++ } ++ ++ auto list_response = list_response_wrapper->get(); ++ // Remove all containers in the sandbox. ++ for (size_t i = 0; i < list_response->containers_len; i++) { ++ Errors stError; ++ containerStatuses.push_back(CRIHelpersV1::GetContainerStatus(m_cb, list_response->containers[i]->id, stError)); ++ if (stError.NotEmpty()) { ++ ERROR("Error get container status: %s: %s", list_response->containers[i]->id, stError.GetCMessage()); ++ errors.push_back(stError.GetMessage()); ++ } ++ } ++} ++ ++std::unique_ptr<runtime::v1::PodSandboxStatus> PodSandboxManagerService::GetPodSandboxStatus(const std::string &podSandboxID, Errors &error) + { +- std::unique_ptr<runtime::v1::PodSandboxStatus> podStatus(new runtime::v1::PodSandboxStatus); ++ std::unique_ptr<runtime::v1::PodSandboxStatus> podStatus(new (std::nothrow) runtime::v1::PodSandboxStatus); + if (podStatus == nullptr) { + ERROR("Out of memory"); + error.SetError("Out of memory"); +@@ -831,6 +867,50 @@ PodSandboxManagerService::PodSandboxStatus(const std::string &podSandboxID, Erro + return podStatus; + } + ++void PodSandboxManagerService::PodSandboxStatus(const std::string &podSandboxID, ++ runtime::v1::PodSandboxStatusResponse *reply, Errors &error) ++{ ++ if (reply == nullptr) { ++ ERROR("Invalid NULL reply"); ++ error.SetError("Invalid NULL reply"); ++ return; ++ } ++ ++ ++ auto podStatus = GetPodSandboxStatus(podSandboxID, error); ++ if (error.NotEmpty()) { ++ ERROR("Failed to get pod sandbox status: %s", error.GetCMessage()); ++ return; ++ } ++ ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(podSandboxID); ++ if (sandbox == nullptr) { ++ ERROR("Failed to find sandbox id %s", podSandboxID.c_str()); ++ error.Errorf("Failed to find sandbox id %s", podSandboxID.c_str()); ++ return; ++ } ++ ++ *(reply->mutable_status()) = *podStatus; ++ ++ ++ if (!m_enablePodEvents) { ++ return; ++ } ++ ++ std::vector<std::unique_ptr<runtime::v1::ContainerStatus>> containerStatuses; ++ std::vector<std::string> errors; ++ GetContainerStatuses(sandbox->GetId(), containerStatuses, errors); ++ if (errors.size() != 0) { ++ error.SetAggregate(errors); ++ return; ++ } ++ ++ for (auto &containerStatus : containerStatuses) { ++ *(reply->add_containers_statuses()) = *containerStatus; ++ } ++ return; ++} ++ + void PodSandboxManagerService::ListPodSandbox(const runtime::v1::PodSandboxFilter &filter, + std::vector<std::unique_ptr<runtime::v1::PodSandbox>> &pods, + Errors &error) +@@ -944,7 +1024,7 @@ void PodSandboxManagerService::GetPodSandboxNetworkMetrics(const std::string &ne + void PodSandboxManagerService::PackagePodSandboxStatsAttributes( + const std::string &id, std::unique_ptr<runtime::v1::PodSandboxStats> &podStatsPtr, Errors &error) + { +- auto status = PodSandboxStatus(id, error); ++ auto status = GetPodSandboxStatus(id, error); + if (error.NotEmpty()) { + return; + } +@@ -1111,8 +1191,8 @@ auto PodSandboxManagerService::PodSandboxStats(const std::string &podSandboxID, + auto &config = sandbox->GetSandboxConfig(); + auto oldStatsRec = sandbox->GetStatsInfo(); + +- auto status = PodSandboxStatus(sandbox->GetId(), tmpErr); +- if (error.NotEmpty()) { ++ auto status = GetPodSandboxStatus(sandbox->GetId(), tmpErr); ++ if (tmpErr.NotEmpty()) { + ERROR("Failed to get podsandbox %s status: %s", sandbox->GetId().c_str(), tmpErr.GetCMessage()); + error.Errorf("Failed to get podsandbox %s status", sandbox->GetId().c_str()); + return nullptr; +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h +index c3d98b8c..3872c4c9 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h +@@ -38,10 +38,11 @@ namespace CRIV1 { + class PodSandboxManagerService { + public: + PodSandboxManagerService(const std::string &podSandboxImage, service_executor_t *cb, +- std::shared_ptr<Network::PluginManager> pluginManager) ++ std::shared_ptr<Network::PluginManager> pluginManager, bool enablePodEvents) + : m_podSandboxImage(podSandboxImage) + , m_cb(cb) + , m_pluginManager(pluginManager) ++ , m_enablePodEvents(enablePodEvents) + { + } + PodSandboxManagerService(const PodSandboxManagerService &) = delete; +@@ -55,8 +56,7 @@ public: + + void RemovePodSandbox(const std::string &podSandboxID, Errors &error); + +- auto PodSandboxStatus(const std::string &podSandboxID, Errors &error) +- -> std::unique_ptr<runtime::v1::PodSandboxStatus>; ++ void PodSandboxStatus(const std::string &podSandboxID, runtime::v1::PodSandboxStatusResponse *reply, Errors &error); + + void ListPodSandbox(const runtime::v1::PodSandboxFilter &filter, + std::vector<std::unique_ptr<runtime::v1::PodSandbox>> &pods, Errors &error); +@@ -129,6 +129,9 @@ private: + std::vector<std::string> &podSandboxIDs, Errors &error); + void ApplySandboxLinuxOptions(const runtime::v1::LinuxPodSandboxConfig &lc, host_config *hc, + container_config *custom_config, Errors &error); ++ auto GetPodSandboxStatus(const std::string &podSandboxID, Errors &error) -> std::unique_ptr<runtime::v1::PodSandboxStatus>; ++ void GetContainerStatuses(const std::string &podSandboxID, std::vector<std::unique_ptr<runtime::v1::ContainerStatus>> &containerStatuses, ++ std::vector<std::string> &errors); + + private: + std::string m_podSandboxImage; +@@ -136,6 +139,7 @@ private: + std::map<std::string, bool> m_networkReady; + service_executor_t *m_cb { nullptr }; + std::shared_ptr<Network::PluginManager> m_pluginManager { nullptr }; ++ bool m_enablePodEvents; + }; + } // namespace CRI + +diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service.h b/src/daemon/entry/cri/v1/v1_cri_runtime_service.h +index 839f6724..4521e3df 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_runtime_service.h ++++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service.h +@@ -70,8 +70,8 @@ public: + + virtual void RemovePodSandbox(const std::string &podSandboxID, Errors &error) = 0; + +- virtual auto PodSandboxStatus(const std::string &podSandboxID, +- Errors &error) -> std::unique_ptr<runtime::v1::PodSandboxStatus> = 0; ++ virtual void PodSandboxStatus(const std::string &podSandboxID, runtime::v1::PodSandboxStatusResponse *reply, ++ Errors &error) = 0; + + virtual void ListPodSandbox(const runtime::v1::PodSandboxFilter &filter, + std::vector<std::unique_ptr<runtime::v1::PodSandbox>> &pods, Errors &error) = 0; +diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc +index aa5ae516..7b40e29d 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc +@@ -19,11 +19,12 @@ + + namespace CRIV1 { + CRIRuntimeServiceImpl::CRIRuntimeServiceImpl(const std::string &podSandboxImage, service_executor_t *cb, +- std::shared_ptr<Network::PluginManager> pluginManager) ++ std::shared_ptr<Network::PluginManager> pluginManager, bool enablePodEvents) + : m_runtimeVersioner(new RuntimeVersionerService(cb)) + , m_containerManager(new ContainerManagerService(cb)) +- , m_podSandboxManager(new PodSandboxManagerService(podSandboxImage, cb, pluginManager)) ++ , m_podSandboxManager(new PodSandboxManagerService(podSandboxImage, cb, pluginManager, enablePodEvents)) + , m_runtimeManager(new RuntimeManagerService(cb, pluginManager)) ++ , m_enablePodEvents(enablePodEvents) + { + } + +@@ -124,10 +125,9 @@ void CRIRuntimeServiceImpl::RemovePodSandbox(const std::string &podSandboxID, Er + m_podSandboxManager->RemovePodSandbox(podSandboxID, error); + } + +-auto CRIRuntimeServiceImpl::PodSandboxStatus(const std::string &podSandboxID, Errors &error) +--> std::unique_ptr<runtime::v1::PodSandboxStatus> ++void CRIRuntimeServiceImpl::PodSandboxStatus(const std::string &podSandboxID, runtime::v1::PodSandboxStatusResponse *reply, Errors &error) + { +- return m_podSandboxManager->PodSandboxStatus(podSandboxID, error); ++ m_podSandboxManager->PodSandboxStatus(podSandboxID, reply, error); + } + + void CRIRuntimeServiceImpl::ListPodSandbox(const runtime::v1::PodSandboxFilter &filter, +diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h +index 0a25749f..6ae59bfa 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h ++++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h +@@ -26,7 +26,8 @@ namespace CRIV1 { + class CRIRuntimeServiceImpl : public CRIRuntimeService { + public: + CRIRuntimeServiceImpl(const std::string &podSandboxImage, service_executor_t *cb, +- std::shared_ptr<Network::PluginManager> pluginManager); ++ std::shared_ptr<Network::PluginManager> pluginManager, ++ bool enablePodEvents); + CRIRuntimeServiceImpl(const CRIRuntimeServiceImpl &) = delete; + auto operator=(const CRIRuntimeServiceImpl &) -> CRIRuntimeServiceImpl & = delete; + virtual ~CRIRuntimeServiceImpl() = default; +@@ -72,8 +73,7 @@ public: + + void RemovePodSandbox(const std::string &podSandboxID, Errors &error) override; + +- auto PodSandboxStatus(const std::string &podSandboxID, Errors &error) +- -> std::unique_ptr<runtime::v1::PodSandboxStatus> override; ++ void PodSandboxStatus(const std::string &podSandboxID, runtime::v1::PodSandboxStatusResponse *reply, Errors &error) override; + + void ListPodSandbox(const runtime::v1::PodSandboxFilter &filter, + std::vector<std::unique_ptr<runtime::v1::PodSandbox>> &pods, Errors &error) override; +@@ -103,6 +103,7 @@ protected: + private: + std::string m_podSandboxImage; + std::shared_ptr<Network::PluginManager> m_pluginManager { nullptr }; ++ bool m_enablePodEvents; + }; + } // namespace CRIV1 + #endif // DAEMON_ENTRY_CRI_V1_CRI_RUNTIME_SERVICE_IMPL_H +diff --git a/src/daemon/executor/container_cb/execution.c b/src/daemon/executor/container_cb/execution.c +index 88c6b354..e5c96628 100644 +--- a/src/daemon/executor/container_cb/execution.c ++++ b/src/daemon/executor/container_cb/execution.c +@@ -62,6 +62,7 @@ + #include "event_type.h" + #include "utils_timestamp.h" + #include "utils_verify.h" ++#include "mailbox.h" + #ifdef ENABLE_NATIVE_NETWORK + #include "service_network_api.h" + +@@ -542,6 +543,9 @@ static int container_start_cb(const container_start_request *request, container_ + container_t *cont = NULL; + int sync_fd = -1; + pthread_t thread_id = 0; ++#ifdef ENABLE_CRI_API_V1 ++ cri_container_message_t message; ++#endif + + DAEMON_CLEAR_ERRMSG(); + +@@ -596,6 +600,15 @@ static int container_start_cb(const container_start_request *request, container_ + EVENT("Event: {Object: %s, Type: Running}", id); + (void)isulad_monitor_send_container_event(id, START, -1, 0, NULL, NULL); + ++#ifdef ENABLE_CRI_API_V1 ++ if (is_container_in_sandbox(cont->common_config->sandbox_info)) { ++ message.container_id = id; ++ message.sandbox_id = cont->common_config->sandbox_info->id; ++ message.type = CRI_CONTAINER_MESSAGE_TYPE_STARTED; ++ mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &message); ++ } ++#endif ++ + pack_response: + handle_start_io_thread_by_cc(cc, sync_fd, thread_id); + delete_daemon_fifos(fifopath, (const char **)fifos); +@@ -1009,6 +1022,9 @@ static int container_delete_cb(const container_delete_request *request, containe + char *name = NULL; + char *id = NULL; + container_t *cont = NULL; ++#ifdef ENABLE_CRI_API_V1 ++ cri_container_message_t message; ++#endif + + DAEMON_CLEAR_ERRMSG(); + if (request == NULL || response == NULL) { +@@ -1063,6 +1079,15 @@ static int container_delete_cb(const container_delete_request *request, containe + + EVENT("Event: {Object: %s, Type: Deleted}", id); + ++#ifdef ENABLE_CRI_API_V1 ++ if (is_container_in_sandbox(cont->common_config->sandbox_info)) { ++ message.container_id = cont->common_config->id; ++ message.sandbox_id = cont->common_config->sandbox_info->id; ++ message.type = CRI_CONTAINER_MESSAGE_TYPE_DELETED; ++ mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &message); ++ } ++#endif ++ + pack_response: + pack_delete_response(*response, cc, id); + container_unref(cont); +diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c +index e00afb68..a9102226 100644 +--- a/src/daemon/executor/container_cb/execution_create.c ++++ b/src/daemon/executor/container_cb/execution_create.c +@@ -62,6 +62,7 @@ + #include "opt_log.h" + #include "runtime_api.h" + #include "id_name_manager.h" ++#include "mailbox.h" + + #ifdef ENABLE_CRI_API_V1 + static bool validate_sandbox_info(const container_sandbox_info *sandbox) +@@ -1389,6 +1390,9 @@ int container_create_cb(const container_create_request *request, container_creat + bool skip_id_name_manage = false; + bool skip_sandbox_key_manage = false; + __isula_auto_sysinfo_t sysinfo_t *sysinfo = NULL; ++#ifdef ENABLE_CRI_API_V1 ++ cri_container_message_t message; ++#endif + + DAEMON_CLEAR_ERRMSG(); + +@@ -1572,6 +1576,14 @@ int container_create_cb(const container_create_request *request, container_creat + + EVENT("Event: {Object: %s, Type: Created %s}", id, name); + (void)isulad_monitor_send_container_event(id, CREATE, -1, 0, NULL, NULL); ++#ifdef ENABLE_CRI_API_V1 ++ if (is_container_in_sandbox(request->sandbox)) { ++ message.container_id = id; ++ message.sandbox_id = request->sandbox->id; ++ message.type = CRI_CONTAINER_MESSAGE_TYPE_CREATED; ++ mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &message); ++ } ++#endif + goto pack_response; + + umount_channel: +diff --git a/src/daemon/mailbox/CMakeLists.txt b/src/daemon/mailbox/CMakeLists.txt +new file mode 100644 +index 00000000..984f9acb +--- /dev/null ++++ b/src/daemon/mailbox/CMakeLists.txt +@@ -0,0 +1,11 @@ ++# get current directory sources files ++aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} mailbox_top_srcs) ++ ++set(MAILBOX_SRCS ++ ${mailbox_top_srcs} ++ PARENT_SCOPE ++ ) ++set(MAILBOX_INCS ++ ${CMAKE_CURRENT_SOURCE_DIR} ++ PARENT_SCOPE ++ ) +\ No newline at end of file +diff --git a/src/daemon/mailbox/mailbox.c b/src/daemon/mailbox/mailbox.c +new file mode 100644 +index 00000000..732b91b9 +--- /dev/null ++++ b/src/daemon/mailbox/mailbox.c +@@ -0,0 +1,167 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide common event definition ++ ******************************************************************************/ ++ ++#include "mailbox.h" ++ ++#include <isula_libutils/log.h> ++ ++#include "message_queue.h" ++#include "mailbox_message.h" ++#include "message_subscriber.h" ++ ++mailbox_topic_handler_t mailbox_topic_handlers[MAILBOX_TOPIC_MAX] = { 0 }; ++ ++static bool mailbox_topic_valid(mailbox_topic topic) { ++ return topic > MAILBOX_TOPIC_INVALID && topic < MAILBOX_TOPIC_MAX; ++} ++ ++static bool mailbox_should_publish(mailbox_topic topic) ++{ ++ if (!mailbox_topic_valid(topic)) { ++ ERROR("Invalid topic %d", topic); ++ return false; ++ } ++ ++ if (!mailbox_topic_handlers[topic].registered) { ++ return false; ++ } ++ ++ if (mailbox_topic_handlers[topic].queue == NULL) { ++ return true; ++ } ++ ++ // for async queues, only publish if anyone subscribe ++ return message_queue_have_subscribers(mailbox_topic_handlers[topic].queue); ++} ++ ++// only register once when iSulad start, no need to free the queue ++int mailbox_register_topic_handler(mailbox_topic topic, message_generator_t generator, void *context, ++ message_release_t release, bool async) ++{ ++ if (!mailbox_topic_valid(topic)) { ++ ERROR("Invalid topic %d", topic); ++ return -1; ++ } ++ ++ if (generator == NULL) { ++ ERROR("Invalid generator for topic %d", topic); ++ return -1; ++ } ++ ++ mailbox_topic_handlers[topic].generator = generator; ++ mailbox_topic_handlers[topic].context = context; ++ mailbox_topic_handlers[topic].release = release; ++ if (async) { ++ mailbox_topic_handlers[topic].queue = message_queue_create(release); ++ if (mailbox_topic_handlers[topic].queue == NULL) { ++ ERROR("Failed to create message queue for topic %d", topic); ++ return -1; ++ } ++ } ++ mailbox_topic_handlers[topic].registered = true; ++ return 0; ++} ++ ++// unregister only when iSulad shutdown, no need to free the queue ++void mailbox_unregister_topic_handler(mailbox_topic topic) ++{ ++ if (!mailbox_topic_valid(topic)) { ++ ERROR("Invalid topic %d", topic); ++ return; ++ } ++ ++ if (mailbox_topic_handlers[topic].queue != NULL) { ++ message_queue_shutdown(mailbox_topic_handlers[topic].queue); ++ } ++ mailbox_topic_handlers[topic].registered = false; ++} ++ ++void mailbox_publish(mailbox_topic topic, void *data) ++{ ++ if (!mailbox_should_publish(topic)) { ++ return; ++ } ++ ++ message_generator_t generator = mailbox_topic_handlers[topic].generator; ++ void *context = mailbox_topic_handlers[topic].context; ++ message_release_t release = mailbox_topic_handlers[topic].release; ++ message_queue *queue = mailbox_topic_handlers[topic].queue; ++ ++ if (generator == NULL) { ++ ERROR("No handler for topic %d", topic); ++ return; ++ } ++ ++ void *middle = generator(context, data); ++ if (middle == NULL) { ++ return; ++ } ++ ++ if (queue != NULL) { ++ mailbox_message *msg = mailbox_message_create(middle, release); ++ if (msg == NULL) { ++ ERROR("Failed to create mailbox message"); ++ if (release) { ++ release(middle); ++ } ++ return; ++ } ++ if (message_queue_publish(queue, msg) != 0) { ++ ERROR("Failed to publish event"); ++ mailbox_message_unref(msg); ++ return; ++ } ++ } ++} ++ ++message_subscriber *mailbox_subscribe(mailbox_topic topic) ++{ ++ if (!mailbox_topic_valid(topic)) { ++ ERROR("Invalid topic %d", topic); ++ return NULL; ++ } ++ ++ if (!mailbox_topic_handlers[topic].registered) { ++ ERROR("Handler for topic %d not registered", topic); ++ return NULL; ++ } ++ ++ if (mailbox_topic_handlers[topic].queue != NULL) { ++ return message_queue_subscribe(mailbox_topic_handlers[topic].queue, ++ mailbox_topic_handlers[topic].release); ++ } ++ ++ // For sync queues, there is no need to subscribe, just return ++ return NULL; ++} ++ ++void mailbox_unsubscribe(mailbox_topic topic, message_subscriber *sub) ++{ ++ if (!mailbox_topic_valid(topic)) { ++ ERROR("Invalid topic %d", topic); ++ return; ++ } ++ ++ if (!mailbox_topic_handlers[topic].registered) { ++ ERROR("Handler for topic %d not registered", topic); ++ return; ++ } ++ ++ if (mailbox_topic_handlers[topic].queue != NULL) { ++ return message_queue_unsubscribe(mailbox_topic_handlers[topic].queue, sub); ++ } ++ ++ return; ++} +diff --git a/src/daemon/mailbox/mailbox.h b/src/daemon/mailbox/mailbox.h +new file mode 100644 +index 00000000..1dc2e934 +--- /dev/null ++++ b/src/daemon/mailbox/mailbox.h +@@ -0,0 +1,82 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide common event definition ++ ******************************************************************************/ ++ ++#ifndef DAEMON_MAILBOX_MAILBOX_H ++#define DAEMON_MAILBOX_MAILBOX_H ++ ++#include "daemon_arguments.h" ++#include "blocking_queue.h" ++#include "message_queue.h" ++#include "message_subscriber.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++typedef enum { ++ MAILBOX_TOPIC_INVALID = -1, ++ MAILBOX_TOPIC_CRI_CONTAINER, ++ MAILBOX_TOPIC_MAX ++} mailbox_topic; ++ ++// for async message, it generates a true message to publish ++// for sync message, it is the callback function to handle the data to publish ++typedef void *(*message_generator_t)(void *, void *); ++// release function of message generated by generator, if any ++typedef void (*message_release_t)(void *); ++ ++typedef struct { ++ // to generate a message ++ message_generator_t generator; ++ // context of handler ++ void *context; ++ // release function of message, if any ++ message_release_t release; ++ // message queue ++ message_queue *queue; ++ // if registered ++ bool registered; ++} mailbox_topic_handler_t; ++ ++typedef enum { ++ CRI_CONTAINER_MESSAGE_TYPE_INVALID = -1, ++ CRI_CONTAINER_MESSAGE_TYPE_CREATED, ++ CRI_CONTAINER_MESSAGE_TYPE_STARTED, ++ CRI_CONTAINER_MESSAGE_TYPE_STOPPED, ++ CRI_CONTAINER_MESSAGE_TYPE_DELETED, ++ CRI_CONTAINER_MESSAGE_TYPE_MAX ++} cri_container_message_type; ++ ++typedef struct { ++ const char *container_id; ++ const char *sandbox_id; ++ cri_container_message_type type; ++} cri_container_message_t; ++ ++int mailbox_register_topic_handler(mailbox_topic topic, message_generator_t handle, void *context, ++ message_release_t release, bool async); ++ ++void mailbox_unregister_topic_handler(mailbox_topic topic); ++ ++void mailbox_publish(mailbox_topic topic, void *data); ++ ++message_subscriber *mailbox_subscribe(mailbox_topic topic); ++ ++void mailbox_unsubscribe(mailbox_topic, message_subscriber *sub); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +diff --git a/src/daemon/mailbox/mailbox_message.c b/src/daemon/mailbox/mailbox_message.c +new file mode 100644 +index 00000000..b16a1bdd +--- /dev/null ++++ b/src/daemon/mailbox/mailbox_message.c +@@ -0,0 +1,94 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide mailbox message definition ++ ******************************************************************************/ ++ ++#include "mailbox_message.h" ++ ++#include <isula_libutils/log.h> ++ ++#include "utils.h" ++ ++// Once the create succeeds, the ownership is transferred to the mailbox_message. ++mailbox_message *mailbox_message_create(void *data, void (*destroy)(void *)) { ++ __isula_auto_free mailbox_message *msg = NULL; ++ msg = util_common_calloc_s(sizeof(mailbox_message)); ++ if (msg == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ msg->data = data; ++ msg->destroy = destroy; ++ msg->ref_count = 1; ++ ++ if (pthread_mutex_init(&msg->lock, NULL) != 0) { ++ ERROR("Failed to init mutex"); ++ return NULL; ++ } ++ ++ return isula_transfer_ptr(msg); ++} ++ ++int mailbox_message_ref(mailbox_message *dest) { ++ __isula_auto_pm_unlock pthread_mutex_t *lock = NULL; ++ if (dest == NULL) { ++ ERROR("Invalid mailbox_message"); ++ return -1; ++ } ++ ++ if (pthread_mutex_lock(&dest->lock) != 0) { ++ ERROR("Failed to lock mutex"); ++ return -1; ++ } ++ lock = &dest->lock; ++ ++ if (dest->ref_count == INT_MAX) { ++ ERROR("Reference count overflow"); ++ return -1; ++ } ++ ++ dest->ref_count++; ++ ++ return 0; ++} ++ ++void mailbox_message_unref(mailbox_message *dest) { ++ __isula_auto_pm_unlock pthread_mutex_t *lock = NULL; ++ if (dest == NULL) { ++ return; ++ } ++ ++ if (pthread_mutex_lock(&dest->lock) != 0) { ++ ERROR("Failed to lock mutex"); ++ return; ++ } ++ lock = &dest->lock; ++ ++ if (dest->ref_count == 0) { ++ ERROR("Reference count underflow, should not reach here"); ++ return; ++ } ++ ++ dest->ref_count--; ++ if (dest->ref_count == 0) { ++ if (dest->destroy) { ++ dest->destroy(dest->data); ++ } ++ lock = NULL; ++ (void)pthread_mutex_unlock(&dest->lock); ++ (void)pthread_mutex_destroy(&dest->lock); ++ free(dest); ++ } ++ return; ++} +diff --git a/src/daemon/mailbox/mailbox_message.h b/src/daemon/mailbox/mailbox_message.h +new file mode 100644 +index 00000000..39e40b70 +--- /dev/null ++++ b/src/daemon/mailbox/mailbox_message.h +@@ -0,0 +1,50 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide ref counted ptr definition ++ ******************************************************************************/ ++ ++#ifndef DAEMON_MAILBOX_MAILBOX_MESSAGE_H ++#define DAEMON_MAILBOX_MAILBOX_MESSAGE_H ++ ++#include <pthread.h> ++#include <stdbool.h> ++ ++#include <isula_libutils/auto_cleanup.h> ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++typedef struct mailbox_message { ++ void *data; ++ size_t ref_count; ++ pthread_mutex_t lock; ++ void (*destroy)(void *); ++} mailbox_message; ++ ++mailbox_message *mailbox_message_create(void *ptr, void (*destroy)(void *)); ++ ++int mailbox_message_ref(mailbox_message *p); ++ ++void mailbox_message_unref(mailbox_message *p); ++ ++// define auto free function callback for mailbox_message ++define_auto_cleanup_callback(mailbox_message_unref, mailbox_message); ++// define auto free macro for char * ++#define __isula_auto_mailbox_message auto_cleanup_tag(mailbox_message_unref) ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +diff --git a/src/daemon/mailbox/message_queue.c b/src/daemon/mailbox/message_queue.c +new file mode 100644 +index 00000000..7fe044f2 +--- /dev/null ++++ b/src/daemon/mailbox/message_queue.c +@@ -0,0 +1,234 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide message queue definition ++ ******************************************************************************/ ++ ++#include "message_queue.h" ++ ++#include <sys/prctl.h> ++#include <isula_libutils/log.h> ++ ++#include "utils.h" ++ ++// default set subscriber timeout to 1000ms, maybe could be configured later ++const int64_t subscribe_timeout = 1000; ++ ++static void message_queue_subscriber_free(void *key, void *val) ++{ ++ return; ++} ++ ++static void *message_queue_thread(void *arg) ++{ ++ int ret = 0; ++ ++ ret = pthread_detach(pthread_self()); ++ if (ret != 0) { ++ CRIT("Set thread detach fail"); ++ return NULL; ++ } ++ ++ prctl(PR_SET_NAME, "Message Queue"); ++ ++ message_queue *mq = (message_queue *)arg; ++ if (mq == NULL) { ++ ERROR("Invalid argument"); ++ return NULL; ++ } ++ ++ for (;;) { ++ void *data = NULL; ++ if (blocking_queue_pop(mq->messages, &data) != 0) { ++ ERROR("Fail to get message, message queue thread exit"); ++ break; ++ } ++ ++ __isula_auto_mailbox_message mailbox_message *msg = (mailbox_message *)data; ++ // an empty msg indicates shutdown ++ if (pthread_rwlock_rdlock(&mq->rwlock) != 0) { ++ ERROR("Failed to lock rwlock"); ++ continue; ++ } ++ ++ bool should_shutdown = (msg == NULL); ++ map_itor *itor = map_itor_new(mq->subscribers); ++ if (itor == NULL) { ++ ERROR("Out of memory"); ++ if (pthread_rwlock_unlock(&mq->rwlock) != 0) { ++ ERROR("Failed to lock rwlock"); ++ } ++ break; ++ } ++ ++ for (; map_itor_valid(itor); map_itor_next(itor)) { ++ void *sub = map_itor_key(itor); ++ if (should_shutdown) { ++ message_subscriber_shutdown(sub); ++ } else { ++ if (message_subscriber_push(sub, msg) != 0) { ++ ERROR("Failed to push event to subscriber"); ++ } ++ } ++ } ++ map_itor_free(itor); ++ ++ if (pthread_rwlock_unlock(&mq->rwlock) != 0) { ++ ERROR("Failed to unlock rwlock"); ++ } ++ ++ // if msg is NULL, it is a shutdown signal ++ if (should_shutdown) { ++ break; ++ } ++ } ++ ++ return NULL; ++} ++ ++message_queue *message_queue_create(void (*release)(void *)) ++{ ++ __isula_auto_free message_queue *mq = NULL; ++ __isula_auto_blocking_queue blocking_queue *bq = NULL; ++ pthread_t message_queue_tid; ++ mq = util_common_calloc_s(sizeof(message_queue)); ++ if (mq == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ bq = blocking_queue_create(BLOCKING_QUEUE_NO_TIMEOUT, release); ++ if (bq == NULL) { ++ ERROR("Failed to create events queue"); ++ return NULL; ++ } ++ ++ mq->subscribers = map_new(MAP_PTR_INT, MAP_DEFAULT_CMP_FUNC, message_queue_subscriber_free); ++ if (mq->subscribers == NULL) { ++ ERROR("Failed to create subscribers map"); ++ return NULL; ++ } ++ ++ if (pthread_rwlock_init(&mq->rwlock, NULL) != 0) { ++ ERROR("Failed to init rwlock"); ++ map_free(mq->subscribers); ++ return NULL; ++ } ++ ++ if (pthread_create(&message_queue_tid, NULL, message_queue_thread, mq) != 0) { ++ ERROR("Failed to create message queue thread"); ++ pthread_rwlock_destroy(&mq->rwlock); ++ map_free(mq->subscribers); ++ return NULL; ++ } ++ ++ mq->messages = isula_transfer_ptr(bq); ++ return isula_transfer_ptr(mq); ++} ++ ++// message queue should be global value, it will be destroyed when daemon exit ++void message_queue_shutdown(message_queue *mq) ++{ ++ if (mq == NULL) { ++ return; ++ } ++ ++ blocking_queue_clear(mq->messages); ++ ++ // push a nullptr to notify the thread to exit ++ if (blocking_queue_push(mq->messages, NULL) != 0) { ++ ERROR("Failed to push nullptr to message queue"); ++ } ++} ++ ++message_subscriber *message_queue_subscribe(message_queue *mq, void (*release)(void *)) ++{ ++ __isula_auto_subscriber message_subscriber *sub = NULL; ++ __isula_auto_prw_unlock pthread_rwlock_t *lock = NULL; ++ int val = 0; ++ if (mq == NULL) { ++ ERROR("Invalid argument"); ++ return NULL; ++ } ++ ++ sub = message_subscriber_create(subscribe_timeout, release); ++ if (sub == NULL) { ++ ERROR("Failed to create subscriber"); ++ return NULL; ++ } ++ ++ if (pthread_rwlock_wrlock(&mq->rwlock) != 0) { ++ ERROR("Failed to lock rwlock"); ++ return NULL; ++ } ++ lock = &mq->rwlock; ++ ++ if (map_insert(mq->subscribers, sub, (void *)&val) == false) { ++ ERROR("Failed to insert subscriber"); ++ return NULL; ++ } ++ ++ return isula_transfer_ptr(sub); ++} ++ ++void message_queue_unsubscribe(message_queue *mq, message_subscriber *sub) ++{ ++ __isula_auto_prw_unlock pthread_rwlock_t *lock = NULL; ++ if (mq == NULL) { ++ ERROR("Invalid argument"); ++ return; ++ } ++ ++ if (pthread_rwlock_wrlock(&mq->rwlock) != 0) { ++ ERROR("Failed to lock rwlock"); ++ return; ++ } ++ lock = &mq->rwlock; ++ ++ if (map_remove(mq->subscribers, sub) == false) { ++ ERROR("Failed to remove subscriber"); ++ return; ++ } ++ ++ return; ++} ++ ++int message_queue_publish(message_queue *mq, mailbox_message *msg) ++{ ++ if (mq == NULL || msg == NULL) { ++ ERROR("Invalid argument"); ++ return -1; ++ } ++ ++ if (blocking_queue_push(mq->messages, msg) != 0) { ++ ERROR("Failed to push message"); ++ return -1; ++ } ++ return 0; ++} ++ ++bool message_queue_have_subscribers(message_queue *mq) ++{ ++ __isula_auto_prw_unlock pthread_rwlock_t *lock = NULL; ++ if (mq == NULL) { ++ ERROR("Invalid argument"); ++ return false; ++ } ++ ++ if (pthread_rwlock_wrlock(&mq->rwlock) != 0) { ++ ERROR("Failed to lock rwlock"); ++ return false; ++ } ++ lock = &mq->rwlock; ++ ++ return map_size(mq->subscribers) > 0; ++} +diff --git a/src/daemon/mailbox/message_queue.h b/src/daemon/mailbox/message_queue.h +new file mode 100644 +index 00000000..7905840f +--- /dev/null ++++ b/src/daemon/mailbox/message_queue.h +@@ -0,0 +1,57 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide message queue definition ++ ******************************************************************************/ ++ ++#ifndef DAEMON_MESSAGE_MESSAGE_QUEUE_H ++#define DAEMON_MESSAGE_MESSAGE_QUEUE_H ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++#include <pthread.h> ++ ++#include "blocking_queue.h" ++#include "mailbox_message.h" ++#include "map.h" ++#include "message_subscriber.h" ++ ++typedef struct message_queue { ++ blocking_queue *messages; ++ ++ // lock for set of subscribers ++ pthread_rwlock_t rwlock; ++ ++ map_t *subscribers; ++ ++ int64_t sub_timeout; ++} message_queue; ++ ++message_queue *message_queue_create(void (*release)(void *)); ++ ++void message_queue_shutdown(message_queue *mq); ++ ++message_subscriber *message_queue_subscribe(message_queue *mq, void (*release)(void *)); ++ ++void message_queue_unsubscribe(message_queue *mq, message_subscriber *sub); ++ ++int message_queue_publish(message_queue *mq, mailbox_message *msg); ++ ++bool message_queue_have_subscribers(message_queue *mq); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +diff --git a/src/daemon/mailbox/message_subscriber.c b/src/daemon/mailbox/message_subscriber.c +new file mode 100644 +index 00000000..8ef3cb58 +--- /dev/null ++++ b/src/daemon/mailbox/message_subscriber.c +@@ -0,0 +1,85 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide message subscriber definition ++ ******************************************************************************/ ++ ++#include "message_subscriber.h" ++ ++#include <isula_libutils/log.h> ++ ++#include "utils.h" ++ ++message_subscriber *message_subscriber_create(int64_t timeout, void (*release)(void *)) ++{ ++ message_subscriber *sub = (message_subscriber *)util_common_calloc_s(sizeof(message_subscriber)); ++ if (sub == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ sub->queue = blocking_queue_create(timeout, release); ++ if (sub->queue == NULL) { ++ ERROR("Failed to create blocking queue"); ++ free(sub); ++ return NULL; ++ } ++ return sub; ++} ++ ++int message_subscriber_push(message_subscriber *sub, mailbox_message *msg) ++{ ++ if (sub == NULL || msg == NULL) { ++ ERROR("Invalid argument"); ++ return -1; ++ } ++ ++ if (mailbox_message_ref(msg) != 0) { ++ ERROR("Failed to get message"); ++ return -1; ++ } ++ ++ if (blocking_queue_push(sub->queue, msg) != 0) { ++ ERROR("Failed to push message to queue"); ++ mailbox_message_unref(msg); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++int message_subscriber_pop(message_subscriber *sub, mailbox_message **msg) ++{ ++ if (sub == NULL) { ++ ERROR("Invalid argument"); ++ return -1; ++ } ++ return blocking_queue_pop(sub->queue, (void **)msg); ++} ++ ++void message_subscriber_shutdown(message_subscriber *sub) ++{ ++ if (sub == NULL) { ++ return; ++ } ++ ++ blocking_queue_clear(sub->queue); ++ (void)blocking_queue_push(sub->queue, NULL); ++} ++ ++void message_subscriber_destroy(message_subscriber *sub) ++{ ++ if (sub == NULL) { ++ return; ++ } ++ blocking_queue_destroy(sub->queue); ++ free(sub); ++} +diff --git a/src/daemon/mailbox/message_subscriber.h b/src/daemon/mailbox/message_subscriber.h +new file mode 100644 +index 00000000..de4574d9 +--- /dev/null ++++ b/src/daemon/mailbox/message_subscriber.h +@@ -0,0 +1,41 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide message subscriber definition ++ ******************************************************************************/ ++ ++#ifndef SRC_DAEMON_MAILBOX_MESSAGE_SUBSCRIBER_H ++#define SRC_DAEMON_MAILBOX_MESSAGE_SUBSCRIBER_H ++ ++#include "blocking_queue.h" ++#include "mailbox_message.h" ++ ++typedef struct { ++ blocking_queue *queue; ++} message_subscriber; ++ ++message_subscriber *message_subscriber_create(int64_t timeout, void (*release)(void *)); ++ ++void message_subscriber_shutdown(message_subscriber *sub); ++ ++void message_subscriber_destroy(message_subscriber *sub); ++ ++int message_subscriber_push(message_subscriber *sub, mailbox_message *msg); ++ ++int message_subscriber_pop(message_subscriber *sub, mailbox_message **msg); ++ ++// define auto free function callback for blocking queue ++define_auto_cleanup_callback(message_subscriber_destroy, message_subscriber); ++// define auto free macro for blocking queue ++#define __isula_auto_subscriber auto_cleanup_tag(message_subscriber_destroy) ++ ++#endif +diff --git a/src/daemon/modules/api/container_api.h b/src/daemon/modules/api/container_api.h +index 830fd696..55c59980 100644 +--- a/src/daemon/modules/api/container_api.h ++++ b/src/daemon/modules/api/container_api.h +@@ -289,6 +289,11 @@ static inline bool is_sandbox_container(container_sandbox_info *sandbox) + { + return sandbox != NULL && sandbox->is_sandbox_container; + } ++ ++static inline bool is_container_in_sandbox(container_sandbox_info *sandbox) ++{ ++ return sandbox != NULL && !sandbox->is_sandbox_container; ++} + #endif + + #if defined(__cplusplus) || defined(c_plusplus) +diff --git a/src/daemon/modules/container/supervisor/supervisor.c b/src/daemon/modules/container/supervisor/supervisor.c +index 1b7da383..83d46268 100644 +--- a/src/daemon/modules/container/supervisor/supervisor.c ++++ b/src/daemon/modules/container/supervisor/supervisor.c +@@ -38,6 +38,7 @@ + #include "container_api.h" + #include "event_type.h" + #include "utils_file.h" ++#include "mailbox.h" + #ifdef ENABLE_CRI_API_V1 + #include "sandbox_ops.h" + #endif +@@ -51,6 +52,7 @@ struct supervisor_handler_data { + int fd; + int exit_code; + char *name; ++ char *sandbox_name; + char *runtime; + bool is_sandbox_container; + pid_ppid_info_t pid_info; +@@ -152,6 +154,9 @@ static void supervisor_handler_data_free(struct supervisor_handler_data *data) + free(data->name); + data->name = NULL; + ++ free(data->sandbox_name); ++ data->sandbox_name = NULL; ++ + free(data->runtime); + data->runtime = NULL; + +@@ -172,6 +177,9 @@ static void *clean_resources_thread(void *arg) + pid_t pid = data->pid_info.pid; + int retry_count = 0; + int max_retry = 10; ++#ifdef ENABLE_CRI_API_V1 ++ cri_container_message_t msg; ++#endif + + ret = pthread_detach(pthread_self()); + if (ret != 0) { +@@ -218,6 +226,13 @@ retry: + (void)isulad_monitor_send_container_event(name, STOPPED, (int)pid, data->exit_code, NULL, NULL); + + #ifdef ENABLE_CRI_API_V1 ++ if (data->sandbox_name) { ++ msg.container_id = name; ++ msg.sandbox_id = data->sandbox_name; ++ msg.type = CRI_CONTAINER_MESSAGE_TYPE_STOPPED; ++ mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &msg); ++ } ++ + if (data->is_sandbox_container) { + if (sandbox_on_sandbox_exit(name, data->exit_code) < 0) { + ERROR("Failed to handle sandbox %s exit", name); +@@ -329,6 +344,9 @@ int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const p + data->runtime = util_strdup_s(cont->runtime); + #ifdef ENABLE_CRI_API_V1 + data->is_sandbox_container = is_sandbox_container(cont->common_config->sandbox_info); ++ if (is_container_in_sandbox(cont->common_config->sandbox_info)) { ++ data->sandbox_name = util_strdup_s(cont->common_config->sandbox_info->id); ++ } + #endif + data->pid_info.pid = pid_info->pid; + data->pid_info.start_time = pid_info->start_time; +diff --git a/src/daemon/sandbox/sandbox.cc b/src/daemon/sandbox/sandbox.cc +index 7b6496ed..c70116c1 100644 +--- a/src/daemon/sandbox/sandbox.cc ++++ b/src/daemon/sandbox/sandbox.cc +@@ -37,6 +37,7 @@ + #include "cxxutils.h" + #include "controller_manager.h" + #include "utils_timestamp.h" ++#include "mailbox.h" + + #define SANDBOX_READY_STATE_STR "SANDBOX_READY" + #define SANDBOX_NOTREADY_STATE_STR "SANDBOX_NOTREADY" +@@ -527,6 +528,14 @@ void Sandbox::OnSandboxExit(const ControllerExitInfo &exitInfo) + if (!SaveState(error)) { + ERROR("Failed to save sandbox state, %s", m_id.c_str()); + } ++ ++ if (error.Empty()) { ++ cri_container_message_t msg = { 0 }; ++ msg.container_id = GetId().c_str(); ++ msg.sandbox_id = GetId().c_str(); ++ msg.type = CRI_CONTAINER_MESSAGE_TYPE_STOPPED; ++ mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &msg); ++ } + } + + auto Sandbox::UpdateStatus(Errors &error) -> bool +diff --git a/src/utils/cutils/blocking_queue.c b/src/utils/cutils/blocking_queue.c +new file mode 100644 +index 00000000..7c9c5f50 +--- /dev/null ++++ b/src/utils/cutils/blocking_queue.c +@@ -0,0 +1,185 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide blocking queue definition ++ ******************************************************************************/ ++ ++#include "blocking_queue.h" ++ ++#include <pthread.h> ++#include <time.h> ++#include <isula_libutils/log.h> ++ ++#include "utils.h" ++#include "utils_timestamp.h" ++ ++// create blocking queue with timeout(ms), if timeout < 0, then with no timeout ++blocking_queue *blocking_queue_create(int64_t timeout, void (*release)(void *)) ++{ ++ __isula_auto_free blocking_queue *queue = NULL; ++ __isula_auto_free blocking_node *node = NULL; ++ queue = (blocking_queue *)util_common_calloc_s(sizeof(blocking_queue)); ++ if (queue == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ node = (blocking_node *)util_common_calloc_s(sizeof(blocking_node)); ++ if (node == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ if (pthread_mutex_init(&queue->lock, NULL) != 0) { ++ ERROR("Failed to init mutex"); ++ return NULL; ++ } ++ ++ if (pthread_cond_init(&queue->not_empty, NULL) != 0) { ++ ERROR("Failed to init cond"); ++ (void)pthread_mutex_destroy(&queue->lock); ++ return NULL; ++ } ++ ++ queue->head = node; ++ queue->tail = node; ++ node = NULL; ++ queue->release = release; ++ ++ if (timeout >= 0) { ++ queue->timeout.tv_sec = timeout / (Time_Second / Time_Milli); ++ queue->timeout.tv_nsec = (timeout % (Time_Second / Time_Milli) ) * Time_Milli; ++ } else { ++ queue->timeout.tv_sec = -1; ++ } ++ ++ return isula_transfer_ptr(queue); ++} ++ ++int blocking_queue_push(blocking_queue *queue, void *data) ++{ ++ __isula_auto_free blocking_node *new_node = NULL; ++ __isula_auto_pm_unlock pthread_mutex_t *lock = NULL; ++ if (queue == NULL) { ++ ERROR("Invalid NULL arguments"); ++ return -1; ++ } ++ ++ new_node = (blocking_node *)util_common_calloc_s(sizeof(blocking_node)); ++ if (new_node == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ new_node->data = data; ++ new_node->next = NULL; ++ ++ if (pthread_mutex_lock(&queue->lock) != 0) { ++ ERROR("Failed to lock mutex"); ++ return -1; ++ } ++ lock = &queue->lock; ++ ++ queue->tail->next = new_node; ++ queue->tail = new_node; ++ new_node = NULL; ++ ++ if (pthread_cond_broadcast(&queue->not_empty) != 0) { ++ ERROR("Failed to broadcast cond"); ++ } ++ ++ return 0; ++} ++ ++int blocking_queue_pop(blocking_queue *queue, void **data) { ++ if (queue == NULL || data == NULL) { ++ ERROR("Invalid NULL arguments"); ++ return -1; ++ } ++ ++ __isula_auto_pm_unlock pthread_mutex_t *lock = NULL; ++ if (pthread_mutex_lock(&queue->lock) != 0) { ++ ERROR("Failed to lock mutex"); ++ return -1; ++ } ++ lock = &queue->lock; ++ ++ while (queue->head->next == NULL) { ++ if (queue->timeout.tv_sec >= 0) { ++ int ret = pthread_cond_timedwait(&queue->not_empty, &queue->lock, &queue->timeout); ++ if (ret != 0) { ++ if (ret != ETIMEDOUT) { ++ ERROR("Failed to wait cond"); ++ } ++ return ret; ++ } ++ } else { ++ int ret = pthread_cond_wait(&queue->not_empty, &queue->lock); ++ if (ret != 0) { ++ ERROR("Failed to wait cond"); ++ return ret; ++ } ++ } ++ } ++ ++ blocking_node *old_head = queue->head; ++ blocking_node *new_head = old_head->next; ++ *data = new_head->data; ++ queue->head = new_head; ++ ++ free(old_head); ++ return 0; ++} ++ ++void blocking_queue_clear(blocking_queue *queue) ++{ ++ if (queue == NULL) { ++ return; ++ } ++ ++ __isula_auto_pm_unlock pthread_mutex_t *lock = NULL; ++ // clear all nodes in queue ++ if (queue == NULL) { ++ ERROR("Invalid NULL arguments"); ++ return; ++ } ++ ++ if (pthread_mutex_lock(&queue->lock) != 0) { ++ ERROR("Failed to lock mutex"); ++ return; ++ } ++ lock = &queue->lock; ++ ++ while (queue->head->next != NULL) { ++ blocking_node *old_head = queue->head; ++ blocking_node *new_head = old_head->next; ++ if (queue->release) { ++ queue->release(old_head->data); ++ } ++ free(old_head); ++ queue->head = new_head; ++ } ++} ++ ++// ensure there is no other thread executing enqueue or dequeue operation ++void blocking_queue_destroy(blocking_queue *queue) ++{ ++ if (queue == NULL) { ++ return; ++ } ++ ++ blocking_queue_clear(queue); ++ ++ (void)pthread_mutex_destroy(&queue->lock); ++ ++ (void)pthread_cond_destroy(&queue->not_empty); ++ ++ free(queue); ++} +diff --git a/src/utils/cutils/blocking_queue.h b/src/utils/cutils/blocking_queue.h +new file mode 100644 +index 00000000..1c52a9d3 +--- /dev/null ++++ b/src/utils/cutils/blocking_queue.h +@@ -0,0 +1,66 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide blocking queue definition ++ ******************************************************************************/ ++ ++#ifndef DAEMON_UTILS_CUTILS_BLOCKING_QUEUE_H ++#define DAEMON_UTILS_CUTILS_BLOCKING_QUEUE_H ++ ++#include <pthread.h> ++#include <time.h> ++#include <isula_libutils/auto_cleanup.h> ++ ++#include "utils_timestamp.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++#define BLOCKING_QUEUE_NO_TIMEOUT -1 ++ ++typedef struct blocking_node { ++ void *data; ++ struct blocking_node *next; ++} blocking_node; ++ ++typedef struct blocking_queue { ++ blocking_node *head; ++ blocking_node *tail; ++ pthread_mutex_t lock; ++ struct timespec timeout; ++ pthread_cond_t not_empty; ++ void (*release)(void *); ++} blocking_queue; ++ ++// create blocking queue with timeout(ms), if timeout < 0, then with no timeout ++blocking_queue *blocking_queue_create(int64_t timeout, void (*release)(void *)); ++ ++int blocking_queue_push(blocking_queue *queue, void *data); ++ ++int blocking_queue_pop(blocking_queue *queue, void **data); ++ ++void blocking_queue_clear(blocking_queue *queue); ++ ++// ensure there is no other thread executing enqueue or dequeue operation ++void blocking_queue_destroy(blocking_queue *queue); ++ ++// define auto free function callback for blocking queue ++define_auto_cleanup_callback(blocking_queue_destroy, blocking_queue); ++// define auto free macro for blocking queue ++#define __isula_auto_blocking_queue auto_cleanup_tag(blocking_queue_destroy) ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +diff --git a/test/mocks/mailbox_mock.cc b/test/mocks/mailbox_mock.cc +new file mode 100644 +index 00000000..601b804e +--- /dev/null ++++ b/test/mocks/mailbox_mock.cc +@@ -0,0 +1,30 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2022. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-04-02 ++ * Description: mailbox mock ++ ******************************************************************************/ ++ ++#include "mailbox_mock.h" ++ ++MockMailbox *g_mailbox_mock = nullptr; ++ ++void Mailbox_SetMock(MockMailbox* mock) ++{ ++ g_mailbox_mock = mock; ++} ++ ++void mailbox_publish(mailbox_topic topic, void *data) ++{ ++ if (g_mailbox_mock != nullptr) { ++ g_mailbox_mock->MailboxPublish(topic, data); ++ } ++} +diff --git a/test/mocks/mailbox_mock.h b/test/mocks/mailbox_mock.h +new file mode 100644 +index 00000000..ce48f0fc +--- /dev/null ++++ b/test/mocks/mailbox_mock.h +@@ -0,0 +1,30 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2022. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-04-02 ++ * Description: mailbox mock ++ ******************************************************************************/ ++ ++#ifndef _ISULAD_TEST_MOCKS_MAILBOX_MOCK_H ++#define _ISULAD_TEST_MOCKS_MAILBOX_MOCK_H ++ ++#include <gmock/gmock.h> ++#include "mailbox.h" ++ ++class MockMailbox { ++public: ++ virtual ~MockMailbox() = default; ++ MOCK_METHOD2(MailboxPublish, void(mailbox_topic topic, void *data)); ++}; ++ ++void Mailbox_SetMock(MockMailbox* mock); ++ ++#endif +diff --git a/test/sandbox/controller/shim/CMakeLists.txt b/test/sandbox/controller/shim/CMakeLists.txt +index 6423bb80..26a66e51 100644 +--- a/test/sandbox/controller/shim/CMakeLists.txt ++++ b/test/sandbox/controller/shim/CMakeLists.txt +@@ -7,6 +7,7 @@ add_executable(${EXE} + ${CMAKE_BINARY_DIR}/grpc/src/api/services/cri/gogo.pb.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cri/v1/v1_cri_helpers.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cri/v1/v1_cri_security_context.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cri/v1/v1_naming.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cri/checkpoint_handler.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cri/cri_constants.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cri/cri_helpers.cc +diff --git a/test/sandbox/sandbox/CMakeLists.txt b/test/sandbox/sandbox/CMakeLists.txt +index 138d4d8d..2a35388f 100644 +--- a/test/sandbox/sandbox/CMakeLists.txt ++++ b/test/sandbox/sandbox/CMakeLists.txt +@@ -23,6 +23,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/grpc_sandboxer_client_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/controller_stub_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/shim_controller_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/mailbox_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c + ${CMAKE_CURRENT_SOURCE_DIR}/sandbox_ut.cc) + +@@ -33,6 +34,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/entry/cri + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/executor ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/mailbox + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/controller + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/controller/shim + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/controller/sandboxer +-- +2.34.1 + diff --git a/0041-fix-cpurt-init-bug-for-systemd-cgroup.patch b/0041-fix-cpurt-init-bug-for-systemd-cgroup.patch new file mode 100644 index 0000000..1f5e9eb --- /dev/null +++ b/0041-fix-cpurt-init-bug-for-systemd-cgroup.patch @@ -0,0 +1,74 @@ +From fe11b34a3c2843ea2198b310160b182d63aeb63b Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Tue, 2 Apr 2024 11:22:09 +0800 +Subject: [PATCH 41/69] fix cpurt init bug for systemd-cgroup + +Signed-off-by: jikai <jikai11@huawei.com> +--- + src/daemon/common/cgroup/cgroup.c | 13 +++++++------ + src/daemon/executor/container_cb/execution.c | 13 +++++++------ + 2 files changed, 14 insertions(+), 12 deletions(-) + +diff --git a/src/daemon/common/cgroup/cgroup.c b/src/daemon/common/cgroup/cgroup.c +index d3f1445a..007dbb70 100644 +--- a/src/daemon/common/cgroup/cgroup.c ++++ b/src/daemon/common/cgroup/cgroup.c +@@ -146,17 +146,18 @@ char *common_convert_cgroup_path(const char *cgroup_path) + return NULL; + } + +- // for cgroup fs cgroup path, return directly +- if (!util_has_suffix(cgroup_path, ".slice")) { +- return util_strdup_s(cgroup_path); +- } +- + // for systemd cgroup, cgroup_path should have the form slice:prefix:id, + // convert it to a true path, such as from test-a.slice:isulad:id + // to test.slice/test-a.slice/isulad-id.scope + arr = util_string_split_n(cgroup_path, ':', 3); + if (arr == NULL || util_array_len((const char **)arr) != 3) { +- ERROR("Invalid systemd cgroup parent"); ++ // not a systemd cgroup, return cgroup path directly ++ return util_strdup_s(cgroup_path); ++ } ++ ++ // for cgroup fs cgroup path, return directly ++ if (!util_has_suffix(arr[0], ".slice")) { ++ ERROR("Invalid systemd cgroup path: %s", cgroup_path); + return NULL; + } + +diff --git a/src/daemon/executor/container_cb/execution.c b/src/daemon/executor/container_cb/execution.c +index 88c6b354..4bf3621d 100644 +--- a/src/daemon/executor/container_cb/execution.c ++++ b/src/daemon/executor/container_cb/execution.c +@@ -435,11 +435,12 @@ static int cpurt_controller_init(const char *id, const host_config *host_spec) + } + + if (conf_get_systemd_cgroup()) { +- // currently it is the same as docker, yet it is unclear that +- // if systemd cgroup is used and cgroup parent is set to a slice rather than system.slice +- // should iSulad set cpu.rt_runtime_us and cpu.rt_period_us for the parent path? +- // in fact, even if system.slice is used, +- // cpu.rt_runtime_us and cpu.rt_period_us might still needed to be set manually ++ __isula_auto_free char *converted_cgroup = common_convert_cgroup_path(cgroups_path); ++ if (converted_cgroup == NULL) { ++ ERROR("Failed to convert cgroup path"); ++ return -1; ++ } ++ + __isula_auto_free char *init_cgroup = common_get_init_cgroup_path("cpu"); + if (init_cgroup == NULL) { + ERROR("Failed to get init cgroup"); +@@ -451,7 +452,7 @@ static int cpurt_controller_init(const char *id, const host_config *host_spec) + ERROR("Failed to get own cgroup"); + return -1; + } +- char *new_cgroups_path = util_path_join(init_cgroup, cgroups_path); ++ char *new_cgroups_path = util_path_join(init_cgroup, converted_cgroup); + if (new_cgroups_path == NULL) { + ERROR("Failed to join path"); + return -1; +-- +2.34.1 + diff --git a/0042-fix-message-queue-concurrent-bug.patch b/0042-fix-message-queue-concurrent-bug.patch new file mode 100644 index 0000000..063742a --- /dev/null +++ b/0042-fix-message-queue-concurrent-bug.patch @@ -0,0 +1,41 @@ +From f90a145d9d29682295aebf2bcd30865ee5f6491f Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Tue, 2 Apr 2024 07:53:54 +0000 +Subject: [PATCH 42/69] fix message queue concurrent bug + +Signed-off-by: jikai <jikai11@huawei.com> +--- + src/daemon/mailbox/message_queue.c | 7 ++++--- + 1 file changed, 4 insertions(+), 3 deletions(-) + +diff --git a/src/daemon/mailbox/message_queue.c b/src/daemon/mailbox/message_queue.c +index 7fe044f2..7e53301e 100644 +--- a/src/daemon/mailbox/message_queue.c ++++ b/src/daemon/mailbox/message_queue.c +@@ -106,11 +106,12 @@ message_queue *message_queue_create(void (*release)(void *)) + return NULL; + } + +- bq = blocking_queue_create(BLOCKING_QUEUE_NO_TIMEOUT, release); +- if (bq == NULL) { ++ mq->messages = blocking_queue_create(BLOCKING_QUEUE_NO_TIMEOUT, release); ++ if (mq->messages == NULL) { + ERROR("Failed to create events queue"); + return NULL; + } ++ bq = mq->messages; + + mq->subscribers = map_new(MAP_PTR_INT, MAP_DEFAULT_CMP_FUNC, message_queue_subscriber_free); + if (mq->subscribers == NULL) { +@@ -131,7 +132,7 @@ message_queue *message_queue_create(void (*release)(void *)) + return NULL; + } + +- mq->messages = isula_transfer_ptr(bq); ++ bq = NULL; + return isula_transfer_ptr(mq); + } + +-- +2.34.1 + diff --git a/0043-specify-runtime-as-runc-for-oom-test-CI.patch b/0043-specify-runtime-as-runc-for-oom-test-CI.patch new file mode 100644 index 0000000..3f807c8 --- /dev/null +++ b/0043-specify-runtime-as-runc-for-oom-test-CI.patch @@ -0,0 +1,26 @@ +From 7af700c4021ef9961aaac37ffa5767bd4f3dd184 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Tue, 2 Apr 2024 08:00:37 +0000 +Subject: [PATCH 43/69] specify runtime as runc for oom test CI + +Signed-off-by: jikai <jikai11@huawei.com> +--- + CI/test_cases/container_cases/inspect.sh | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/CI/test_cases/container_cases/inspect.sh b/CI/test_cases/container_cases/inspect.sh +index 86aed3d8..5d976281 100755 +--- a/CI/test_cases/container_cases/inspect.sh ++++ b/CI/test_cases/container_cases/inspect.sh +@@ -147,7 +147,7 @@ function test_inspect_spec() + isula rm -f $containername + + # use more than 10m memory limit, otherwise it might fail to run +- isula run -it -m 10m --name $containername $ubuntu_image perl -e 'for ($i = 0; $i < 100000000; $i++) { $a .= " " x 1024 }' ++ isula run -it -m 10m --runtime runc --name $containername $ubuntu_image perl -e 'for ($i = 0; $i < 100000000; $i++) { $a .= " " x 1024 }' + + isula inspect -f "{{json .State.OOMKilled}} {{.Name}}" $containername 2>&1 | sed -n '1p' | grep "true" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container with image: ${ubuntu_image}" && ((ret++)) +-- +2.34.1 + diff --git a/0044-set-oomkilled-in-cri.patch b/0044-set-oomkilled-in-cri.patch new file mode 100644 index 0000000..a9d21ea --- /dev/null +++ b/0044-set-oomkilled-in-cri.patch @@ -0,0 +1,27 @@ +From 5393ce7d02bb73ce4760edefa959dfb4846f1958 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Tue, 2 Apr 2024 11:19:06 +0000 +Subject: [PATCH 44/69] set oomkilled in cri + +Signed-off-by: jikai <jikai11@huawei.com> +--- + src/daemon/common/cri/v1/v1_cri_helpers.cc | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.cc b/src/daemon/common/cri/v1/v1_cri_helpers.cc +index a3488894..ea5c8bb5 100644 +--- a/src/daemon/common/cri/v1/v1_cri_helpers.cc ++++ b/src/daemon/common/cri/v1/v1_cri_helpers.cc +@@ -506,6 +506,9 @@ void UpdateBaseStatusFromInspect( + } else { // Case 3 + state = runtime::v1::CONTAINER_CREATED; + } ++ if (inspect->state->oom_killed == true) { ++ reason = "OOMKilled"; ++ } + if (inspect->state->error != nullptr) { + message = inspect->state->error; + } +-- +2.34.1 + diff --git a/0045-add-cri-1.29-update-design-doc.patch b/0045-add-cri-1.29-update-design-doc.patch new file mode 100644 index 0000000..7098ecf --- /dev/null +++ b/0045-add-cri-1.29-update-design-doc.patch @@ -0,0 +1,312 @@ +From a1f8641f7b440362a615e72246c92654008f61e9 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Thu, 22 Feb 2024 11:09:50 +0800 +Subject: [PATCH 45/69] add cri 1.29 update design doc + +Signed-off-by: jikai <jikai11@huawei.com> +--- + README.md | 2 +- + docs/design/README_zh.md | 3 +- + .../detailed/CRI/CRI_1.29_update_design.md | 237 ++++++++++++++++++ + .../cri_api_1.29_GetContainerEvents.svg | 8 + + docs/images/cri_api_1.29_overview.svg | 5 + + 5 files changed, 253 insertions(+), 2 deletions(-) + create mode 100644 docs/design/detailed/CRI/CRI_1.29_update_design.md + create mode 100644 docs/images/cri_api_1.29_GetContainerEvents.svg + create mode 100644 docs/images/cri_api_1.29_overview.svg + +diff --git a/README.md b/README.md +index 970b6e72..694ddbc2 100644 +--- a/README.md ++++ b/README.md +@@ -240,4 +240,4 @@ iSulad Version | Kubernetes Version | CRI Version + --- | --- | --- + v2.0.0+ | v1.13-v1.18 | v1alpha2 + v2.0.8+ | v1.19-v1.22 | v1alpha2 +-v2.1.4+ | v1.23-v1.26 | v1, v1alpha2 ++v2.1.4+ | v1.23-v1.29 | v1, v1alpha2 +diff --git a/docs/design/README_zh.md b/docs/design/README_zh.md +index b7ec3ddb..3382bfbe 100644 +--- a/docs/design/README_zh.md ++++ b/docs/design/README_zh.md +@@ -69,4 +69,5 @@ + + - 查看 Sandbox 模块的设计文档： [sandbox_design_zh](./detailed/Sandbox/sandbox_design_zh.md) 。 + - 查看 Controller 模块的设计文档： [controller_design_zh](./detailed/Sandbox/controller_design_zh.md) 。 +-- 查看 CRI V1 模块的设计文档： [podsandbox_cri_interface_design](./detailed/Sandbox/podsandbox_cri_interface_design.md) 。 +\ No newline at end of file ++- 查看 CRI V1 模块的设计文档： [podsandbox_cri_interface_design](./detailed/Sandbox/podsandbox_cri_interface_design.md) 。 ++- 查看 CRI 1.29 更新模块的设计文档： [cri_1.29_update_design](./detailed/CRI/CRI_1.29_update_design.md) 。 +diff --git a/docs/design/detailed/CRI/CRI_1.29_update_design.md b/docs/design/detailed/CRI/CRI_1.29_update_design.md +new file mode 100644 +index 00000000..0a0c860b +--- /dev/null ++++ b/docs/design/detailed/CRI/CRI_1.29_update_design.md +@@ -0,0 +1,237 @@ ++| Author | 吉凯 | ++| ------ | ----------------------| ++| Date | 2024-02-20 | ++| Email | jikai11@huawei.com | ++ ++### 方案目标 ++ ++当前iSulad CRI接口版本采用的1.25版本, 预期升级至1.29版本，iSulad CRI接口需要对升级后的新增CRI字段进行补充。 ++ ++该需求需要iSulad对K8S新版本CRI接口1.29提供支持，在K8S 1.25及之前，K8S存在CRI V1alpha2 和 CRI V1两种版本的CRI接口， ++但从1.26开始，K8S开始仅提供对于新版本CRI V1的支持，对1.25及之前的CRI接口，此次升级中，iSulad引用CRI V1， ++对于1.25及之前的CRI接口，V1alpha2和V1功能保持一致，1.26及之后新增的特性仅在CRI V1中提供支持。 ++ ++CRI V1使能： ++ ++1、编译iSulad时开启ENABLE_CRI_API_V1编译选项 ++ ++```bash ++cmake ../ -D ENABLE_CRI_API_V1=ON ++``` ++ ++2、daemon.json中enable-cri-v1设置为true ++ ++```json ++{ ++ "group": "isula", ++ "default-runtime": "runc", ++ "graph": "/var/lib/isulad", ++ "state": "/var/run/isulad", ++ "log-level": "ERROR", ++ "pidfile": "/var/run/isulad.pid", ++ ... ++ "enable-cri-v1": true, ++} ++``` ++ ++### 总体设计 ++ ++#### 变更文档 ++ ++<https://gitee.com/openeuler/iSulad/blob/master/docs/design/detailed/CRI/CRI_1.29_interface_change.md> ++ ++#### 总体设计图 ++ ++ ++ ++#### 此次更新包含以下变更 ++ ++##### [kubelet: get cgroup driver config from CRI](https://github.com/kubernetes/kubernetes/pull/118770) ++ ++`RuntimeConfig` 获取cgroup驱动配置 cgroupfs 或 systemd-cgroup ++ ++##### [Add Support for Evented PLEG](https://github.com/kubernetes/kubernetes/pull/111384) ++ ++`GetContainerEents`，提供对pod生命周期相关事件流 ++ ++`PodSandboxStatus`有相应调整 ++ ++##### [Add swap to stats to Summary API and Prometheus endpoints (/stats/summary and /metrics/resource)](https://github.com/kubernetes/kubernetes/pull/118865) ++ ++`ContainerStats`新增虚拟内存使用情况信息： `SwapUsage` ++ ++##### 变更排查 ++ ++1、[Propose comment to standardize the reason field](https://github.com/kubernetes/kubernetes/pull/112977) ++ ++Must be set to "OOMKilled" for containers terminated by cgroup-based Out-of-Memory killer. ++ ++container status中reason字段在cgroup out-of-memory时应该设置为OOMKilled ++ ++2、[Improve the API description of `PodSecurityContext.SupplementalGroups` to clarify its unfamiliar behavior](https://github.com/kubernetes/kubernetes/pull/113047) ++ ++描述修改，优化`PodSecurityContext.SupplementalGroups`的注释，明确容器镜像定义的主UID不在该列表下的行为 ++ ++- Clarified the expected behavior of `SupplementalGroups` field of `PodSecurityContext` ++ ++```txt ++ // List of groups applied to the first process run in the sandbox, in ++ // addition to the sandbox's primary GID, and group memberships defined ++ // in the container image for the sandbox's primary UID of the container process. ++ // If the list is empty, no additional groups are added to any container. ++ // Note that group memberships defined in the container image for the sandbox's primary UID ++ // of the container process are still effective, even if they are not included in this list. ++``` ++ ++3、[cri-api: document expectation of 16 MB limit](https://github.com/kubernetes/kubernetes/pull/110435) ++ ++```txt ++ // Captured command stdout output. ++ // The runtime should cap the output of this response to 16MB. ++ // If the stdout of the command produces more than 16MB, the remaining output ++ // should be discarded, and the command should proceed with no error. ++ // See CVE-2022-1708 and CVE-2022-31030 for more information. ++``` ++ ++##### 其它不涉及 ++ ++1、windows相关 ++ ++[CRI: Add Windows Podsandbox Stats](https://github.com/kubernetes/kubernetes/pull/110754) ++ ++[Windows hostnetwork alpha](https://github.com/kubernetes/kubernetes/pull/112961) ++ ++[Expose commit memory used in WindowsMemoryUsage struct](https://github.com/kubernetes/kubernetes/pull/119238) ++ ++2、iSulad 底层暂不支持 ++ ++[Add mappings for volumes](https://github.com/kubernetes/kubernetes/pull/116377) ++ ++[CRI: Add CDI device info for containers](https://github.com/kubernetes/kubernetes/pull/115891/) ++ ++3、K8S KEP 初期，其它容器引擎未支持，iSulad暂不支持 ++ ++[kubelet: add support for broadcasting metrics from CRI](https://github.com/kubernetes/kubernetes/pull/113609) ++ ++4、其它 ++ ++[Add user specified image to CRI ContainerConfig](https://github.com/kubernetes/kubernetes/pull/118652) ++ ++[Add runtime handler field to ImageSpec struct](https://github.com/kubernetes/kubernetes/pull/121121) ++ ++[cri-api: fix comment lines about PROPAGATION_PRIVATE](https://github.com/kubernetes/kubernetes/pull/115704) ++ ++[Kubelet disk api cri update](https://github.com/kubernetes/kubernetes/pull/120914) ++ ++### 详细设计 ++ ++#### Add swap stats to ContainerStats ++ ++`ContainerStats`新增虚拟内存使用情况信息： `SwapUsage` ++ ++```mermaid ++sequenceDiagram ++ participant kubelet ++ participant isulad ++ participant runc ++ participant cgroup ++ ++ kubelet->>isulad:CRI request(ContainerStats)isulad->>isulad:container stats request ++ isulad->>runc:container create request(w/wo --sytemd-cgroup) ++ runc->>cgroup:read cgroup file ++ runc->>isulad:container stats info ++ isulad ->> kubelet:response ++``` ++ ++#### define reason to ContainerStatus ++ ++container status中reason字段在cgroup out-of-memory时应该设置为OOMKilled ++ ++```mermaid ++sequenceDiagram ++ participant kubelet ++ participant isulad ++ participant runc ++ participant cgroup ++ ++ kubelet->>isulad:CRI request ++ alt CreateContainer&StartContainer ++ isulad->>runc:container create request(w memory limit set) ++ runc ->> cgroup:create cgroup path ++ isulad->>cgroup:monitor cgroup oom event ++ cgroup->>isulad:oom triggered ++ else ContainerStatus ++ isulad->>isulad:get container status ++ end ++ isulad ->> kubelet:response ++``` ++ ++#### kubelet: get cgroup driver config from CRI ++ ++KEP：<https://github.com/kubernetes/enhancements/issues/4033> ++ ++##### RuntimeConfig CRI API修改 ++ ++新增rpc，获取cgroup驱动配置: cgroupfs 或者 systemd-cgroup ++ ++##### iSulad RuntimeConfig 设计 ++ ++仅提供对runc的支持 ++ ++配置文件daemon.json: systemd-cgroup: true表示使用systemd-cgroup否则使用cgroupfs ++ ++在iSulad配置cgroup driver为systemd-cgroup时，底层runc采用对应的systemd-cgroup ++ ++```mermaid ++sequenceDiagram ++ participant kubelet ++ participant isulad ++ participant runc ++ participant cgroup ++ ++ kubelet->>isulad:CRI request ++ alt RuntimeConfig ++ isulad->>isulad:get cgroup driver from conf ++ else RunPodSandbox/CreateContainer ++ isulad->>isulad:get cgroup driver from conf ++ isulad->>runc:container create request(w/wo --sytemd-cgroup) ++ runc ->> cgroup:create cgroup path ++ end ++ isulad ->> kubelet:response ++``` ++ ++#### Add Support for Evented PLEG ++ ++##### GetContainerEvents CRI API修改 ++ ++`GetContainerEvents` rpc接口此前尚未支持，此次提供对该接口的支持 ++ ++##### iSulad GetContainerEvents设计 ++ ++CRIContainerEvents 模块 ++ ++增加接口mailbox_publish ++ ++事件包括沙箱和容器以下几种类型生命周期事件 ++ ++```proto ++enum ContainerEventType { ++ // Container created ++ CONTAINER_CREATED_EVENT = 0; ++ ++ // Container started ++ CONTAINER_STARTED_EVENT = 1; ++ ++ // Container stopped ++ CONTAINER_STOPPED_EVENT = 2; ++ ++ // Container deleted ++ CONTAINER_DELETED_EVENT = 3; ++} ++``` ++ ++ ++ ++### 使用限制 ++ ++以上特性仅保证容器运行时设置为runc时支持。 +diff --git a/docs/images/cri_api_1.29_GetContainerEvents.svg b/docs/images/cri_api_1.29_GetContainerEvents.svg +new file mode 100644 +index 00000000..be855b5b +--- /dev/null ++++ b/docs/images/cri_api_1.29_GetContainerEvents.svg +@@ -0,0 +1,8 @@ ++<svg version="1.1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 711.9182835520442 573.5964413809206" width="711.9182835520442" height="573.5964413809206" filter="invert(93%) hue-rotate(180deg)" class="excalidraw-svg"> ++ <!-- svg-source:excalidraw --> ++ ++ <defs> ++ <style class="style-fonts"> @font-face {font-display: swap;font-family: "Virgil";src: url("data:application/font-woff;charset=utf-8;base64,d09GMk9UVE8AAO9AAAkAAAABO1AAAO73AAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAADYTJHQZgAIluATYCJAOQWAQGBY9lByBbpzpxQ1QZOitEWvl1ExHg7NqqxsS6RlCwSkciyuhqMfv//89KOsZwUAeGmJXV32EeijNbRUcYuVUTmWNvG5FWRBfkAW4YWVueXU6tq9d0pZufMcACcUzc7AmXTyBwYmw0WojBScItqLq853lRPMu/i21F/1VrveHFeyj+LdHE3AfsFiWacmNotmSIhpJesVGSFIFWJDgsMxNmRIsQQzGF4rvOX33MPtY+LeOF490dgR0G/l0zmGbP8ku9beGBOf2+EcKFm8Md59ESM5GZKYtFok/F14jmMBPlbsKEWTmjIcDMVCgM0RQv1VD4IZddn+x6g2Ff91p6BriTOHYqRgBPP/Z8O3Pn/WXBXUTBKoZoSbZg6N3qwPP/937uvc+bD5huHSziTSgio5mQNpAsr235cGq9P8/r5vc1H/KatrSSHj8fGwzDhavuia24cMyBvutkjYQtKFtwTcS1wVWcODY7KMSFOM69OS/+4fm59f7vRf5FsY0aMXLQQgsGrRwggmTYYCJVEmVBm4V5Z0ThidFYPeyrtRKEoeme3dvtgT6iOeJnlC8MsAKKT0UoFD469oV+l7DD8pn6lvkVc++0kSEfkCM0Vauqh+RnHNszHtYIqREKELpb3S0eAhMvIXmB7+tkb9/wpNO+5fOjmT/aU+6YYwu6wVlib1xrbGxcwNgduqmiNNOESimjUWmogISEAOOOe086XjvNccGlpjQnb8gjOY3Gyr1sLlmavf8vKSIJwtUoBiEBfY1jpaqrq1ET/z2/7f+/4/hmD/yU4cZxDvyseBGJ7/W9rVwMSlAkJUsk6hzqQCsqJVbd9NbifX/4r6pJ4LDfsD5vl0a3Nl1TSqk09SUxpviFT9KCpj3jFmScMz4t9f8VWXkadd4bSR35QqxNH33Ai5BewBI5hQAbQsZFe2nQZ84A787uLKHXduwYMHUawjI9wuP+7R+jZdW/WpL3hdof92mkcoac8YBwj3gJB7CB3aayqzgrsSIikYrAZYK2u3l6AJfgdm+OybOvB8+JTFAi4hU026JOz7fHCxQujUCxVVutY8Wgje+dK4CA+afpvl+9Nyecky85rLBnngvwxHEpBa9ATSI+MxnxjMcbadI6LaWIPXHlOeusTrrDBkAFCAzMc6UTOFEFKTY1Nm4GuAPasdAah4+DcVmNg/pfU5Nao+e7QykouG0KYR4ewkKQVmtP1n91ike6Kvt60/WySWeBpeOKr7QCg0hoKAwgPPOvqrl+UPI78CqVirROp/RhKm2YMy0C6CJQdAHdBNBOBOmKyGuC5AZe6X1LKQUflBN+2M7DB2VHoBs/KOcRvCZQfn6i0lsdM5Ypw5otoyCn0UlftixbMiaTk2XKy5Q1Y17G2zJOB7nKUm0/PkgM7J3K/X5azIxlS8eShkmeIYrIIuvN47j+8Xk5xmbzq/fd0AYlB/aifvZu/3or31eH2eCosZNM23gKUvMyo25K7fx8x2ZRqLM4/xxLbOST4x/7aOuym35y8olHctP4b8V/TX/YCP2p/1mult99+Gj0u488j59Q7/DTaWN4EkPx7itzRBhgghZ7XPAiQJAIcTLMtFCJKku1WqHHWlvsst9xLnKDYR4xwicQZ7tG+knSE5NnLF69LLqrR0X89vtHVudoGtiw59gDkletTr9hov04IcNsWAhCFGnUMYNrWMA6cqigBQoCbHTgQRhxDOMAjuIkzuIyzuAybuEuHmMEbxC773VEWIigqlrWqT4NbHCjmtiM5nd+y1vXlq5ob9d1c/f3eC93qA/7pk6NEihFn0Rmmnk61thin7O8wilOc4krXGeGedbYJEaSKhN6GOUQD/IKz/ACr/IW7/ERn/Il3/ITf/B1//9y0DjoHH5x+M2hIRCCxhEpTKlFmUZcYipGDGMmFmBNbIBtYYfYXewxq9h9hvn6/vr3thQeZ+KUArHScmqorznNakXbKqkpSqIM+SLlVlB9GtJ+HdUZXdZZXdGQ7uuZ3uizfnlBkASHEFk57pwrbnvKU17wunOuGjNrxbYj2+1zj/u92yM+4fO+5gu+7jt+6Od+40+uMZH/QTH0f6P/F2WjGegMdD6aieagxWg5WoU2oG0oH5WhWgfrEB2GI+UoOOqOGcc5+Rat6/9k/8+mUCCeJiIC5sZapHx9QEX2IMLTPuIbQjZoyoY9XgDCc1CSUcEfzRVFJcPJ4YfarTTXnEEp9Fll4bmCyF6huXD0c2NkyBrUclqFDS0fCcldmnBxq5ZzUdjQ0hBhra3noZkdrY2QIXdtwDXbzuJpombNK3hcpdld/aaHaGW4yOQ0Uq2KuSpqqHgkIlXMSxG5hZWvipLO5Yja+dGe2Ec0Xl7hJVeLriry+vKRHMnapHSSWh3Na/r0xosnKF8vBiOnHHm02DmrmKh7JKa/6Kx4A1EGB5L5gPZja3EC36Px8gVmpEMfpVo7eKal3dCDQk3OQWrtEfywxsP7kXKdc+xllnS10CSorh6UNHRFJNThKpI3Uj12NefH5EBdrYWSRruZkfa+i7mL+aR7b+irdu+/Svdr602Xpz3AJtUl38juYpi/pk0HYWKZtRHy//WcNGHEROmgcepGC9vOyzGRY1+qfJxX8Ka4G2gxbNKvLqrqBjSlQRMYHTrDDGsYBjAs71XrIXW0LYetKxhHqGHoNTTSyi1NsvSW9pYPZAY4gkxWtrU0z9vbRMU08fOJQxc+Syy+gtk9G54gA+xj+zrZX5Y0rjZrdBmGy68X/sBnzCF5AdMtKp4mqPbyQez0lZGifaq47Tf7SIZExoiy9kkO2JjFGs8VnQMdK9nMR0/nKEUbGADCEcjocG6DUGjz+CoAhQFwBBIDEwuFxo4bN+++CUBhAAhHIDEwsVBobBxhCwAAwIcDAomBiYVCY+OISUAQBH0SBIEgCIIgCIKgcvQdCMIRSAy0MCC+M6Y5BcVI/gkYkMn8sRZzoj6jjymGesj9+I2a22/ocz8NNPA2C+fgZtOvbe7ZbLKKuW1NchzK6mJ1Pf/1IF9PGNzuWk+fWr5AJLsGbXx6WqXtz7Sw5sK/2TRigdQEnKWCSwHOJDBxwFWcy/MuPpBExwGTydS++B38ygHYhoswSGJKao0cwRU+cxa87AMKDql645JDKYO+5m4QVzcJ3y7stzuYTaZTHAbNHXHpPfKmuHugSHKiUk/ZPC/TXrC0xqt/dsGQ57sgXd3hs/6nDbLRY/DknG/yxVziZVz+ldwy0Iy6cNfv9o1UVPeO7SRI/Eej3M5tWU/0ydP7mv8W+vb7QPcr3ytNdXF9bro22Y7bj7Zf+/5e7C/73n67/xrB1Iy9URj6uDPOjzvj1fgeZsnDRCrKMR3LsR2VaAcbarhhDXcEIhLDcSROx6U4E5fjZgzHoxiJt3PJ9GZwpV151l4c+73B+fGXExniT6JM1AELXkAM6XgMP5YTZEIWvN+8x96yP86/HmkfeR30BDeCq6FruBPGwn1qRw26oBG6Ql8N9qqpBupSDaqaqtbOeqAPtE9H9a6ua4yag2bZtBiT0RqTSZtKYzeKbdmu2Fors26rtyFbtJX2mzudd3CO+7zBVdzBKraziKWsZzf7OcUPuIzfccG8beSYgWdMypiWsTBjeUZ2xraMHRl1GW0ZwgxVhjlDyrAzchn1jJlUWRX48WpiZWxuhR6qmw57kBDolGzsgvY89rhb7XvJzwQfY4+20SX4OtWreKpuSFaXlECpiOVamcv4JA4/x8nKhM2qKCxtMjn4oyulp5+2+DzPxYtYzPSoPf9Yvlvd4vcuYU3142xsRkmAfbwfK6ODt6039/4JpW8VEqlHIZIunBL+6JmzfHyE3o3mF33fTPtyd+kt8NrwT5yY3tXcECqX0RZRjsN416y3bBO8IjDJcPLMjeX3UFc4USotsEQLOQvWwDz+tuofrKKIxg39EBCg4K6oeW4pSCLqm7ih+kAy7/Fydux5Ldh5nZZU05IuqzLvripqXcA6pZ2RPKH/d7Vhy+50yl+yFh/UIyLAjn8fv/fx2Z7SNsWc2qzj/U7z+cFG30qoPh7Z5qC3Ze5YhCSBZYH04b4xhJmKKLbQSuzNojJnYgVKYcrTV4jLFCCbX7QrsD8wOFpEHcVC6m7/T4/rDbOCMYvv9JxgP8P+5bnnhxblt29k79MERhrZfpNruPWEn9zyaWm9D6Yhlysalgwi4BGLoGwXLltLytTuk/Fts6n1vh2+DbWgbnrIDLRawWQ8nikM/EuPyg9/viTjuTpg3B1Y+hDUoC9ieivGQlh0cicgzznLfcQVZ3dVKCuRSmjBAmXfyhNu9pTGGImlCL53o0K42CnEDXXqbCl182f/Bf89gv5vwu2e7vJ4iWYg3CGwkKTThtD0GoseIwAWO58TjMhWCDTwf4n/MWnupK2x/YCzIig48kILRHftWSKYgi+uRo3xYmrCurpHP+wg/v01/53uutoVJiUQVDulJgTOfnkrR0ZI1k1tENAMdR8iv1SLeO3Yf8I/i8/b9WdwvtwVMZlpeDIIzYdmpyAtzZMJSHtCcxcsWTFrteeNzqfiMmfScU8kIho1b78owsx+mq4uQutyR6w+9Yo3PmDBKfNIugWybM4TIVcd0CWURQ0Sc9kHMwiRyovyick6zb2ikU77TH+AKiu2Gf+Ahqq76ShpCk2KlUBjcPCK11f9dj+F7vSs8H0W2JSabHDxn24YNFev43yHqeq6XfE4R3rTOnZgFh8dEP9Q12qAHDk022XQ1pUSyjS/MSNipVuDYUYp0cM2GyEIDULG3HlLVE2Auyq87HOQvZquGjVgYq9KECHaIH0yk/2vr37SWfBV8B39wF8uFIxNfTsoDBvADTTNVVQ7OCrfY6L0G9yaU8aGZUKY7KcewrBtVffi6ub8GXxbdpI97b8hBoFQUgYJ00i0awM2XFgyVBGcFpXLAuR+2yS9JkI/gib26ZWf37eeNzxCOI7hrUwsKtpDVjRC3jQBB+N9CokJWXc0xw4tO43puOVMxB3S5XQnpNSwKH7otNIEjQiM4uWWjq3nF5hOi5K7xSdq3G9gYjUCzpPda9tRcOW32Y1gO7B4VKSrcYl2suSDeHI+STDgD5mdKorFl+SBLm5wuaWCyjmlZRMIsqBi5kErzkGOJ4nBBsasiHSBSPGEh2GWK1XRkbr0idAO/blDD3UvNXmqJJmxpciGbqkJFkRNIUlDf+tr0yuVBmbNePeTRpvLyLH+s3osJPvlgTLqh3bYTHbiV3M6AGRNwjDRIuSterpZLzTWgcuy62rHrJUFzGaxIq+ArjzfVthhOorC0AiCppSU1uIeWScjvuX2saqaGLe8Fh3SttzX2jZ4L/t29aumKWwl4SBuLsVe6wvvnzl7f2Fj60x2nQvYoDXgEEtrWEGMMhekBIg0R6qwt5tu7aJsxFY80wzcmiOQL9SbhS37pegG7yUM0NERxJnuG9BGqdPXDMtrch5+Nbp6qOhNVG7d8BXcqZcV5coyFjc48CSzVlnjgNh2I9tCF1i0cmNaz0tWcdOWmnZQokq6pKW7cjrIj6FvBr7l+gH1AYTeF4etWFaoxef12lP4cFlvw7p8amqeUgDZqPYsC1MlUuNK8Mrdazc29gBZsdmRjPBrIuC6r/YdOIIzHVFYFJYrS6tbwdJCTvZNziE5kR7IqdUadOyGS3ESUHqK7/9NM3cbbyjUVkSsAOtQcaUIuezfbaVYT2V/XL5KOV9TQ9fG3WezS8MLr2cJBkVOfn9tH754Wao4QR+eb7wYvd2+107VAeA12lEZIBOssvLIFFCpkDpjRTY7Ovv0+7XIpQne3FTbfpFkF4sx2A8e2b/y6YM3OW4SCexYtGl0iDncNe3uxpa942A1SXQP5Cf61mjfYINvow1RZVm/4BbL9Xyj+fTag1N1gDdXtRHSP7w5j1qrmZ5xB3ApbsCAIKQ4zRDLkDfbE3FVSPJxnsxGk+P9wytWacTijZsSdBKq2ytb/prJ+O2mCchsWw3g0mDn9m7b4DN2d5seOaA066UNdPbUzuH6nQfXAunVgqo2ekuBJiumUfT9cxcjX6hdfdEzvjRfml7aBmEgqFVDoUAjb8kGGy6WB4uKKOG8vup5eFWweTsT9ItV3gQCp1IF8dX6Ok5ox9gPGraY2+n1iq7ZkiVZ5Dm9YDs7FyRqz+xpVVUolMUgZ/h2DpOmplbsrFN3TFtxNEArmqxBPWcSCxlJGisugPhwyHi/GDsfHF06QQKjxAP8pKgompEleE9YZHz32WddH0hyGDqskcSu9dgEuaqpMCjrBRHiIZeQsMiKnimZr0syQ+0SvxE9AFhxhQTs/AE/sanpzViT/b1wMD7ZvX1rBxCO8WcNLUCqHPuaarv1pOIBIHGalH2hk4EPA4jG3RO5MbMxXOhX7kx99f2ds1Vs7czum7f7DmvB5YZGY71ArWtIxnOaBSRdxAprUkE9EkEqH4Y6kuSWwazEblObCGPKRPDEqNqV1ZqTdw0xlgornRgwgOezRgQpEMGwA4+rQo3NbYvbG1gBuLaPrA9GzVMv9HbRsnav4EhDT3mZOlKNK+/e+Fx1s1pz/M9+UXf4Ja1DUcdd6aYJ8L++VzqQssWypCkghOsGqraGxEuQ2mgxXrGzpnFHzPRhhCT4J3DIYv5KjPnM8J3hYgyJRUFRERCiihSmv6uN1bjpzcsbV/NL2xfqs00vXq6mMiCiD4gbnSqNoxUtBqPV1/g+LIxqtIXWiofbm1thsPl6Tpp+Np3aKwM27ZYdKGpYoyiRVXwlWj4WDsx7Sn4iIWHC0NlwyGTXXr6wn4uzjyhq6JJ29+I1ABL4ss3q49ZkuZsSPlHwQmoZbRecMh/egNGDZJQi3KCtimV30adH2/PUB1KNGxptAPUUU7ajIqzWOquk8wrlY3tfbHumeo8VEOyryY9PbkV+J1BKHY2KWtbXAMeel6CQTZtybf3V627dbiPd71q46Ddo1HO0RRCRHyo+zBQ6YTv23devSKVABnQx4MJ/5Tlb/kCNO2giJf1rDycrANLWTQHC9x/Z272Dh6XbUfmlokgarA96u5NbV5DMzZEIEqnHX51L1IoVK7ndbzlCfP0n1UX1AtgGWfbL81d6m/JpVqBUQQS48gdViQ3PSvZD+VA4KO8s3VKOzdv7/ejA1bbLR/zvRoPib+DjXfcMJQd2I+MnyDoe/DePZDoN1QCCWscaqmlTQKd1AtMAB7bbbXfVsTEmNmOQ5EIvYqiRay296XeXArEJZJYnNeiPqoaL4seQk549GuJ/GTLKLduXFx+/SCHfYgMfIXRFrLHzo25iixS8ZaBbAuc7Cett9j01p1gzBodStOCUNZCCwIubdyN/utMxWO14HRr2m54XGI7nOpR6FsVAzvk0gLiVrFY4QZKKQSnXCC1RAq4xJbwHvP8Nyy0kkV7GC9pOVkK5M6vCwqa0z74TqahlwTSNmZKWbEHNZ6K6hPrG5FaXGRswXApnlnYWSWn6diVOZ79nOvEtn+Ox0RQ6xrJqA1cxLbjQJHiEZXWqYnb54dX2zEoxAQcYyxtX5JSdZTobF3KdImDFfgmUvYKII2bqGld+gzJ1Sapw1KJKJmbE8qJDxZTF08osFb/EMhMWs+mPDlXMCwmssLjEqUIZM9Ut8a2YS5QME1arzFoJ5lck07VmWSrXpid4zLMi0ww0pN7udPnlN1/hvUYAl62L039i1+t4moKRhtpjPKdhhJMXOPL0MpLpz0T2r4vrH581ODlaJd/z5uxALVOmys4gc4g4tnkm0yt9Yk0AMSu0KvP62glaZpuc/2JzG9suZErr5XUgDRhB1PifHrq0pvMFWDgK+c4wj14B5Om3O0g/F3lco3RR/aQgWEeRUE/slB77cjCLjTzqvCmGSrPM1EwiJDTNyALmpLmIgGrQ2pXdTLbk7vLBQQvjmUcktrqu72KCXYVU0+t1lYKg/gArCHwpVrmaQL0yckjHi1APcQhyykKTXX9tS/qqYwCZr16RoNHrtq9ceWLl/S0rn7jmlkW/waImOdjpXOn4bKbQpUscuG9fGb+xzmZMxYR6klAHLRZ/qJA2mjDlf+3hX0OMsUSsPQVhOPHsYmjyqGpQD4YJCCSMPCQv2q5IGDmI31OLvrQ01dvWVIanBA+Njw15M/XUSGnW/6I+tzKLo9Fj1+xXGPPUprENKxNidYwuiSiZbFutTvZlMVs5l9sqMQaR8LGJZeJUV3OWPWBkUfftVXC87aB2G39YDFYflLVYi5pIQDDNFBxKBlE1ZOZrWzjmgpJnu0o0AaR6neXUqQkJ19loSiYKkfHMwYnKfhMEUXOx02VkOGu+xrm6x6OvNc+9DajfWqqbU5OzpSUNwPN1zKNjWoHzF6kuw+EcRTkak/k0VudILpcSJXaPdFg+zHgJuWznOoTERhyYB5y6SCPWjrpkd39NaLQajfWd/uHWHoASlSRGba2322QquVondGQV20ZoNwLfO7l751szicl4a26KyEdv7bSPATtgJwUb6GRJtSGKRl4JhcO8qCVFZmPBKUkck2FO76rN17PktdM99RuLxKbn5gs3J3FHSuk4aqAk1ZACcNFGkJFaYxvBhvVyQ90LPqguF7fHJZ1+zh76WuBJPmLUuJoaNjBbPhdW4Abx+f51GmgAZIOYVBBv8i7WgQIt4w0bU4yN8UMzmMxcimVAllVR70rk+mJb1x1wRcsWr6SuCegtXmcFbZXI4pazvJwVljqqZGiqagmkGOQkoc5QGbD5S1rE/m9oYZeVhKXYlyqQFwoul/d5Nrp6SiKVmlThv1KiIog52ffqXvnT3xxQlwGT9OGGu9bWtn0+Dyqyyk9G51mb7sw/akWV73//dJNdcvmSQDNd/m6qUZ9XHRhWK0kRVetydUlKiYgFYr/8kEnFR2xXNiI/QK9CACmZ/+YXnf3LiOjMsS8ZnEgG63G1TZsNvzeGc4oeqmGeBLxogCYm2XWeAfNvJLvNO5S/onTtM7bnt9NTY6Rds26C4bbw/urcKix4S9t5hWLznuKpgW+FhAY3TMCEqVDWsJqeaX7XIyBw19HqWn68/u5qx/PVytZMTO8CGZaKfXMnuNX8WinQ8dW260lAKBdV2DWmsePLDauxHGxbE4d0lFBYwuzTG/gjJBPBQ0io/Z12b2eodtoTARgDpdIxSo5+/4l7YUDGSCgX6ha4JVZg4upSBZBjXQOimGEU1DcuUqbPWXsd/MczAA48M4B+0gpp9TEqiOV2NFhVIct11HPA3pfGoCQu+SIU8A0mREVOuSKXZNJys9d6O5kBWbq07svP1xb5aq4qn/nEGlSDh+O2g28bS6CG2abuNYKw7PjKDbQPhnlDt3AYZMLk87dDwAMyto8/2B+x8bvNlNB3R4DT6MZHUtVtlAr3xh0mpQ7aKnJAJSshCpAgdz2uiiAG/rD99o/XUODaTWr5BKoOdiMZ1Y0vug2o1ShyFVF2MZQNMMf84E66z0huVdwVpZcJeHZLG3DV0RwtJ1H6bzdWnZ87BkUEktYTxVcdByuZp1u352nwaA64cpHGX1aA2Pm3jjxjy9gGCbqilSAW9cM8wslgTjApO7u5Dfenl6eYOWrGoMI2wYTCoLyj7iub/VcZdDk6C4sywsci7sbhk8hs18B+32UvzeL7EfPPThkDuoYxqaeNzWhT9pdGnXHObIsHEEhWEMWpZYT7RoYhiGVN4sKs3B466Q88G8K/PDkP9IkN7Iz1rzCAmf77F10WCEy+W1mHfweg8htltqr2EqKSSNuco1kxc6ksA19TpZGFKQrTCvpJwff+YqgZbvttM8tTT5c0rLFVibKgWXl9dQLDI5NSoWLG85xuYziabFhAzljEhIlcrewjQnPG2JIPdCPjGA09JQRnNgLtVExzqGGWC3q32rimghfAFmMmuxwCFtd6EhrP7n49Pikn+aZp86ouacvA1mn3IePGfB5gDn3vkOwjCdNwINloGRy2xsRY2mQHAoyRCV2DVxhW5eUmnJwyej6yx6xUC4AFeVUyJfzK1gSvclTCoA8QxHkuhm18w3WELQ26hn0kcLfuCMexDQkAqbLKQ0aa4sHhljGG4RerELEjFJ+H9wGSr0eBevkxnn9Yj+AAEBlFqaAH6TGe5BUJUrcxwkWEAT4uJRw6HLZjXYjFmShkWEliKEhru1wMFQCNOrbRFVcPSBt1Aqe/GuP0BfzzGz9cW3I+s3JH6d7zPNHtRqA9bJS4IkTpohqK7cI6qzTpkji3uiYyGhZTZXFVE3ukSre9cPlO7S0ip6Bh7JQaNYUOwXblfCEELP1CJYAPUlOVjV22Q3ttPSpzFb4McMBEtXLzVyjNsgKRczcssaTDEjBDutnTgaqiTweDrT8SZi+58DlOCJA4XNTg6pG3U/rroamj7WTT52noPBolRUWR0KhujBe20ZBi/F4jVCOhWJwwKeXeqiF7lu/lHJE1eS3Fgno3UW1k+1QXR85btD3d3fid4lJjHUo3/rYpFQj5gWawRvTge6osbDe2V7rAwld3NSiXRX4eSTWdOV7Ob2nz9QKO9coZJknLzPQ6e/Jt+VZ4Mt1paA0aQ+thM1FdlCP1gXeqa7C620j7KLPb3JG2iqcmK8GUhSfWUbuXEtFXMZXtSe1Z4xlrg/nFGcjYOAWV63RehdtXnTvwHDIGuUElACLra3wCP9kk8z6Kp+485PnsZb+/93UWFVKh32aqSlfTEFNvkIBF4hRgVPUdi4tcKSxQDROKmCafwvL+3tp9tB7gt9AuTstW7Sf33LtyyzO4hcy2GMQ4HlBH3PWx/GOYBBg6pAoCzzBh+wGv0xsrLcud0OmyRYgILA719aT4ChbQnzhreVORUK1hc2QgKVp5pL8dMgHUrY2l5sZyFTOYl0Ie3AKZEO/arV6ApISCJVZEp2ShrijMQSgQTlGz9Ugrm2pkBKqe0ibpmTOQh/rAHv43T2BXA3BtpQT20woXrkiKQLjapGZ7h4vfq1XxZVLDHWtO2+bnFdWc4i8PrmBwRK1beJqSkhdouYDTZVGSrLkdCcwjdqTN9Ne9LNbR39Pzd4rTalZ93HbVhLdXXCCRZVqCgaAr8Gj87tPL7j3n7G9njhWADt4REmjnqRoioDdkDTLdubxDL66vjczXBWQu5CDUuenOC8ThncfkIbtmnAze4Cknur452dvk7onBc28W1MBtfGGvALu64uQCktJsnRNQHztGC4oC8VS1egptGIEFYKWtUI0AhPxo1jBcxcFaEp+t5ZUtEbD9N1IWtncO9q4hTrnohxYZcT1vNB3L5gxTN9tEM7meAeUFsOfSEBcB1tYFzCW0oOLCSvJiuKrd7NS6xIm6O/wtfvYGmRkRmJ6hTbcPrV96dZKYc0uxRmqTod/YN1QnUYBQWZV30MiB71ndge0w0Xx02zV7k1tlVu8PnhVE2BQEaXppK1/ZeMwaq2H9yW2jO5kVAjeIOQT+/dMN+LKlBy5ITd6ND6GVkyUH1cn9l0p1t/C0VvcJqUhZXU5qo/0JIKKDVEucSEAUUmwEouWEXvoJNWkLHblbNbhmvkXKnKyVmaxl+irxlHkwL/5Ra5I9SgxJVTBQwMcrFwu1tfhmdkFyAQSgwNckiY0Rl64mycf2XBdaxDnwBMstw5oUfzL21uQzxDz3RzFe8C1Mzaaqzi4Zh7UjbSzv1cbSQZQhONLcDC/ev/rS5iHAw5nPly+VltPTfd2eU7+qmzN5wlkJo6boBQ0yCh8oXmKKtEACgJtluJp6ALTs2NRoia7QexrLVMRfjfgkEFUqJmHYye/+FBiF7+HvZiIxE+IYVmdEJUI2HdQDDFyz1g8Z3yjfb5NpU4wr89e8J0aYqZCzVD1CY204dedr+ZHwA3nr3NjNrgs3e42TyL+kKQnl7yaf1wD+jxs3TCvVV1rHrbdjPpCNr9wSoULmayuo1SpVxEr1WkYulKtpEAt0bw7Hu4mSZkGtVhIqiLGUikkCcBAAcijrypeONwGX9CUfWlmqmUhN1zXFvviwT8YbCvtLxD3UHczdsNP5nHTQVGI/JR2+t0U7JK0P5LTmTAONui5V9bSYqPqsW6fzUlGcxfUq4IQyFqaxDSNnZi9TJQNrFMQgLI3JJoR78LIpAFfQe8H8NgFSaHZPhjiWRkW09lBpfpXIt4Gx/BH8ouFFgpmqEdCqrzRly+h5gK5+fVQAX6JqVtWopemdzmFeg0jjeBvZice2UQGpMHlDy2mBM91p0soCroBx1/GyD4tOEsKoFHT7ndb0n3iHBclEgKorBFnGKUFWMEzGd9xpIFOV4sTu+q6vH108hVZa1xwrwxOfZBVqPOVBm28UF7/OGhEQq7OKAZhmjNNcE+hyQTGEZTdRdZT1hul0ZzWEX3bNlsk7GdfaTeiWevtnKtqfMOVk1tegShNkeVSCRADUxkJMZlILhLjULCWm7Xl5ycQdqVJylBWKGpQRQvWGHBQH0bA2BLh3lzx1iAGLGksqHCV0bDyP0tzXVOUs3izGtqJIkq1NU70al0hAq+uiATV/cRFplJfYqQkspiM7UPViNi3jWxLH3z/JBY9ZuzfnIJFlu4uzTHYuY9ulEtHp5wS9QlvNZykyhU3FUrzVSjCNuRKm4rDMre2zU93UGNwL6RmwiyHvdrRl+bFONeLAdUsNvRUnTtXp/6mdEZrhQk8ypNZtbJYfS+mJ7TmOLQTYE43poPYIKwi//Qino7vU69FW3xQKX6mieISp2kuwQ8hwKOuRoRcfYcbSFzpdnVdPiNe4JHOgdMFTsco7JUmeJr9GX4pwBzsT+VMH3cF7JZoApHN8GUTX8mlDdsXuUj/tTm9ctLy9XEoYrUnanfZ1OvSVh2GrhngG7h2b22A8MMjgOB7bvhyRiFOuALGSfH9Fd1+DySdEwIph4gg9j4yBh/gIzpdNp8S/QXy07fKAqQpu/dOuGm1hZdocSc5cThCbiapoAv6jjzFDot/8ocOukugMCyVJPDrJtAopaTE4eaC+igv68rTrlyDpI/UFnyK0oBbUpttnn9b/mMxvM5kUNSWAmYENQVUB0A5kArSLFgF+v1Iq0AuUARuBOiAbdB1YAxQDfUAnsBrYACwGHQCzgAqgA2gE5gHdwCpgAFgIbAU2BWwHtgDbgG+CHWCQ2XlgP7PnwEpwABxiDgYcZv4D1oOl4ARzNHASdDHtwHnmjOAiOAcuME3AJXCNuRy4FX49sA4YAsPMbcFd5g6gB7SCR8z9wGPQr/0E6Mbv1Ay6+wnqFd4CevABGsLbQG+/gCPm7wEd/gCngH71h9OAp0D/3n8oAqoVXhJQDlSZXAJUgnerTQdqAF+ABMxVeBZQCBSAOcw7wFv35gfkBN4ErADfPLAWfOcbRAcgUAhoMFjobQdeXB7mKJfzlL+39BHAl+uXXvl/so5N/2RJKH9Sj7U5/fmhxseQzubqP3GyV3+yNnYcHb818XdREE6URRU0WPiFd/AeFsudUi8d0iU9clrOeDmvwh/o5/j3I+uDlACCuSAn/MtJnGbQK5KQllx0mWyGALVe/0NHdV5P6S3mr2ab+WTKzRv7k31i7dbLI3zgMT0qi2XGJmM/4qcTH02OJJk8mxxv7pOxZi5nVO+uXE++5Fv+Yx5b48U9xfcOczlXcPvO4MKlT5b+tQ+tuuv31P/sfKK5o+ltzbT/2rZ2/tWxdnO9W72S/o/9N5O/Tm6fLE0+TSV+8sNbXT16oe9i+pq/1lSu2V0rvLzm8quXT1xJy36Q8+DUZC7kKtf+3wTt3+9u8Jn+3Y2/nf3vQ3mz6dxffR1zIOSnYSML3sXCxcriT27//tLS8v/Ie5LPW21f/bWCduFHC+8WLhedXv89qlOx70Z3YzzaWNKxZPPmNyX/714rdS/tVPpqx8mMNptWbsr+FKedkpVby/m5k3l2ZbvKPpXh+ZdVn6iaVRAKSSFYeJ1wi/OFmlY1FyqFlX+te6L6/1M718+snaSNq7fqUuOPmz7YdKn1VvOXrFNtmL0QE7C72H5sPKdJ6yLiibarbQvtR9v7tP9DZkmW9Mkf8ydSN6ljtID+GS7VVeYxeC15vXhDuC2O5Eyuw33H/wX/C4KHBFHwBYfwSsgV6/t90ZD8jPxnknsURvk36eelRWpDlbVPyWq0P5V/Ux7X/2ZRT0WvsW7cNd5R/F8JytnK5UqbmZhr1suWaF232tahdU9VUq2orhW9J//ZTd2+2cZKRJ1qsiurIt89p76Ke843onpSvvxW/hFv4V/zNaJGxERCpIkMkSV5sn04J5nLOXN/cN58M77FhFwvs/VRpThMfU28hHnkBZWKr54RT6h3tX8vLZNeN/8snzE+biwZP2h+2XzTes76Bf1LesLJGDONa+af3H+2Br2u98D+tX8nmA+3w990RuJ/m5qXWpDKia/Hf5w+mv6lx8n9X1ea6unfujvZd2XLLaXdbnfb/fawPW5POjc6XEfoSJ2d7i91Z3vfVXq6PFeBDctGh6O36k+vbU6+0Jzf3LP+S+uP1rsba6eg81dTxVsT7i+5X7j/5+nxaDxaD7WzsPslb937ZW/K96Ozl2aUfmHgSOB/ULBUH7pywo/8Yu2X80+iumggmr9Tv2uIfyauENNH7xEqkRJbxJ+RWEG+YFCwST5PJol/UVgvfEg8SFSTv02+k/rctavUakpIM9fv03/Oo6irWTH3H4s+5HZz+7n7+W/dXimyd36VXOX5yn+8/302NfeT6t99+Hn+9/N7Cx97Ip8WycyaeM32n5fm6OW6L8pJ77/t1m/ojXI4q7U/EMXqXibzWQOct33KO7p78qQWcK2sTGv+kkDXjNWp2M49I1Q6L/q2WHP2BWDFnVwzBqFVqTVW48phjWgW1QgAq2uyBhveX4cDCDiwzY8u4tMErObflbyTvAPyR09/8Vrtj228o58s7AB/nZZNdnyWpFfGOpHuAfL1l505dxgxrAWyDzLMOTeGYxpvdRXF6aVDVqrJGoBWyAWyvl7CNcAdbjReNhsNNjaI79ewimqdXvT1XIQQYG6A9LGwTo9MAmeLkxvgZkmGleW0z8L4QB4biCACM6Q6Nwwi7OqW2+6F7+lOABJYISyTmeKVOsaMDKH829PqIqiLLw7X2FLIcZsqmBjpU8aYsnaVU9rmU2b++BFIPTHEGGm6HegMNZ4tcybgY4uhCgmKaDXkzOC76dpWsK5sS4DmV3nIm/OcndbuBmHXfJJvsTD+sdRuKFZ2T222Pgk0olNlHwdc6GaACrecfCB0VwvmHD5TsaSkXlNAr3mngRIfgBxbDY9VVFEFpwHK7VYqVwELH0G33Wss5cPV1bCslXILKfg2kHP+5RrUnzUSh50HK+9mhw+RztN6hTnR/xx32e/84W71GRAWuNBih/zhvlsGCENv0CWHUFbKHqtMJRP0DHYmvU23MNgfPIPzSIB66UhwB4TY6wYiwMKof9r1UcqjlqO+UvNJebyXL9okq0QcOxkmOrHV0NzS7zgb3ckuv976+mx7Y3LTnPnCfrNBJCWLbgHtXWurUmpJdvGoMwoqk+sm+BEgOUqR8UW62ZzHXHreFSGOx14FDUX7mkCJV7Nn0dVPNN/mu3d7xD7XEzdse8+Wzfjc37VU5IZ4LMe7kiuCMRhfJ17Q5pJA4mriTy9/nV5OQLhf20HsYMeyAzf19y6ZmcJig9+S6m8UNlRAndgQN+DZ05p5gshG6UbPSKFmKoKGvl2rA/fIHzQtcaKNkkSgCpWwBEaRe4I9m0dz/naSkFW/pwPUTGHwvETmjRclZYYI5U3VYsUcT8Q9XNxziuUgIGikhWj8/AIafZRmBY0jM7GUnbmFLFBkP5r6QLWpAdb272zUNGoBTDNqR3eN4RaowxPBagWjUW1Rcch/v7zyUuNtMhs23/gIkMQfsdI4V+a80am/guRmhiA/+R1MghTO3rpSA4EMWng7xLo+Y0MGC68IMN/7WpLVtlIHgT7t98OjKDb2GhTI4MyGe0s91U8bi+EWAX2ShzuIgKc4roCd7Mz33B7K+3X3PWnvXZu1EDjwb7N8QIvLJtjNUItzFCpqGs5IBcCC9itFn9xmXf4/dIhriZBnqyUZ1pFzGkHh1yudZljbjoBWtUMkXOaEX7xdbErxhW9KFBMPjPEyJ7HziGbku9OpneY97bc2JDJDIapthguGa/1MpaP+4q+szFdH5Op3URYsmnPCFAns1NnkU3GkZ09orWn62jd3Lu72nlLxexunYJG1Ya8AuDOBmsAZlKb4/XkH5Xpla0DjDlGIHJYzgkgUrKTZ74x0Kd61xe7T0lZrdvkrD3U59jYXwxBxrFCQIKJg9OMIyYq8+ilkjKGQk6lgVrxzpGDX1Sha1RJdSeo6jiKtrTbBO9oz/FZp3VAju6YpQBpSqYoSuhJgX+8vamqDCxuq3Aa8TChmHDlW/Youh4ZKXDf1lZhMVid1UuHblWYO0EHxUEIfIzZGHMoGdK9ay//aOOlXV4sikKS6wvapZIvQAU9hCQ5I5NyQkSWnpIo0tlaMxK5ZFVdtk3bYIoC4fJFzmvdNv9gpmmwt+dG/EdyAIpAFWqNSks3wOXndgtrphouiZyv3FAuQx31ihYZwBAvZMDSRVFm17TQTH/P/WHSwKtenDHx1uxjtpw6rVmEZ6ZUpPz9TRfahB/1nKNgXb+5ijOxxmdQGTwFtqiLtw7QoiRoC9fzx5xkXvEwt3EY6bKk0J20cyD6NVWjQlzEbBqtIfp3gmBgtwwvvaHv2qzDjxPTH3ITZWDUBpKukCof8jlRCyaAuyiJx+Lys5+vxdq0EXOSEWfW68pSK9UefUzldBRweqbbr8wMxwoBsnE733ZEKZkBOWKuktOnFjh+pUS1Y9g2jotCE08QyTwDdhrM21PkrQwq6GzGrk5RQkMRXUxGAwkogvxpfw86oxs00akFFXmWXXMUfqHhMRO15Rr/TPzXHsDfA6bN+XMFHmvXa683j6P1xLxiLyb4os9qZ01fSVCpjTriaUQj1chJ8+gmzMxT8Dake7+S+C9EbQjILkBE/bDlakqLfbu2v7gABfwBPTy2Zaf83aFabWmtZmf4ddn6M79SPrpbdU1GqSYnovxh+/gRbAib0/300uO+LdzdhW6vvXEZWi6qiGYRRVhoB94lK9b90rkT99iZQHjZ2K3PW86SXaE7bG8cuMMdG+gAu7135ygfo+hZxZDprXeEzeeufat9iVH+48h2ZNdkXj9K5Ao/W7n+jnXz9f7CmxDivEBeENMmyWUkXMBvNOJILtTShNjKyTlsKARilp8Z7FwgJBwesZEEOMYW6TRjUoYW8KWFWoGs4YanKfPYmWrdl9UkKB5TYXunDiVO+ZSKIKaoMMbvKo6bxjAuoI37cXIKhmt5P6Q9qMWL6iyOoAawnxEN6nV06fP8mQKfuCwOYKGCSopywHvvJcBJSb+OxRA3427Z/+Lzp2kXNNRO7thsKgX707IBog2dnyO6M/8PfLged30mDZafp5iYwA1eHTejpTeIraIVX87JfEWS3DYKgSm5K3d2OomhGDMgL9mGtQazxO+u810A4e1eqzq9+qe4OXz8QevXVkSPmcoXE62UxX90GFjFqfrg0li9ve2NoZ2XBQn58TAj1+BARufLyUqCKgEBIBLFTjx17Sm+a+CP8wcC1mnI6IaaVWHd9cHwsvk7PPx+qHeZ3Xtrczo+pQLABWjPv2OK3TVk5kaP/sX576na9XV0v1SOVU6vWqmDskY6cGpmhU4fmAXQKihyg1qms+B4ChJwrDZrLX/M1maLaLYacT+p7w8rveoc1rzNPOYmHpdMLRbIqMxogW16Ka7B+5/71L6L92VKnqDGlPi/jwHUvh3NaW7PN/APqrOrYEmAZcGj5ch5qgD6eiyTW/fHLiiY1JQ2DibypqaxqOqnD0FOloD8BDbXjjFhJ7lGNgtTgYYyoyuqaEb/ypl5pUpLECwWPdLPrBM+p7Fwq8FMvfcG3spNUq5IC2B7egfTBC0kTdnX3hxcQaSzbjjsXf/+Gloz27t4e7eoxF67aDBFqyvKVKxN9bdUtK99p8tUR1xIey7oAoEJH15ne3/y4BvnZoTNW6finnjU7vVvRo0zoVv9ucpd8dSAK3pr03AdPdt1KK6X/RMa1PVu2sE5TfNPo+68rGiAr/sCE/7LYQby4OOHH0AVfjzOwLg8YaVxtzPlyLGBaEEoTTPe5tA7r64a7zyMNze6ZMIRJgb8MBPZ8AZLDFVUSl6fixvWZhMF0RwdcMZZimKmqaoruhl/Hv5v2K3Z1eOUw713+Sr3UpEvn0ltUS89vAb0HP7wKu1X1+X1O9drnwsBSugt3LDV+GBXImeTD0bJeQTm1oOB1UhbFWFmH3MWPELhx/wuz+btnP6uoW5fZhY3o3uzeY6ulYeA2PX2rAIWi2F9GrVLFbGdv/Fxxpr+7e2/Nf/vW5nBPPgpBPk8WEsSXoH1aIKiWJXzqcoUhuDMr8zIbISoDPrmA6+MZeRQNuuvakN8qPLaf/HuDq/K0RAX0lx+8uogs1MJJSgHRQu99uq3NOUD0ItbUPTcS1uX01BdzigMHVitX9XNYx41CgIXKutwetHzSEVtAwhs7DPterpkpywo9NZNN48AK1N0wMsKExpVg3gVyeK6LwSioPpcnMwlR6zekdvXki1JDBDyvKDw6Ro7xrKTI3BMyrBn2cHgo3tT5+3SoAjITSsnmPaPcNQ0SgqcIOdB+cM6vsXFPNc1zVfX0QJwURMdUo/wT+vdb+dZTnyOeelHft+pTaAZKQNiwdvXrMWILoRLAxdNuaxe1xnbXtGNwfVJqXo/EjRUvtjHNvP7Fbu5gR4bXwKlmWqEhBiTKOybd5Jal7FxeyRSK/KcX/oYBB0ZzMd4NraWkZ4WDsT8IHlUj+ps0AnlJUhEcKxK4xAz7fA9Mqy/323BF+srclLgurTazBKF1iy69k9ktv6qPxFvzlrv6xa2NdAVY6snzvaGbr+iIO5McTvoAJLEKU2q3NMcNza5KRBKUSq59Bau5JQ0QeTXv6UEGWDw3AWXhAXvCnmcM1Cl1e7akO7pGvp4Gp9h3MWh2VAaptFa93NcY7rA1WO4ntOTF9Pm9S7VV6cvD16/b/Oy+fdHVoywQdFlT2JYnSo7lN+OzF3Imtbp92BRA4Oz8VJj2uZVMtHwwdBv777xsWcialW07ua7gzzOyoOuLhTB9cXnDLOwDgu5v5Zz5DF8CE4nFoGOsyZi2NGXxjj3ALeCwOdOAFCkJNTTg95sYB3bSf4hOveq2qdFSnBgklFaYMmKu3rNs3fMdxRcHfPu6DcaAR2bnHYlE6S7RFOXjejSTfh39E3i5pETJNRHqKWndh5Ztsx8haUZJL8F6kSqhkUvfIAksaq3w/R2psvFN2+nsulEwABYMPm+fKDnAC0hhadW2mAMNbTBm06h2oypKYjNAgli9s/XCSWsasY/88S5JrN3RQDty4D3ZGjeM6+Hdl7UIMKYtWdDpMWIVGXEp4VoLb8+dKFg721ycae/rnII38MQkaunht94ECG2qbqr1/SZeA3SBi7K12RVDyq0+NWP5v+bMb2/ONfcyKubxPeVB+24nxUeskxZPufbbbZKoFfSf+RLfwidfXWhkfCUFpvMjbgRlgLhvqq65bkZOmn3KaERyPUTG7ik3QS8QpKJ0/9IG2vMcpchSjPJSUYD8ijBy2G47AurTP4wprJKeMUFAORXuz2u/IZthDXyQb9EJmpAtyrisf85FRLpk+PBxkBDqMmZH9MWzB4CA8x0JPYDPecUq4MJ8DH8LDuF2uMW4Fs7B7Kc/XGdmwqo4dB0xci1No7t7mvZnr4k1p/Q7ifxMcrqRCgPRq4TnLD1kiOl/8WMIPfF8DQ0DEJS0KkrB9MOTaJ8GdPLx0WH4Wh4pua4p4hOGX0sO9AY2QooRGGEBe4Bwr121diEct8KBF7mRhE0Ibc1VVasmA1EDBmO5jHHywdz0EEj6H70vx+zZ6tbVLtgc/ZObl2H9Dj/8ElpayMitIIpFNA+st60L6B2GfqYi2SgZM0xR5pxnn3V8RRFEXnx2boLlRUmWn7lYbnVmwR+IDz043+zTIA67coeX1+Am3vtp0ejsLo6qNiBwMR9l0h8xHmL3GjctLz+ZebzYllD87A9xt2B9SrXH6HVH2a2pMi01LF9x7dEiBop/cUeEBxrc6PNoE3uRm8cLwRW//VobKtzhUCmbMY+FhOxbt+3xJjBBqzOpPXYu06YP3NxliCEI6ymusPZElicujdA2syBrOTSTwe/94PpSAf1g5GDbqIqrpHq9hFs+d8NKQNkJZQszmdGl8mK85b/Ve5jcHg2dra/IrqIgMnYbMDNVqQpDqr27GAk2VdFZ5vAUb9Q1zERi2Vh2QNuFVMAz88F+39rd6a8lXacZ9GsAqpKE0RFx0hDqJbqpEx1AVE6+PNj1CErlXtZk1zOdb6p9xVQt0Ve6O8ACCNtqs4MteqrFyPqTAMjh76caeccfpvfuX9o2CxnIpYw30PGMXUBQDgs4Z01AmsM89OKOUkWVvT361G6SGif39qXOsNR1fmfF+k7gZv1OTAu1XXjqCAeCFgUPAnYq0ljrVNGwEOEqzuHTJ6WOqUx1o3warTj3lKiSoNvVlIYUFJbMDcW78kM+Qm/pk+xhbIjhaONP/evFxgaxVVgcv4DFjCB03I0Tl6/EyU8+8+XAQrZFmSBWIoq2dOZpa/fGJSavrtCABcHQjUreTqTrWs9XJkoXKDByBL8FvcaxDtXYKQUIylakwrXC9xNxV2++C3V+GkvdeQ9IWH5Hh56+KJlDK+jywkwIWLaOa2i3NgYYvi5BtiD+ehK7gli5BgM1x4xwzNdshJCjFrbfogdqHVVWJYfLKJwpWSTUvYiPQCN5YZ2EGlMkBmzb6/U5dJCfLzWM5zz3cvcf7zFVbdAaFzMukJf40bt47edOoDqUIm7ZnrsYDgFMYQwnh7THjrNz3Pq+FlN2b+w3G4vjM2CZo9nzlmgAnqJUQqfHm2WMCOV4AbMiabvp8rq9tbOes/TA8KyYWSBYO6Nsylv74AFl6k7wLpOf8x/uv7UZjv2wgNJgujR5whx3NxJ3JG4V2lhCCZH8yooflYoTvHg5n6afuTTTKuan8xdn8YvNK4Pzh7NHK9/NN/bbfDlZXSFr49/1kaAwrlaaqE7uGvQFrZKj1YoldILCCnD1RRY73+fFbPyR1X9m1FIC71PVSPfMnBLA3dj2/JF8y7gz9K9DW9+uHkm/Gw8pP4oCnvSNws9A1SJ15rJyqnurNtKGhjEEZf3mkII2Hr9YTkn/RvjtD5Vv6K+IXXjiWKrttggIAEmURBH5A3FPuA76h7TEZE0xqFsYpC1x3ItdiiorOgI6X0dsQOUpJ5NkFBe3poSZgMTJBtvP4ZYMm8uldjUJ1Ax4wPFoKmI8H0tAEhc5LhCxGBtSQHK0deSKe1YTHGwdsYYAUvNPoyMQk7wXtqJqkjsLLqcOUXUFo2rtqMdBt15z2D60skWACbwX9TSR5ao1sfP8VvZevT4RsyN2O+309tod/3Z4p/SGGCidIk1fn3fKoWC5vgGDoSjKoh6c5vRGGhYuJPGXz+FMOUNi9PNKvTi788JMa+K+/bw+BdRgU27Cs2dMYwfV9dH6RAbtN1+Ey6fcdXxEQ5KYzgf/Ickt7xmtJA1KrX4TWLlQjaFb4cUYLUaOVwf5NB+t3Mx6ZkUsewupOVLamm+qqmyUtUIkKbRIRDwwo42AFlR7dScUlpeIPiUCNl7ulOHuhlfeeBa5ft0UK2tyPp14fJL8xDwmneLmu76Lapw1GtsCaA0ND5ok3KvWohN54VucbbF7xRofQACjsOdJ58foyJrkG1ybhpkRMw0M5T50y9hwkbdCVM66tbPViscnoP7k2sBiTNeB4/k7O3vUX9lw3KyvE1EyyoodrU5wu5Oi60uKxy/RBUEr0so3Qu1cQTJZWmDbYSMHWDIrydDXP9jPov3IyT8YRq2zNJQtlfAFx6y69QuSDUTu5SdYxAKisPcTI6m7dl6Tr10APJzqVPS4Dqkx6Ux4tTEfz42X0nn6QVbDirHOcI06oRv6LoD1ZYMGpkA0BhpTfO/m7b4XSef2b2UCmjneNU1MbQtj2637FbfUwQRe8aTVciiAkX675rSunp8fceYtU8ROc17lX/Ue+kzYH2lN+bKpcTtzhDeUoqkAF/otmfz5B9fzKkrVKtj7gZOGrr/3aHqpShWSuXolbZNzvuAuoIp6uuoAA88PLBiLhV4ZBYJjInMNa9k4yUbT14hN7dJp0ajijLgaOHw5mM0/3/uN1S3jTkuw0Vs/mXx0HFjsSb1uiB0lj8MzNXV13zXPktUNOKq/OJz2Jf984eQLJ1U6F6/SdXcDlrtICH6IffTDx1+6SNWNOqTOt6X+ir8LHFBS436tbWx/cdQmNanO1dWytEWINZCE5AXXjrVN77Bt73bSwsiL9XGmRXFPfcPQSqns0NRteq6ndpaStkZjX8kpsjKShv1m9KGRwbBE7LnIVGiZDoSIDNpGW1ww18wFbSNfSNerFb3dzAs40MerlmyBUsTfvhuPha64LTQJ4EZO4eQjkHDSUcon63FqderCH1YC3uFVJjBaAZYPwKPN5R296Q0yrF+0ZQKcyOq+AYVSXbRRPac107KHd4b/9eI2YFPwl59t0I18dwou3bOAWVNlFdqB1NUQ8MuaYANiLfhl0ahf6zypwIxlOI5djuSuMpGbVibLGK8PascMxUdFa4P2rUe9rd2jJtNeJw7dCWlX0nIKChCW+IYUA+FjFwxepmkvyyH6VWFc8ks20CPZvgy7O7z4KqqjN6sFVazVqv51TpEMR/O11GjtAflsosO88vuyXVYVr4Bj7We4y/yV7MjKhdxa2myVwyQvBspl7tsTLRIjs1XnBnAtr3HwgmTXCjS1qRG3D6QkE1t6Q+X35eZLk/ZJY4OaUF1qQ+wLRW6PWqQbLMvjvMAIAPXpqkikotSOfGAmedtBi08c3Dk+iUgaxcq3jpEj2uPFyIUthB5xDq7Zn2zLxXh2ynrNh7ydLGvY7rb0Vx1bVmji4dz0ym/9ZPN/E8On5zuoYf5LtN+0C6CCOfRKqEKGkQQeRckSFxeUT/ttIuyKxL/x9f0MlzpXbv04Ryr8sSNpRTd1WhZ66Ywv0N6qDtnyhg1I1QlH7N+tYAn6hLgsN2F0Sv3f9/Ce0nyCF8qdslgCTCKvqCxvsx12RK6bHHR48Y57FiKuJudd6eCZT7Eb/iBPx7NsrokQe75t9VwcSF87Htv7JiYkKGVAdQomKGGkYkhCmC5LYoqos/f9QdjKYuqinOy3hPRmC3cI+zlnLXkmjRsT4U+EttZSvOP1/eKad+9hg+o4J5+1XRDXrpCAVaOW2Rh2ymTT8ZRUS8qx4Cx3QvOn35q6+NRRhxhbGWjQ5w/kHBrtLq8PClK5XCBV0KxMr0ABxgxFicXOdZWvFe9pDk+V3BKNATHwkpny049pZnNn5tgfOtBlLF181v4axl9ImuWmCdWODqK9pjZ6efqRo7UD42ZtrT23HXL3dFdqPg5oDI8te7jaamnT1I5DvWvvLLb8IK8YmDOjjFAmn+q03xRTQIaIcTNzXCYm8bS21dZ7ipcSe3XsW0CKl3fKsLdXNTB0d+yw+0Q6gUt5VVbBmcTuDiMJB5LPspXKmyIWL+fXON7AgVQr4YrYsMvA3qGFMU2WDaUpmHOKQ+FaxNhqI0lo0/O6diuZCLeNTaCCJLosyGBIQdeB663BvWRxq69Yh7je28Y5/VX4oBgOQpJe9nUYJ4yn9Gi7Ssslr2kBK6P9XkgANE2VotqzrfiZBAZkTcbwLyCpFmjMyv5OkKpNy2lGKSaqphmbx5lTMFXDxHFu3zdHgWoU+B2kblXXulum7mpxHIqpSmISSRHvP17t5Lb7u8Ixd3xzX2/IJkPBCC7xhkw22TreDnYzf4vECuV2VuM0siTZOwqnFQ2TlAVRCO+Wu2oMiAgdkR8Tw0DqSawYuNcRsXqgD0oT/wUglYrBbDU6/4a47qpgqbjdUVrXPaCRgL4YBnizDcJJS5q6pn3htSqFPPR62MCOqFpyYEQlb+to83XvAp+GqVY4jI1AagF1Yr4tOECq+9zeBlybYxXn0FLRVuP6PM6k6ycCUEQfiC2YiakkRSuB443DJmiXXHgqNhSkHftxLpenRAQLomQg6BYUig5uu3b1JXMH33LCNUBAI3z4sUtzcY6ONQv7uPjzuAHEliSDB4YD2T7rDaB26Tgmapm1VtiOxE7QVUBq0FVDhNA8I1p715TXm7RRC+TNqwKH/VJNkcLaBRxc/pzff7YMOKqm1GBA9syyDYafHoBJzoLwWWPzgJHMSG/qKZ4R5HtG2L+rzuiKYVqW3mnQAc+TicdMIr8cJbioXv1v0dv/elwmJ4Wm6fCaLmrZdGTvwtCfq5Sqy8rp3l7PrlOaOZOPO2oYNiXXb+Bx2KN71BBIgfgAxNX5POQyV5fOnPw92vK8Y8sZt0JUqlSYY5upmRnWHv88y508uMB/3KN7KuDyjZIPKzo1KEqUa2m5c/Fo50i/h6llp1cbE1k0U0mWgWuSfkJImbVOOJTdD6goYbHsSu2KKdl531VV6Wwm1cMqwAr9ehPmRawm/tGosz05vH/DU1hoYXLe6ziMizY0ioRkRXXKV16w15kStlArbC24Xkm0aN42vmrYjMdWPslQyvmxxIFTUlhc0YArL/Zh+K7xnLU7bDbC56/HldpPTe7QU5TXY13Z8Q3OJ55sGI7VTpvDdmfQ60y2A1UxCJW3b6yXdzhQjbf32jJIG57ZZGC+6tdczlrAFg1oTk/opx9NSFpdw+pbt0q67fmyhymWhV8WF+bW2lUtU24IQGIUCbIcrqODh6Wj9mZg5KMEA4GpahykOVkQUZQqlJMy2APT4SEeqbGcGp640qzKLCXbiiH4VkXMeTZwhSCPOWuyoPmCWlc0VOOmeE2tY1hRDkv9gWmGoeAJjed93QLO5l87mIEtckVo2AmuWgpt9YbeIDzuf3fJn8ZhGKkdhWTiY8VQcD5v3ZC9FjomOk1HY8NKw3G4UJIsnk2F2Ny0XrV95IH9WuyEg/jdZwVhtRglP/n8zPL5miW5PboIPAbdc9hM/CAa1gARc+X/i3H+rFG9JoGRuI6w/m0gwUhwz4aX1WyPUS/+tfa+GFGlSIhhqqxZCbLTui9EAAoqZoZKvMY5VTs7kbf8Lu63GsPbh73x4WA8frfX4zdWVcuMybVmTEE856aKL+68HNJtPXL7UXp48HuVuGN39sxR0NKDRq1ZcVcMXQP3q5/f+7B3c/HSDv1TVvO43qKIb3M1jrt50/UWa6KBabJD7O4Mqm0nY6qWbPON0oZLll+ZXcWLa8ClmqcHzxz5oqPUmfc+KPbyFcKJrYE6iIeG6YdDp2WNlbaWZiM7as8akLGwpL+3Iy7HGwlhoAQa8YSjDa8MeRuIvEAEOIa6SRWxQrWkxJWfM1cqVzvjmFMfaz9KpWMfd3rxiNM40HuE7OywYtbGNiwOc+oQ1VOtsNFZ73C3uPWlA+bAO5Rj3PDdN5RI7nYf8BmPd+tX13YasgYmzWpXTcvNUggS3sWUgniwl5ZRXK2YnKmM+mvOSDvi+rlb5L31bMaRUE6v6sDEsrsKRNHYLaGg75hsiqG1q3ve/uV+yd0i3ItEKFdNPi+6y9fKkXRNbN4q6DWQbzJLg5Ymq6IkcvFN0SFARZf78KlQqU6TCiLcyJfOy0QPpB/3sTP9c5Zg4a5svueCabLzqstW+lK3y9gadS5rSxkRiU70QGhtuLa90OxkNigzQcIp4T4TAWib1jxodlZq9qUEiS2GS0NLUBZkkFwZT2T/C2RXz9ufCVCx+/+0MbGP74od/O859pzCf9NDClLEe68zaUTOfsNUCv2TmToicctpAMmxWXfgo6ZFDfCYzzANJN/bP34nYjKdrUqR4ZqICen3MFcgLdJGZq/+ggnSZQmy7YNK0sBiogH14ANABw6v2ScEvLzTpqmwbaZVyCLzenewu8qyQFrEn/+YAw593NbBnmtAjWVADV5t6JVZXjEEDGsgNeOrFC0OjP1NS/0kXZA2a2Fmt/N33VyovqABLyJqhVFCgcTzhIcRQpQ4dJwTX4pI0Me/fHUAI9wx8ht29Pe7vnIPKR/cNyWt19ZL652ZBACqSilyhlVZ6yu9+UjxfJKvewoYuhmTBVS/Jjr+IgojSlVsS21gSbfkezvPM8zZmuAhOFTS4PzU6452MbQ19IfSBmWZ66zLQyUCMuQLuIsDUNLHXApno4fCtmrGmVLItQcJyEyFTRUBpWkywpCYrASWp3A6KehFp9bIpVl/u0avFuyo5uS1b2v7AJWo4grbR+PlVgHw4WIC0+BJUuVioVzK8JjcyzGugsf+YlvKMoOicsc80R3sEocACgJMMSTjF9wJLiwk8A+ghXSElxgO7k//AKawQtraGgKnbGMLURD0aqn+VswRfaiPABhbJzGsgnfytJpjX9sgTJJ1LtgIpMuawFNcY+kWdlQDNKopURYQLJCA+38WzYzJCkyIbWlb03HG0l1V0xw3IxlamzC8WfQo25LPwCYBiAlNZfiRKSmhqgwAj101MatYsa9nbFfll1JsJekhPlHGIgtwTR3HZmIaqlM5nB8LWMqTmz+5TZU/6TRADEpyIBU2CpOeX8R+ivLP1s0Tb8DuYLTrIxkhRw2hbPVPdzDG9KBYCGQJwEweYjX3OuwyKcUY9SLWBK/owhNjyRRJ13mZIfbXbikAsSRayPVGA+tLm4wlHap9gLUl5syr69HcKdHhPrlENYsLU/v1JST3f64FITEsRM9w8LJAKcPZq1uyCgziiuXAng534eNoZnvxyEbhgHtEgXLf0QfZcaAAUCEyY1lvfPNTbV3dihG3hbq5HUUW9Of2VO2CfqPY/TEm80E69vu1H9pAyLQjjeQLYUiQ6Zh5KhRCAVl/m8AIKKB26HftmgsFk0aW/tLsXIIuXzGnJ9lp4K+8WR1AO080F03DX79IJjtccXA9bFVBEOtFF6/fYGAsFukraLOXLOyVwB+2MvXJdTpy3AW7cV0HaDjYyUJWLLjROiAsSCm19eyMdYVDEMs9+LwF57C+OrJd+8Bdw3uyosgEZZgpQaR1hRkYuaMQmAJ62rFQDfK6FEu05E50P7Fprm2Ck6meFzOFAf2Ny6xyWdd7rubPSKa/YTBlX8SzBDgdKokYyhSoGkQu5aPT/uizSwSofZLk4H9i667CftfWg06+vdCBy+GGHwhABlXDjpwJSEPVM+xe/qkPdpfZlvLYSLoc7DGsKpkKBha0Jt202wVQTdYpLI8HE9MEVsnwbEblnCplveAAKnPlcmBlPG2lzSAAsQ0vMp1OXoCI7vgRZ6K/TduWapAQ38CvXZXRPd2av+ulvUVeL1qJUujc3Seo3312OERUF/0nAizQOuL5PjDohB5keU82LUzYkdl+ZR8j3uSA51UHf4Y2b9ItwNingP6kCMSDC45rlfaOCxaIvWeKEEEJK0sHH3hF1uAFUoOROO/zaJDmBYNtb9hNd19IbtTMFtUaAnU4l7wFp0Xe3IMvvcjyaY18XoAYcuN+JbFuwVMBRHh83fdZt2/U3WBkRtbb3P4PaRL5pu3WtndX2S6v4VMtQIDJAqntwUkCNwRtOcaCjtxf+RjJbpxo0RV0NpE6xQlHigD+jyf7qlGm8j4pBUvwEyTV6M/KZmBBMPXWTQzz8lQ1uxchHTaY8RpjwKT+FrTANQBzvEKRCbJm1G0w+UDjooXEgdDxvzQJeo2xCfYwYVahIWStBHoBFcs76C6Yuy3tsmN4KxRhA+glqVUcCqldtcKmw1OMadZ9aFuCQ7qH5HWJuCQx7TQagEMTtJ2wNZD+WJZA8t7Raa8Fp19ODpsIqBirqJKagqmhZqgmF9dP9lxFhSEItN3mzV9uolKMu/Eyfkni63UYApyP1eWnr3M47M5x13gHUoDctYO5jkEMEAeWmqgLam1h7NTxuSCor3+/9v0cQqsv8x1YGjfMNgKKQqBYbRoGeo1hqGkwKGmYi0Z0TEUr+ydlkBOc71AYri7CQYoRIuQ1GSIdKg0NNiJt4TQAEum/IkHApsgojjCoELEq//IeATu7QmMVs9KS1ygfzmCVBNRW2HfOiG6um50PdpquZrkd4tXeaF3dB61tt4OAlRF3AvtKD+g972w0I9WuXzwoQ7FMkMXvE/o+mo5cJHAMCa+RvRusSt5x8wzPTX6Nzzqs6qSrjI6TMxCF5dyEgoJ9Gu9cHmLBc/ws9ktTVr7m+BxlxFhcDC7HVuU0hp16iExN2zwOnKHyMGRNRktT5PAY/K3WEQ3abAej2bT9LOVh1lSunF4kBFxyUls19FbwOfAORkI4aLWdZONY+cELzkxrgoDL+DblQ8uf8LjZDdPQBYOQd3QepgrcvxzAJaewkzK+u9SNjhLMBZsBqrJCynppqAAWnMDf42TzCeGMZz0ZH4E7321Z1Z3pYVB84N3y9X2SztzWkgI/kzjnPTruSyi3JpPLY8XdTsHh9IcvQ+/WrSBFNdsbzMf8tY1tBm2odi/Ff34e7Utjv1qE3QTHY8Kj7tjP7A8iYjSK82MHv1FcOH++3lrYfe8mEFJ1MYT5RLhpoxWLVfc6G4SyG46dCeKb6fx756NLr7pB/AaDAdAfi3aynwE4TPXtCTxUH8nyu2ij3rZUhwC0/SQoMay8IpnsLpsxP1q7mITcZTkE7b0BT8W1w7kYngYJzdXVULolhJPbbyIooOtXfhM5RevEZMDsE+au5fdAc1f+jTqMRDwZDa/9XjZRseAZSeYl5uT2EUmyjI3AhZp+YW7rh35Y+GI58R6K1hrPe50hWTxw2DQonPxuToOAanmUdjoZoNoBxnfuZOzycGAxamZ8oJn7+7e/3bA+On/sQ5PHQavQibYMDRkAtPDKdQeDu+815nHy+fOqWtrs7wIOUIKvK8iBgN121eJgyNSCvxuPsn3GmA/EuHhuDuqCpFQwZbbJekisy451w0iutQDdAsnILENknnVK3Vcm6uouLbHdTkmEtgZTnnoolQ1z24ohJ2cUFeM6hro8BUHxD9StgU9kXCpsegzUqIspqENvecyMyb0s7BA6r1xHJb78s7UAjJFWUYIJUdeQAfXBqMHkyMEFF+kHfVHKrkBUX5m/AgXHa3s0q87wHkwDoOBooUxySpAprjOgNOsEgqQkIgLo1pIkhTJX7j7hGZb/PTt+igOrm4r+F2hp+NG0C+WzD9d58ID9j5RjZAT5TatjfuPNTgPLj52KkEz8RYHrFOpASSoYTmVVap+ANEyH1bsvQBxwwmotkQT52uU4vjF/5kGCAKmAm/keHObWYvS3yLz/UU6AjIMw69XEK4CBXO6SxLhzH4/2viJCeyC/5udpGqHpSzeSBDztVZ0FRo5OCbM7D3TPj1naVLj5l/sRX6TSBJL95j2MoWA2LW5JJqdGMCinBVG0ugXLwK0oKrzod2JkwWeeWs2+D72DCBJVdDY6M0UZFBUzDX7yjIX0gWi07eMMBjnjphGwboh49tMgDt2zgiKKCmxTjoowq04/UNuptuk/DyHrbBD1xazXPXH2zlNDxtsVA256IpMhWn4NiCaTra2MO9gdq85Xbx8bOXx46Fn9rmXNOON+rwlTOcuwRwAkMEEAtmIBZSJeEOnNURQaj69sWVZoZ5UxnitWOzDa2R9ICAhYJuxIZkwQZaLAiLrY7DIyrExoXbUBZBh8pX5e02O8aEYUZBuu3tZgADAYH0B3NQnFFw2sn7L9dJlN0ppUY8LouUhn49BsUaUgArS+zUlUwxhOhwFdLjVjMLuZGcMLYQOKqCP1x/FkGGemwlhBCyAvFHQeSyyZU/HFuCdiwT2RQjI3MklP0FA6FcCYd6Ry/05v6DGFxuna22LmrtHLFSismVIAp7V4DdVgK7e7T/UAM8YsHvgboNoP1JSqUw/bTQWp+s2bz0J/tF9eQm3uqsC2afRvV9losn6nYoOcXzqchwezOzeHckzhWgodhmarnrkjAlHbzpYlk+1Sj9Qp1EBhyOhBCdRAayM0xG+GGJLWcfERSjOvJ2jbVKspL7gxq4Ck1rQDRpf7tPh9rwcI4nmtD9sr1VLoX0c+cWYxNn8I8Gb8KEfNlp7qXx2UC9CgStSAgo5wcB5VVHR2cmhK97wo/9RCDkDzozuV5aFoRT29n6yL3cfaxDuOZ1re6sfOI0ZmPuChTiuLHDotTRSp/CuNEWd+yxXA+lrkbz+1ZYGYBtyzZbsCR193ldnXtff55VYg7uuLfZgPKGtrRZm9i568IrR8yCR1y4YEO9ppwRQjtD30vQ+GC0Kzx74f2EKys8+2zpzmTOhFqmLMmU601O61CMgUD3T0GsKkL2951hdOVRGzpb8go4vCaxKG3LQJLGCe5oQssggKd4lr1gZ+oZ7j5he+vPbuO9aTcSZKV8pFxnqJu0BIYMy+A9wcXNzdJzErIBb2XHussr5Ufhp61kExc+c1W/NtMMTOxuao7b9/izU7FRq/OT9CfQ/T6M7HBVxeWv7m7gMrgKY/W28yl+Z+D4PmEyL5YXgtnkH0q4VSaeYjH1HcgSLJkym3899Ce355DvFaU/UnKPGpbhsVYfT0r34ztoup35dCYHRFfWa/JUq2xgRaKnmIzdoy7O1UeAvl5WIJ6hMyZZMNbuCLjctXRoYDbACzB8tmnmNaN6+vgloCrwbHR76sHwSe9pRiObYuwD5p7B3cfa2+vOQHRJnt/Sxceaa+bjF7DrM3d7jnwZ+/EEJbPlE/TVvYFsFI/9bFngX8lZcrHXj7/qWwBwlqkhIVR+pmlrbF9iVSBt4eqtGGWQbTGnq8oH21HGLDls+bbV56mtT4RrOcZEGzfH+1DQtyHaN0xvLMbjBsTjoWkJM6NaCeqpVs9P0/lX2d7krm+6ZY8mnzJr8c38DeMbffwUblm90UYwBX1TANKT/DA9tcz/2X136bl9GAnvZ3fWn3DSNw/AbqsFzNl9nW3O+n0RNsp4fh8SmpCcNW2gmA3a7lk2XwWM9LItQj6cg3jkGz5h1VQXv++NlBtxuji2pTPW0yZkCibTC3Hp5PixpVk+GwFikRoBiziSY6PSd63BQVE9WDevhr1cA4aPsWpbiEyQnrfIKbAmlYTMUKWK/AGSKSx880ndMvq9SSs+89GwBHbkzGvQ39gdQDwK4hBKetCJWf9L98O4GAdfYSEi5i4Fq/EEmYqcxCOUtZPb8Nuf7XF+1IwiMOL8MIHxbKKAUhY8zRCLQ19lroe5LQERkspdTtCWOPJc4GBNoRUbKRnALaQ5VA7fmy+BhWz234u9edKeS51+OH/eKszKihl7mFTFSSiBXKngrtxx+k7NAp1XMuGhjYURyYAGrR72hIhrFqfXdVB9lAdpCFDYBtyYYwy10Sg+oXiAtB4qgTy+aneAW3MNPInKp2JCH32L63DfW0IluIMw1Fh46+YVS7IB29u8JeL9zku8nnaAGSVMcSCtZm5RdqF9IU+SJJdJ3pTznxCmuwxtU8dB3UsT8BJnljUkJURJbuRnN1SWaImZ3BE5dP+uLae32wtiD/n7UEHS663cd/D9KFB3Sn3Rt9OAuEUTfdQIIyiDCcBm2GHo5nZKgJxWYC4+BIK6IaXt7rP1r8ZWC7dpPg/oPjkO9vG8faZGLMN+LO0KRxufGXvQgl/HlTOGK0ZAeYguiV4JLUxSsXYjC27XoX3UYEPYlMuKY8yMUwMFMFD3M1JLLuto7kyeuJGGyZFxY3eiDitwbzyP25TiQOXWblmLQdsWsDJ8ZdCYxqt7IRPFv8zgTbhCvkl61V6P7Tat6Tzy+PwF80q1gW6k132/Os+/eucynPMQTHZwI8C5V6PH3wx7EtiAC7aTvdI2ALJTajlfXKbx8jzoMPghkJb3twW0L/ypTKdXYRgV2JWBr/4zakFOXAdnxt3h6BU8cfrEzg7LDYvtmXd2e/szIAgu0mga7+bwLkNjZiW6rHB38pPS8dLL/WNS74A08h8M+tOoRCdkGBpVRCFgx85exllrAbKRIPioaTa+4YWjmZ99Co0ckmiqSXrAz7vVKPtVDe8EriZWOCu3dKwU+4KdcfvRJ3qKGsq0CaITqH4uB5WiMF5uBw3Kixq/x3pmkyLVdyavF3wrvSHgH3lGTzouhczo/wPeyvAtejRZyPY5Ft+puesfuzxFWrW2iBqb4UIQUAz240GujF/cS8RCVkr6kTYKitcFjBl5Eo1iU4LI0Fw5wAh0J4wM8D5WoskR0hbpoU0mZOU5kvdhLQ/LEMMtQustuB1m/y7QjMvZcVTxA069hCLzl7QNKoU9iYbnMj92tJinK/iwHNyWpbxTY4tr/PYvAK14JZEGe3Y4SstOcz3xNErGxMdbE+Fkvg+BKTX72Nxg1Sd2CVc+8zavPhr6UcBn3RgIY2TbFJH5sIdLg/lpzYm4P93YKLPg7JI0s2AgqKBpcFJ/iN++SUJFFk4VNp8d7YA3cwzU9xPbYKRYWXKlV4Z6RPC1FGrGaEOM660sZFLQz5E8c9sWYEL738W6GEZ4SfkVihfvfUcYfpH0EW0rjE+5bRq7bjS9S/MaD+n/qt+D8IWCzbdhAMxRi35IHe00AffN7gcTr0f3L8Q9gUt4Dmvz7gIIqmmYIZifZyhwPWFsovnNxOiOQ/a0PvFU5gwmS4phDSpp94QaomtXA50sySQhHAAphEEiipKFNFyD813AF+4v0lE7NYcw5IaFAryoBrDYNODuTA4shv/fPxdSgFltk/fGpmBwhPGNrWjcD1fqLoP6JtIb5qzuauYXMGCBGJnUPNWTe9qQPb35eqEuDGdQoroM2eRWZ3Cdgy+1YYz9V693bf2jSwZv56CSx3/YgJCtU0aOd/voAO24LcfW982yHJ3eIjWBjx3Bg8GnePRRy3yH3/GqicvEma5UBj5mKqa8dEAsotmc59qBmai79hF24oagXt+frQ8t3WaVzwrzGZIW/0Kubgy4cSjEUCs4j23dtvwyvuPnCD57rddzs0gZzHMq5eG++EAYN1sdB6YxMT6H0THMhEKTvzA9fAjqXLoezft63rN7U/bUOm6S1pE66c0/xdhNDjR08ev7YHKiMIIW0WGlo15Bwcj8IQZYOTgUmBMBYRgvgieNL9GBftlVTOkwvBBBgQsknBjPi9uJ0r+tllTABGPBtrItwXTV1uHvDLl3RkbUBf4yHJXgTinxZmrxVQYjDATTgEj5VP6Vs+Gm6c++2zW//5lW27wbqH7+Pu63po4EHlzK4/Qt89fraoQXn8ZAt8DVzaxDM++NzaUeEP+0S3Fzdc4Nv1NjeGn3xXiMFaYnQCQomvDedpQczPVTUgWVhYSx7W03dSRb8ICg8L3G0v5xBGQaBfqZoEmcOiGthpf7UwVsCmLPhc226kneypiQGkdBdlBfiqh9pLVaIJQVcXFX2GDuA/O6A1XWwyMr9SBG9JgGMvmCOWqKopwAQ5rdnyGAUROZNrsWPRsd59jbXjA21pwrV1dv1Cpx5FtHK3MAYF6JAC1+gi3NlhclU07pNlHWYOeY6hXR8a+QuUVk9TwMsFEr8KGAh4rahAFk6T0Q01nek9fLzigx25BYnx+309hxzaancoptvAUqQO4FtvjI+zshJgGaqAHyxzJiK+MMz6VyefmvvDpJGPOpCJOAraPqfTvrJVfn+EHk8CPaGn0BpIRo6+4FeLclrxkm1pCc74rc3LhwyADDUIrJC+DOL3PeNZ0fgdFnFpS3Hg3wGW5XmUPX9n+qAIBSFfHqptUoKTapNDDyy8CMWjp5FFH3trqhDVxpdjAZbn374lopNNpFNsAqTSHst+CwyzjJ3/mihTgrziKpcK9EtODmDt/Q8jbTvahD0Yv8U3hk+UmRt2FjpNF9MfvnNs1H5rrKrHg5ofpkBBRzqrYkjNmYW2BozgF5mFGv6/oHAmWcBOaGaGuq8/XeTaI7+4J8sW+wmxEVKfvvZuGLSph/zbstHbWj7u7IEcbPrXGAuRtGVSgcMSm4rQhbd7ILI6Q1S6+kKuYcSK3Hwqj30g2oxj/EkmAlq+tiAjp5PikoID1/65T0z65/em0GPuCA2oZDXdQXoqjb1erzOTHZzidnBfMsomfGtjbPjBqHbYNvQ2yMNA3rYz+qZbt6pbIM1Zq00kEc0NdYg26G107yr7VuafOLLtOq979sxlFsEGqbCX+bTaq39q8D8iSReKOcATtZLjgnjWrwZQ1jHVkWsGnhZO9h8umI+ACvnKsk/U2LJILd9YzukjgGyDh/4bkzBeo3WDZ5vEBf8PxP6/YonaXADWFvzCDQ9zSG0dfhiMemz0PUZBvoYA9VQL/fXGAr7CZbEPg368ewE2NBhMG+WFKVs8dYoYYstsqSPDQ6z4PCpw9MlMUwhegEqSf6UgvK3OuD/2CTHREVunW8oW8MYFkYKaqklXYAGEiiIhXq7sKq5dBsADO2FQRacsTt8BzUiONMMZwtZ273T3YKvn8+dW8KliFWUdcyJHNIEDnjVrl6mY52w+DGYY1VD99J8/+DCMXt356LNvo16XHhRGMk5N9MySSMaJx0PI/IwlEBlDBb1X2Kzwj119zAYENV/02h2XI2Ua5LwjcSO+KE7K6ObzqzCDHI7CCYNWRlNXjZz94bTIUczxUbxsVeCp64IHm7OTtRJiapcTacSBpUsfe5zMu94hH0bxyVOaiJXGy2qSXjrBEwcn10/BPJxd80jMIH0v+lP5fbdffoMZBSqzGnXgAyA8NTd27g0Zha3TbRWqHFxUmVagN+wKNbQPCnoRJ+zq7imDPZUSLXjBGNzV+7zNNPXln520kZdzEFdJpATOHvPH28jUuodJD7SL7j5OPVdQvbWB5DSpR/At9QMqKmyb/+oyolrL3Dv2AiLoor6P5ipehR33ULaqaDVqL7zNrfk0LokNtjSyN+ncl8c108wjdgyRdz/T2HqVQVOg6kRmJLRf2ZoYzCbjq1uATS3uH/Wz+x0mmhL0sgqeQV87lBD1qL7adPWZUPCgn8nrujqlFBRVZTX915fE2qnV03+UTrpjB3GzrwSODIBYl1ivzhRUJ5KeLkTpswsXq3QI9oJosJmi3ZAZKKoOS8G+ruX5yAcop/X8lo4VgUVYdioaWzEkgVqOkaoxHvTAuWwvZnInG9uHFrBeC54/QkP+HVFUgcsq2OynH/AY6Ev2xBSqGXV9rqxkEnmtbmsmU/wnHl2vmvCN7ZHhj3xTw7auNUAWBrO2nVO33IZRWwcOb6ymiB3NDQyINZgEun+TfSP77x3gMZ3xcL4SkY5BmsT3QM4/Wei4wFt+WG3BK581ZlxIUOIQUxyow1VKSPh1RgOhfpYgYJbFlEfP5IxXax7Bw1xRJwT1ZdqUCKydVEGHfLfQhAFcsz0nZTpGw++kg5YBJFMnGtSTfNVCKEHVkA7yo+nneK3RY91Rya/oW/UGAUI2DDlxRadaMNzYNGo+GmIMq5mFEqoTy3PsHjjN3KUKQpNVuYeTDsJIzJdqTXh+Cm2lvVI5U7CigT+kF1vE/urX8bI+qasd98VwT2mbI3Of2xxwXXFME72XaAmxig3feMLa9Zvu1IaZSRhbTJQW7945VLzh3vjortfBLbWVd8JALlqm2DmMFGDKVqBZZKxs4GK5Rw3amwV5xp26XNzeLDINdwawfs2z0IhFgg/8MKSNUzIwlPPDhQFjaob5AOk+b7uzr2rEMxP1BzcZ+sCFfrNEe/qlIQ06xlaOkOy1fKJprgLBNrKE3RhpobkTpb+VBO2ChM44NkRu7bry1ko+XBV7vfUbhu1CG6sBdsuC76KHGp6fkW0E+2vlRRTOVHVQVvHmSHtKC7o/vX2i1rX6H77qh/fpvIOA9EA9NsTY7aoRMn97f5b/UkX0ddQMa+deNVgqw29eMIZxLR0abJCBeNs04h+eaxKZS4fMW7LAe71Pw5IdNsmi2XUjyA7NmIuCIlTB8f1H5MA6FG++Ihtg+5xN+g3oR+RXzo/zWeKbTdNNB7vZmcf8NTztPZjVqeGGrxF53b7FXHRKALJeseC7m+OuTa9u0DW01h0IFmdLX3erZuXfYdIYkpStDnQor9QxdMcG++bWP0c9dnRz4pr6/4s6NlT8Qk+FuKULNv6ngJ+2KD+e+Nrr/OJk/pxqFBx6jO/XWYUfv6KmW69k5KRCYSqp9tA0zApCZcD2nTuSv/wI2PF8R4MoiHWJAnrx11voSQ+XbXge2MaWj24ayjexJAQaNrBtvP01OX2PUgW3ZhU04AMRzKyTsVDBGsNfWrHABTvSls6ct3S/wV+B6B3WNnqe7Uhu3Ohmzp59+3wiMlxR03wjlb7Z5SHDyqKA+onii3K0/FJPes4AoZAiSJB5aTMcyYg7boagzamjEMFk2GyHFjeGJn/gNEWbp9aB1GRnx/okn5KDmrQqSbKjLxFbaviVf2LSZNpB7pGdTp2f8oXa3XYgAir061vBD68+aBtak3zQsb+i5ZyW2RwAxhBVwvAhwcynwI7lhgZUcnQCrdhk38j9GxRMUCEvAgvYJAMXHmj87+waq0ZexFL3vQ6CTweXti17SqR1Qt+JnDiK4u/+oZA10Rg90UbJtD1hQAixNj/xnb79Rta4eUq4+IBJ/4+4DjMqHZLNu3bDcWynZaWkIQQFXZAVvj3Ha06Y8kj+ejxxPChHDV02ygXD8Qp1o+GNixN5IiwaoqZrtqTiRxpgueesWRJt8Gk2L1WMiYj5vGwKFGa4KPB8omdqf5gH9vP7yADFg/5IFLnTcqLPj/ba1CTeNv6twXVjNRJnH6RjAILcSbs/QaXDv1PSPMAU6KCBpkRKnX0gXqlPswAHmKpokfbcwcS5osF3PwUbwLXBA3rjaw4Te5MrXa32mUflLnTzdDhElbO7Cisow0AnbjpfITV39PS4HkiqfhmfwHGlrVK2jW2GIbBgmu4KMVujtKOrEBNKr9wSgJ2gDeNtF6JCQbmFA9ewORf2/+PWZHJDN/3Tw5APfQ++8vYHPESXJ8I63LgiVrvokwPWH8VWNYAHvfn5qQbXkEPjc2cdO2HTM/Jcllfn1kJzeTTR4sklHkBX4T6OWV3+/bX4BrskLD1Z2TOfCwpMMOU47C3jHT8nfB/DD4G/BYKk1DWDqcGMEaQpm63KI6uzxN9mpGHN+rPVOyv7LFu4TN0RjZeKKVBUNpIXHefCTd3lJOD0boa06rWHWcR6lwBTeYfxHuil8jwJ6bgFSzpRS9mR1LUaXGRe0u8OA1Dfhh9uTrh2vfpBz9A7Wv6gZW/oK27FLIqSRKVZHIK8PLlq8Qbr5oakgPd7DNIEEG6sg07H35nt20ASf6Xag3fvt5wJBZCgpBFWSUDHWdoSm9cIDQS6CawJswwhNfTogPpa0cf6DZczmoL0Am7weL0Zl0Cbcj/f8f+zu9pp2zFbQa817JlAEpzo0EjxFRUc1HZQ00gH6ad4x/kWo6dywtmTAF0Hlk7PF/+kDXKbm8iNgj38HDzRqzRFNtrqWnXIIacUs44NpASndAiyUyVoSq7pngrI5KR2yQuMjGrKaw480aG0UNhFpSD6sQNzgXqNIqRzBIZJ8XgMEGUew9va9gog5TplDYZQ9eMYLSA6kV8vQpmdqXJFYMLO+RzERzblz5DvsmiM5kVIjrZyCQ2nuQpoNzv6RK2o7gduFaGNVL589mYna4jNUUxr9KpDnhIdUP162YTK47eGSMoaluGAT88au3fRrevJRd7VMTe9VR69+8BUqy6QYbiy1OIE0DvQ/dSnUA6TrLLzv1SbWLVz41tbnhyFHN/ocBkcttEtocWesZbDMUt6w7ctoiFukHsa4bCiFWggyhBPR3AEUIzviwgtQgbkx/IvoHEo7/vH65ocAOrVDiPy1DtbqVOKFwRgYmjbillD37qL4YSJKUu0ztFzRO5DvkYJR69JcAzoWttB7LO/SlUdoB/spVe5DuLF7s991Uhgxy0HdQycqnujZPBavN8wW8CCoZlzLwtIcOTJI/txntcxmfWuenAOdnnaL1zOh0VVMEEEuYwmpp51LsB6pHwjKBTNH05PqJl8HYrAhEwpU26P7Jfi1ZccxFtvz7y8CZrGf2D+GNYzmNio8eEmr7E7amRZzVWb9lTgWsT+6zd+TEn/iGoQ0vdbDjwZ+kpq90aTtidsYmASEpgsdP0g6HlhwShNucqqp0GGkQUBTQiKb5JiqHpF9GM1leNA1/ZW0kFhlOPLUJtJFIS69IWjGeX8my+924Xcs3ycyDHcvKRMTLGrLzkSSQLJ5MV/e+5y2rQkLwgBPTCR+DylWH2TzXRd2V0wxW6VUkTiin4HdgM6fImssz5RtzS29OCzNtq1caO5/kPOo1jcLGb9IGkOlylCKVdq5EVPmHFtn1PY1JeZDZ6gZt90GrjNXmPGLVBcsjAy+GHAszUSSkCyx5hux/oK3d28+b2gx2TkIkgQZ4ZJoxV+zvWvLEUnwPwW2Ej6+s5TeZN3GolBtKYFxi1lJYEPgFnLFQMeMATDagTd7kpyQ9wTXc/m3usnbIfZFoNtA6XlRaJcabYbmIy6/omHE40qqqScu5ezn515t9PFC3ABsJ9bKYdU/IgEQqto4dMHsFr4D6Pltr3imj3xPAVBFwMNdxmP2KBPH6q3ZPXNQfWx+5Oj+ov3Sn3UasyL/pf70CFeUZkX/vjWTVcmzp+98Cf9NNrcBoAmpOVViJiONz/yPWNUhDW2ZJhFM/Ag2disRfNvTAZPj8cvf5oe4xlekSH9MC1cRwRgKEmqof6QGxd5TnAibbMMpLU1LoDylrNPJzw6O3UL4sUf/ujvEB2j3+JK/JOTwbJ944MUdLLaQed+49uLxKqqIzUNIr7jB19bTq6hCmywRdMsGr773Go0A1egmsdP/jiUTOzyJ8g6GtxveG1tfJiMC3wizEQgw4pg396ltPVqCGWgvaMKYQNg1O6uYxNdqSWAHT3xwMps3bMIOkX8gojeUArgF1dunLahHqRXGm+lV8Vc/7WFIjmnoPs3eX3F0d1FASSNLvXT6U/s+zpgE2O20RSGZmTqYT2f5U3gfBV4zGgWmXu206Gu623fRONkJxMh2z4ZqFu/pDc+88oCK9CoqEqaeA5L5Kr5DqfsGGbbAKAT5vhNIlz59Zz/FX0A7mCgnstDV/8kMXTjS/nIKwY1NFWPzqKfGYgkTsg6RiLBmvV+vOJ2rw21h0h1MAF/3h9Aw+Lo7yfhcpHro0/Tjar51tLArSF49qhPuvNFZYzlvT9qn0j3mv6dML6nkPUOiI/B0uPj7iAK+Npx0F1rzwxG+d8N3O6KDb8ZSrmuz9s3HOzPtfsRacj0L7nmgwVvRn1jcSQskgwUtmycjqN5PuatpZloAIzvig0a91H3E/QC6LAS/5dvwQQwrhvo2V2x4HPfjOuKxDAYdGDSoOlfdS8Git78GZsExZKgbHxH4d/mzVXmrIQHhpnKUneKYneMPGkLGZp+I+M0PvhqdqiK/AYsia8So+4G1UMVHEh5n1SMvcy9hPXbFhs1mzh3QD9ZJjHbJnI32IPsr0AR5GZ9qlcIoqAoiRSAbwGMy7JkfGsD7ODTA0v9rj/o31vTzp+DOPLJ4Q7pzs/rfzuxpAmW2h1m3uKAtO01vKkkdruTQpPk8fb6hnBJiizJfoXApzEzdV3orejfGZ1BYaE8TOcNJQLR1UUMA2nK3fpDEBnOhkSmuOb3zkGMD/dRO2niw3IVEZzpTrQ90Nh3H6j8B7TZb1f6Cd6EkwH2hEqS3FjVUDLcTKotfCf3NV1USIW46AHICliEXp245KHCRq/KbQAdFNEy4xiARTsUggIKx/BlxvxX97qJGht3A1CYHR7q5PGz2baPuJcQJvRMEp6z6k5hTVgH7iu2U2o/EDlF4dFetxVkBXPMejFZQCRgaUnm0emwH5cFSTlGIZ+oKwrLcV88ezkLabXPuVAiHkXoaF6fAoZWJ7AYnH0xRkMjd5pQOGlKpsmsyV9TtRNle7WowpWNcrlU86/k4xLMJwLss3QncwC55J7NID5qh/roG499ld+15Pq3QQF/CQKaSo4d+WtiiiO5QVMqvZCZ9UVm4IOI+3ycRr5tl2lHdKsldISIdvyGHzXUlIsWI8XM1XECWBHLEPUnRufy4gF5/nNcCHFVAQdqYKEvxOSuWkC+eF1gQFuy8Ng/d2SrUYwtN6haGMPx+gfBNF+2yqD00i2ZvvdiLlp4YzIE6LUFD2YeYfi9BRUI8okA6XRXMRSktLVqIOtu6huqCNZ2xVYNtL7mrD3bzmjn3OR84XUX6uczQK3upbLDjgVv+M4BysMyWzZKJUt5n8vHK3AjELqFGHoBs6nkw8xKMZlDxA+eCBWX5q8Ua5udq+RGnAdloLysCGE5OGteHqbPHjkLXWhyc6LYAEqEtbeBTO38tgb9/RZwMJL+tw7X8eVgCjOp2Ll5IXfWgnqQiKyfSOfEXP/l5aLx9w+/IeCCci+A4kOS6EKMSocELWtOzIidI4FlsUikuQ8pw+45EaL53q/vJJ5h/VblrpnShqyzDqRpCoycWcZrRuEd9MtPb7Umbb+qwkH22B+cRBNtuqKx1xIIf04LwkTxCa9MfNbIjDXA4sUkMGRJwiwypirRGEh8qpUmY3Wsu2PLzFC2SiIHNWW7f5pYn33qxKWIfoIUOoUOOlpjwB4MQA34SRutuGmg88Uzd3eKaPvmmSTzYpH5AUSqBjUnMM8mYgyUWaRCV/KO2No8G+8JWuPmTKL3XxIhfvjcL+28IjO8frbAtdhd9OQfE5Jlk6UbKbRwhHljdTHKNfbQTlimfdy2oRmOng6WJO8IbegLadv9uL9FB9jU5RJ9U06IDV8ModzXCvcOR4E2CIeW3QuSL3s4fBJfjr/SQ3XqOA2tgkHmreWpaP4f/Oact3mJH43bs1IC1IoQMEUiOYRfX79ywoFCSK+kr5FeFnPjU7PFI+4dv83lggKyuySCZFGVATadHm8PHj2YhDHcEcITLampmm0v7186KyMiP+5AJj035t7PQsq/W6Ln+01csGo/Zt6xH0/vsNv2VRC35LPHv8J1x7DJvmxcwQIZq/iqoiG522v+YAm40prMf+6cjtGvjh6Gfq3P8a+T8QSPgO69S3asn94CcHATU9Y5OqWmhdBjZKHuVaFPNoiZs1VlQzPYcreFKRAS/UaRJdOu6EIjo+o2ElXXJxS0C+02zZ2itdJ/hEhrSH3x9kTwzjdjqOZUzUVmshn4zVFjJrfwekrBjwtTg/2AYdA3xDP5s5a5pi07ZaPQobNJNFwpF6J7kmf+HVut62v/3QG2qI2ms0Ic+KSz+t6Z4xTdkOz1Hz776Ywa+PoEnB5pN3f4BKpp3XCQnolTb9Ttz7hDUxCIetmEr2WFd399WDNsm1oPlGGw4DhPGxm/adcOQIa3cx0kFcsNzcqFoVJHDzwYL+WfIDJuU70XT7mdP55Zc+JyH70oTSJpGldHdfmk7ksJY2qZXMsltmCYeSNws4WOAZUiDWn03KxDJRuKya4FfBsfwWEgsorCocdpE1YMn/xIioAL+gXQfbE8XBHgOODJ7SjdE5mU37JRXMMGFz7HlkYIaSKerFd4ZtYV7y+tZFBHeQ0iwjHSJwii/JVJflLb1jYBGNlTDUhk1ui1UuNLJAzDPveEuVPYdJpNgOnXZVdK3oQUOKPKVWu1uUr8EvQArpuoSEL36VkxQHjCAQErxAlk94jJCmIAx0iSiEbDYVwVZPLEeuHGlSZnQREArUFDDV5/mAndr8ZoMCH6EcHZGzvBRLYWDcMwEQcp1ilb5WQlFWIfhKqvcm147au0J4AUQ9c0ojszIKdNrhNKZgqCBWRwwM9rrMHune415abSBDwcz7le3mXdlsOrgqwXDdvxKrWiYKxOBIM8m1CgQ3lUGMC5lJc2hh03GqkGoKFiDSqmyJuoLVN/eupqVsHX/J+o1YLCWmWzEmZjMfRBcPB+rQ0vfJLxeJCsu058x4o2nKOcLBQkqylM4DHNVuGMk04CGtD1hl1xvYqb90QXtxVNksyi6XhVjisJxspEJKTwKQs65fv5LgwVhVJkpxzLSERYC2Y5aMQqgLqGVSjrkmCg8qVQL+R6pLa1DsLVgnkwD+M43vzUcmxS6UgQGJlfWI3n4SrAd9QiOLJ8bl9DT1HHg8GHmLA4EEhJrO/jhZzbguZmU9rkmHC7Z4bQpo62wEa7na0OThMrujroHueCUuIp/Igx/4HsbQ22la8alTKu1a6W4gLcDSSxrkjIy3FMgnpHEEicEgEd0pIgJ6Yz24VkCdGAzYFIQ0PBIK7wso2RxvA/8qgjVWfVadKdgDPGwzUIzAEhMENpkm3fnxZz/RTMEnSIjKb+9lMq5Dvg98bfLLHb5Nu/+gRWVt6/8bU+Lvjj0V9IKAzEOy5d4mOFlwg5++kPPClinJFFIpGqWwmKJqjGlLdX4NYj40Rcr2yemwjxQjozH6GX1l8DSEFarnvKEi06VOsLuMOXMBSdBRrVZJTsGy3X/dTSievHTQ6wMtRO6aU+CttESmgW55URe20DwAWPL2bd61lIqaslP55bNZl9fH6yrERaUw0yTL/+FMGGBHHpArTd8EZWrBh0Mo7YiOg6PfxE1fyth+T113gvA00y2TkDgjElNkafcNan5YZ8RxetKkgiA/5XyDhfePg2aaz6bc4PmzC8LV1DoGlc+DCBR1YcOHxsBjBOQ7VXvQmytohT6POv8ILLjyBkBjFL5iF3sr9D9RHy5vFhl2ygIjNVA5nRLfsAKqbu+0w+1l1XEZ+aWAZTNTbIgF3JuKddWTDlFMOdaLTDbmpBHR9TuorC8GB/wQVhTaYySwEnx+RyBcfbSBMX3mQpCtfggeKrtZdiX82cKtsuRcHBCFdEN5w58Vj4AixCodCA+SCy/zaWTbjYYRVjSkkmNIGpkiQ7yFS+2Idnz+we9+bbDLJxx4048mfL4b+n2ZDgeg0voi7gXm22Lhy3VuqpomRN9Lw6ZtBlUoJ+SrgsegQEwpX+degNKFz1OnGAZof0PuplVHgfAarOccyAyAksG9/ojGLWzWlr0L0oRF5fgu7tpFlJaoho5xyorwUqkjGlIrl2nNjlcTj02/S4WnBAN62RuU9u7oIsMNNSq+rK5OJmquVTdcCIlhpd9CXQ/99pqZOzIMWLiI//zkaePvxK4r4ijIN0XysMOqPWTCEGHnUbeDtuS7su+47FuQs24S2oN9UtwW43cOaym5LsQIXyUoNECzZbN1b/Azp14uFoXAWw+G9UfwY4V9MekzDPH4tQoZTmfBQEvqL8+OwpE27KWXJ7hWbaSKsuxihPwiKuFqzmr1gX5HAAzUfHlRwyu9uNYLg2mCnGnRBU+WqrKPf1uXFTP9TDfD2mu6TPBwU7M2rg7OWoCFgCIitWamjeCmVkKfEZ76U97Tw/AF9rMsz5LFRvm2ez6gqhMigPhykZyFbiRcSXE0WVXIly+rxpa2Sntm9tA16vB1jvr9+Kt1tvLobCd2Z8yf1JEwXwqK5Bx2w71ZbVv94RTECDcVmqGsI3eu7dtrDpIF1mCyJrNzk2MBVZcux8fGdNjcNd/fBKc5wR8mJzlYQ4L9aH+5YLJuOI+avEs/7V/vq1Z7EtEaaSa3aahHrDBlbG6GuR7mWcbItGIYB6pKrBBfMo0IPwPfhRYh+k6tGO3rox0Lp6bOr3DoeTq+BydBN2ecueXI3OXd4hv2PRXLlE14yrFIORHgQYZsGxRKwAg1KtHOjopu69XA/scGGiob5Qx7Zbb16I7g261M24960MtsyYlqpis5u5qPd8BUAfe1dPjM4+e7g5t3kbz/r83W7U6TXcP/LAXqRe+CkOivQFcpDpTcZCPm9dU8xXTvePuLdR/VuJQJkFHOKBuRxvMxSLNd94w2S1H98qE9YsBhS2MiDUjahrhpKo2FmoXmP6t9o15HdQpyHQYm2jkXMXD8NbQ64VZUu5C/YA/ns44f3i+IyyxlHNVTZDd0Pan6bM8t4LFRTbuOZ3oO95Cuw5BWniCD14kLq9AC1y33swsSKksJzULMWIQtcoD/24olootnyYXVoFCUkQGT68yuyr/7e/rdcILXGutdbB2AjXS1gD1JW64g/ApjaLWgTbD7o58gbqgjJ8gmbOjIbJ3wSlfoL41NzhnvyypBJngDemQ+wwMYlYG8ypspJsgDvaUsipsZ3gINu4iLqAbExKkxn5YFY/7E68iaNlXhs+2Lq9aw9ngCx7WJf1jEnUcRXG4/Xda0hiZEVCMYGXLV0qTbx90eMkSc6Vkg7ggXHYGhpDnvYWOVKu/hdvlhSnIz/Bxv8mxtK2K9ZkucaXV1+e0NApb/pWzYd2zsAqQvTTvxTA0E5e90PMXI/vqglyRfOKDcXpLGh8uncMEV6FJbGi8k/2de2UXeqiyI8vDo0qRsY/soc/0ewonCIpFzn5Oq2kaSmXpKslSZN1FZqqvmFRKFzNRr1nJqWEIjM8O8UdpuBtZyIDSJJ4/AzFS8DFYq2SSWJT0ex1wYWpgk9V9C8386emSIL/tLT2ccilzklQ9AnhZsLKqFPt1U9zdIexZJDGl4AfTjeCNhoatKpesCO7aqDeYMvVk3tnfdvtNvDLucGF3paOVHEsz5eKUfdiLbhL030mj9M3WC+IshYeUDnaVxtoOCW6cOiXtZdkOox9ZTXjy+mCBzECsiDEYHNoC8++EIVHRESXsrVJwEjmDoLvbiCVCvsQ1H9eJd9aQgvKiB0mwAn63wenqz/a9pr9XgHBR15/1sDW902ehHtpDyQwxVIJgGvuT3wzpk4q4zo6CmEJ5PB4aERnpAEIbvJby6l1VEWaTKBArfBrK2iB2mPRryh4ML2MtlxdGx/ShBe2pLkt1Q3qXUtNhvquIoLtCXTbEycfoS+vztT4TM8jeMOvwLm+VI7h558k/dMogTiBRvnTt5eWP52e03f93X4ueMDaXV6EpJbKgGFhLm1p4lHkxK+0MYx71YVIHu9NzILfPA2+3rJ7pJDy49eibtQjVhY8kERJqDMUximGoQwyAjs1ljoqBLBkEC3Hraai1GoBASaraSfp9NdHu40tAORAqYmGWWvOQ5ZV5BoawhhMpKwp4BIJLPmQ+ZIkyzIL1Fqe001pRuWmwkm77EPJJJr+P6027VbQCNOo2d+aHDbW9WamYlSShV6RkzjkI5V5WbD1id5Wuso3wCU3et0EuR/5PT0A64hc3Xj1Ys6SV8F8q6GkXUGOYwIhMdWt9gaeA1Zbw3VErO68eiFn/fnY3ORSglXocI6pbeEbWLp9U1V6InM2G9hbn148a0rt2yv97FOnV5ab53509ua+bCO+OvpzmTDqS5iRcsXWaL2HvP/YZBdN1zGH+CP1dY0Uzg5CMaXZmAGaZEsWAHns6do4moGwj7lCKd1UzRV533S4HESr9Bsg7R33DlLjTKnWjo+AjnNWzQJvjPjcsTs1h52aDRCtYiPahWCkv7MXLqVqx9pnD+yT/ptcLkmZDrLl9JugCrl0Ra/IIAaeExWlWl4p5/LrwyVxYYu7N1QpzVpTsw1bN4EEhGJB6SH/+FvFMWxmsOqgTWfiXBLj33zEOU+An0ikFVQfqzSwMnh1LlNVKCZsaWyytgtz1I7IXS5r87YcglPe5Po8kvml3MPwyxhyVkkmVB0ANXMNiDZg4ts9KW9XV1Suy6RzbecaD1cC252sI/NRnSOKDMvxIbpQDLvn1Y0W0ZkNONYiD3p0o7b/iKA3v2/ESE7X1tDQTgMafGnlxklQqM70+FH6dauAljFBeRj6yk6LOa+5u0FtUeP+A3fHdi7ytu/AvVXjNbojjj9MQg8Rg0zLXn0yCJ+lQx2/+pNsO1H3F/HJ0Bh/xroOjk+HnSl9oQf9Ic0IUMjfTcL+Rm9mOPZPiCRP2YGvA+66AVP2DceG0QGDiDisd8+6rwa3fZtHQZ/RWuhHeoXCocyWjNSa6Ob++q3Zyeiaid+DDB0+/wiehK+uI2unG6DD4wd8St1D3b5WBOjnTs4+G5Z2XbHjo81wHRdWi1zQuRHBbJ6DMDUMTdBx6p/kHDR/Ioj2DLBBAWve0udYF9j2U31lVSi4TxqvOA/9L6V//tkx2qfvtnn/yr163PFDGRyLDMCWwWvK2+G1SXeKlZnxJ4LWbkmT4arzoN+Rm8vF433yY+X7tX173BHfEGUtcpkXiD8qIOXYa7oyNxKW3hDzofr0L6KnAf/M1G2fiPYgZmFsybb9cjAZm989i+7snft6UQDXeGbDMBbwvvvgE6bXiBFi/z5hP63oadAH0cdmkQLR7pLVedJXyqKDWC/1E+K1FkexgjJxAjOazCqcfW3Uqa+GuEZGFy3ahX9E1CH/gTWl9fHHFynN6TeW+8xoEuEQn8Ty+Pa74TdO/ytKBepKMJd1jyBTkX1cHNcJw/sK4ACaCbK2BMQ5N0jmO7kfqNPu5CKw28XG7mJPb7NR6UmPBC1l0qDmG7kQcBiF5bEAR2h+wUOVpXFRWASJhIAFVoxyhGsA+40qj0aINHXsgYVRqa3Hk/C+B1hZM7tSJ1OgioYJIxG3U0S7OYU/XzsT+42XpbWE5FED1clZrVesS03Aeq75IaRuH2PbCBPHGEIUOXTUb0cGHwszgJoCQh0Dc4BVhT0xJjOKPAUNTCx01d8zL+wGJnBK+22b9znjvFKZURRG9xJsuL1bYpY1pnZz+38vmfcn+hsGZR0FZTRxKG7/IHHscuBg9Z59sZ3J0WW9BNGonTpoP63X5kpX7Ak3ISmC75MsPmDDkinEo7/2R94uTGgpiCAt/W9vn2fLE6xUEWvCtYUJki1yqYi5DMOPNaa4IQd0RpZN8l5aiGfqxtfTI/+5kekRIR9OuPbM6ZlkVYpFMO7fnx05YHToteoAvv64HikXMQQdjYikuFsYL9OaWLuOqUDUgmF12KBPzDP3QEh5vdxL+VWPMeq88GLLZLGK5i2DZvvDfC93C5bx2a7VjLqdYT+53pRBDGj6u6TQIy/adrCeOMhLsV8Z3uZM1wbivkfVNkz2qGaILuw3D289EIbi9MAbX1ulmIhdo9T2DgbDE+0byfHJkP5GrY/3255gYLzyAmUIWLvRyeFuvgWLDCdtuWbL7zWGbSvJHujZfHzFRu0o2tHinj1z0lqpTwvVcgx2Uzk9PD2bhu2wg/tHuEEqDQHHS9V878ALGuOinJSfWINbk10kCQ1esA8p6GMYNsc7w3KE5CbSUpxx0dWI1zO+u3rRJNBgwMWgqUUCn9SSuJC8aLGR+PPREHSXEuc7oJu/g3N9sRzDzz5OBbxbmE6wUWYLqVsf/wfi9feuv3+cCx60dpcC0s3NJdGO8li37vn4mfq0Gk5jqHP/S/JZUMN1ERiGu97LjlME+Y3j3EyB6RYL0yPO9L5ELx3j/33VUMA/JcqodCi4ITDu2IhgMKtK2pENjGm36nvI7GgGynXHqdk4ev1GyZMH4kzGXvVXoFGvaJ/FfjmDsyRU+c3aWi0D7PolT3F4WbP/djOvIZtJmIPXbpQpIaDSDDN2FfVFWYKvq0oUtYxP6pN59sLqcbFcsUjeD7pTrmw8upC1ZDEDVkIxVrVKARmsZUBe2dJQd4qV7c8DVgg3sRTjVaogkkk8g3dl+z11pQPViLCjxDL1EaLOw+d1CEitgkgC5jfc99kWqoncsyUllbsB7c+Isz9T077xmv+7Y7z7QJHspsZODXfWNgbf8wwct3bw2ZHdkfbSsw/Zor8OBSxhn5fzs+UJZwBMp39fBEXbuBE/5aCdby6PW5GFrt62gWukPY9/U3UT9XBBFlnuKWtTut+5AgDZ75+UXxzgsQRQNK7A2nRvq7ELlIHk1gNPJDzABS7qapynr04x9ni2s6E3DMACdeKCDw39hQBWJPesiOZA7LAHXyMM2sPWTKK30Zux21214OORSee+TD/o6Xozgni5oOVblyregnuwETaRpNJht8Mg0OJhwNcXooSp5/sQeeMFrMUgustU4YLCdUGuS3fHe3rpm3FflRgHtYvYJL7iCgUHcwcjXZWoY7JkI4YWDALKsjIbnQY7BhMFQliIqUpsG88JWYN2fX/9QPhLv4zzqszI/hObgFykB4BiE7CyRBTaJ/XAxo2vZklfRwugpVZzKKgZet3dzdQfDPszibhrlC+JreqfcO+L9MOevZ4KhsglbAXW3Suhsq94HMRMVnYA6e0w8klqA9Zblbn+Fpr2p0U3HY3RM3mm0UvN9UkU2bN6YL6YRMPC0aijZ3aHhbqhRJ4O83iXEUNfdX33z+IrXyYdXC2PpgvmM/StXbFMJ+OjE4RPO1RGQDZN2jR/AAqtIIVAego/6o5hkHqvFyoz0Djrc+gbg6QUhLrXvmThUfRivwRujzRFrLaLu6N1KNpt443Y6t5PBw5mx7HdngXnGFcj8I4DdcHZRDI+41XcwLwpfMr4UwNYZfPhta3ge/T6ynC3Us7JpYqUKwUuwTB+5uNXqEJ73LnJ46d2DjYOAH7So+iKISuwfCUjDQoxOnJ/vDnLnUbmsTxIbWzYr1aMmqSPZHMX747Ga1V8Nr+wmF0D6IpbJhHBoCx6qGVSK4tnZEkRFNE6snsiB4BNKIXjEtCZ5i1EXPjkdOnMLOpUgOFua3JA3HUyhv0018Pmb1ryZ6Kdui8qIlPAmZI8DEKgv1TGRGD95mQ6G4C54M+RMZNcAyQosB2xjFsreJJ1ZdIBLQinAfQAFkq76IkanpqgkXIiXDRD9cnDGgxcjoSCcOpOn1VaJiE4wEfcMSfDOM8lh9rEuKU7gRV0EIclzMdADS4c/4O7ggtcprbdbQFwchPlvBL/lX4e1Wh+Gy3qMoIxTKz2Hs/n83lsDFbRhKsgw0qN0Glqi1vnswBENOzGYHXw87H5yVzCVyRKDfAcvKBcopyuoIUx4LP6q7/lG5XTQHZCf8Pc4RuXqjZulk7uPOmYbIdInuwumE5zoHuRPVdfinX4r5/bgiuxdjeuJlyETtxx1H0tVK9GwT5zRlboStDKGAXGDnv5G6fYdHZVam08b5vLobyxwnPCjap+/ZiW1MsjODwKOOZuNg819NsjpKALNqy0fTJ+YBLYiEqoiu76hsO3OfRXn6fqr+UyObduZZRS43PjabbSehkjxHF66cMIHG/+p3sZF3M7s8Ct0OU4buCBqHfttN4tNZyvCuv+ql6cMbVFw4YaNW9W0XCHWsMtdcadNVbAN2o0tIfZdeRcQHL8MIFKr9wRV7r0g4rURaVDYnt+BES+Qoqwbq3Ro1C+ab2wIR39jNOUFTvRb8ghG2hG2omYYtcIZ4VOefFo78ZybihWB7vebxr+38zv5LZokeQKrgx05U3bx5Cx4zX6Igvh0CvZyQMo3zRS+BdByrjgBKyAZ1d0jZwSlDqGkQL70nwOCPUC12BEIbkZsyfXCC8pMlEWEOKOZ8j7fTK0V3ZqRRm+yyb65rih8w0ByM2PPtJj0UNT/I+EyampWLt9Z+64PNv12H+J3aoP1n/Ux7159ghUJOmEJ/wx2s3CWgBe1Z57K1ahSyEN2ZXKyxffWtQywaJ6WVkENfrp1RLUZj+otKAzoBkTxNYtjXjhTQD6cAx8gZLq6DTIUKEtLRKa+YB9pJ5rTWkFGVVTUh04pg6tGkrxLdymwo3mZpIDLAjxoxjgDEIsBBQrT53QfSMT5EEls+UeX5LqBA1KezsKCgCLzcA4E1jFRtzbJd0JDrAR3p5Mb4OlbsywcfOKK8B6Hb8Qoj26zVwN4oUru321Tg2WjGmfKGOntwWLCbVVl5wFAMfxqeDRGoU0AfkpJTWQ3jNlLwX6fXtphNAJMpqmoFlQpYqcCgoLKEVpbdvP2gUA+tXGdx6YHPr3W7Fo5/kLH91f0/JqvP60kz7hmLm006Frda1lxFIzF/KPddW+tG6kUhcHFlQoA2VbKMqzMIP018GJssDNp1QFUPmqDTBHD/THCdtNxif/dJrYZev4tW2EQpYvnkTuhLs+S2HipwGtBMtR7hZnkLoliAz+bJEmaHSiRmqylydqPp/rap2y5a/FY+UNbbjdhmdOvPZsEgtPzOB8okWm8/BvXzCw6VKRFtYvweFr/EmMFGQ7wuh4Yc+ELCvLAvLFYoKnuXVlQ85oBtViG9DFRlmTy8sJ1lRkZ7SEa+KHTprLqOdNycR6t26VC2MuKkBHtcdKGK6bdy82g3HUO+WHnuCOlh+TTmarlhYjACIvD0mIxj2Wpn9LzC+3GuWmoLVz7e3aiIjeUh2NT/RNFjxJn43vZUI5K7LYQXAjYbZaZH95tlwMxQa6ztPgOuJHB/slJEOJUEKXWVVuEpOVq8rriszvN38/a+vtN71ev92ZqFs+kFIBiWBxUqg00EKsUU8K7enFq+6USD73TbxPRoi6JDE0vchXY3v5dNBxejSiPC2yrKRRy0imja2kwVuwwA3wkAB7NPZNJJ+9OWzj25PoIP0h9v/zqh9PU1Yx7bTJGlYTI5DinC+ceK/3h2I1QytTSrzYt4sGHdnWY0GTMTkv3yhGVeCisMQ/3RpzoOvOSbs9kW5Z8NOraO63ZxyWuyV2Rm5x3tsy03AXVStDfBOy0FmVOSGbXnTrFil25n5nf5Edp+wK53zNxWLfq5vKrpKdyMEcpqmvAjNrPHnXvddVKipSjNTkW7JMfWFzE+qbUWLkJee549q6caf++mRnD2qb0uiV4Er+iviSW6JcLcsYWHhRQzyNZzURpUYPJd8Kljiv8Z8AeBa0CYDacGjlhmvIqzlupw1YK4kqLIsA40wfIQ9cSjEoIAGmlgF1svijIy/J0gjUwv88Nf0fG1kkg86Nyw34CsN/ZlovmwgRZR9BiFauwNQHRND4MERV5HsKzClrFEvZO6q1IQITyI20rmAb6QCax7S9pZ2PA+caFiP/Jsqx8bsPXT/xWI6cUhWdmNCO8p6GkMFwxyjTZ0QuzZy7bjkfrQYe4I9RP0C1Pa5vBNidOefoIQ8I1Ju5MZwJmYMAcX00whGACaoyIDaVhGR0hKx96uWYyoY9jggsMNXrCgMMNSzAoEHbnbPkgbKz7EKGwyS2y5CfRE8VUhHWNXxZru8KtIb2Ia17a2sIsRBbjFkQVkE4YIdYXtiIQYRhiR/zXrQzDbggph9k+3Bi3OoYSCxzjZIzJB07ou+WGpNlPW7E4xFPu5hxV1IXqht3+2A2uZsg4Ux+42gXlD19mm1WWhdq4Te5208t9thNe/UnsaVKD7yAfR2/3DIkP8AT+C4CwOnk7Yv7ejCwrb12wbIs9GoiTN92R6tWuSMAHlSZ7qjNMNMk1PqghkbYZUMQI+QBfIGfU4BiEt1m6sGAYeu58FNaFUQpp3yNqSFkBTZpurLHpi1xSLK9TRjgaXbZcTDE5FMM9oacJkp7ct3ONbD9Nd2JbFK1mYgF3HduTla6RsdkvosW9G4fKbCqqI+f6GsgxTmU5jmpKgEin9cRwHSraJdE36319KcAbf/t6hGhbb3TR/1veiOSUJ9UPtpI72Uc39dmb9TZgCSEKSDk76uWSt0qVF/Xf6n90nZCTsu33nCKQhEgcrx+grdNN53PyhAOeBvVkbVKjulyrdz9Ue2UdlZmNUR+lymBOYzxe/W6F7H3FThYDuUWrHzOpatPpyxSUno1FF/kC25FGdmHQtsUyHG3WDN7tNF/WdCDgGe/MIKtgWYrRhFHTSdse77Wyk3xkGqVai9BrDVEHZPhpJAd9jTAvvK7csBeY5sUFiOSIGX8BULOP+sbVQOueYy8qfcX5+ClRnbhp6QfKsn6Eg9iI6CXFVZj2rrsAl7B4nRI04RyaKjdaKCyL1QbbijvULUCommHa8PEMxrXgK18st5fmmAIBZgR9kyvpTXagUk6t29n5o6dnPJK0rYrIJHI3ZfZkakpMJU4ttfycbM88p3TiZE2960LtNuJAUWx7ZnFFdltH0vD6TLj4PI5uSgQJqQ4WUDqBt1ULZDWOlYQRz07EA68h1/3IP+x+e1bFdYpvylZuClEDESVTakgEWkDcnDdiqgcrKkTQCzBvt/Aef//ji5GemouQxrRmZwxmUJqhMcLS3lqMC5sIXZoSjHCUIdXGKrDJZM/LsYEMlinyJ8XP+VVPe38SNdoumta88Sf9G/nHx0LgGnjNr6/Q89rMmwXyVcoYfWcek5V4GVyrYoHBHjOXBSJnQy6P6lhbzBT7R7NiqyxmFVBTxffRoUI/vOq1e5RbQlAOy1nIPqubwZsbnzzp1f0NedRef6W0r3nWpaNhMTfMR60p13drjDbYVsE3+1d6zy3U2+QaiJgl0S7pjUiKV6wXlEI9JOeLnYYPUBtv2EKJZ6Nv7Ss3ek/rNhVceglYtDogYb2/804onJFZ0AC3wSsitctB8TSTsmDJWeeSSavF6nJ4N4iWUzEse2tRiwnFRVG44kCm+PLALjTS0jkRb7xei3C72uEoLRLazYY4VJbgsINx37xVTDD0MU/g6rxWGTLStiRijgdWvs3k5oywsNtE/JOW4JtDvzAwh149UrOOEFnIhvbo03Q7lr+lOPoKfI6T9ohwyXCABN8vvHKtVVYpti9UUSsbN1vbRndSYJDIh6cNXir6mFfb6hKCCwuKBvswLRyQK8FNOMzRT5YAZH0GIMtdrbIlAb2JnU4Fq87UUujZV4a5ZQjAlyzpzcYWOaTnEudqG+1Qp6W0cbL14i1eArPE4XXYyw24GP5gUK8gr63n4/B4e7EH6Q8rrJt47G79BHwG5xqqMZW21aaOK5GOT92jH2vcTqmGQO14OU0/o1/2nkhPsgNlIergw10NiQ7E/rLrTAyNj8ClVVgPZPM9FzwxpbhtP4nPnHp+LefRWpJWTPYt+fuOD1swL+4zENhnBki/ABiGLUoXqM4gQHD/NTW3O/PX4CBBMNqgWwvoI8Z4+3mg+6zIAw+JurEsBasAeKjiMUa0hhacv/dF9NJho2C1cLak6FWKEmmCSEsTPzyJ26PHsnBKPglemnPOwNE+jnggX7ftqKnfLT+WN2NdMXSIhoiGw6lIXcO4uRErCXQIg6IEoHhMuTLx/0XEzi7a5VWH6KXyS4c2geSUzSXxEiW+2JrD/dC1ClALo1rnkivvgpH7/zttW024GC3sfTF6w6TIzlo8GEZOTVRplrv4juxnfG+BKLzvzQWCspyVAVRo801YKpHj2WU3zmmVcSpfeagDFfnCNw6WtBU2ZiDln5RVW5nkVRSmu8oosif/oagLDsFcd20S7DgNDrAhRzhm/+nHVV3kTYFq1T93RLafkh5FExg+uHj7i3IxloEFmf4iW4j6DGpBols83q147QljyXoBej/A+Yf/L4D8kY+HuXz8P6rNCA9Ixu8CyfHf8iAGmXnAXL2pLB5yG+DKeViA4NCka+U0bAmL2lLwlCH+5BEZ8t8xaOvFl9tbaksxIfWfdEOeclNdLEvtOZPkdTixwFLJ9O/0L6Y2TB4XcprMCBv4zowNr4PC5GRMq72W6EPeIAYNaeAvwdoTICaDcaT40H0lMB1C0itkwRVWFsULtA1aV7ZDK/JCadU5bkay++94OOD3TbBnF0iHPgPfTSEXU6tAa9T/zpFzmd/+WxvLorQrajjH+lbdd4rls3/5ILKgKUi5xv+Q8sJrsylLf+q95sIXZvSYaev2cRWXP1bZlOrUacXM20zKzMgbNQHdsfYbkGuHO0E6zgYqzZgRsyHPOCz36MUx0vSCZxxEZiCgW1D0aGIFGXWBxduLYHlZy/+GuiNbgDxjffZWsuyI0Juu8YmaLzzeRPvjGibD5xylj7uqJhqfxj5+aXt9XkWkaNiidMaH7Umx19RWk+YRBu+qfRBOKlD3LDf/sHvTgFypDmOFMWipMxtj4eAI+SsBY29o548OY1vyx7dHnkNdgEJXLWurboBfaGdgM/D0E8CTdgHFlRshyRiAeiJ6s5Gk+9fuOk4BQ71Dsm/RUqqp3RzQHAFR781fHHJAProluJ01L9GQjThFqjMRj0Gl8I5sriTNEEq5RoOdCNz52JUzbKepdc90c6CkDEKlZGdbv3QB6UxZwOB/pztGuCdy67SPGvO7+V2g2CW4GIgOEc6A8bgY4gOP0E69U/6dfTOL23BM/EHCwq09PPrkr28afKOjk9AkVmUWwUIBY5aQVlTTZCkf3j/9Mz1EFr4UAJY9w9QPlkiFlw3NTNPjfJ65M7babeTsOlQ0EghARXkqYAM5eN7MLdaaTQFp+h/1O8ikbwQpLaRy9T57/MucJPdsnSj2s/ZOMwX7j0sorNI7cPXZQtJp/669vJjdAq/ucdDjUaWK2ik9ejevU2hX0eLLgCPwczWmixb7chdrSRgb/HD4+cr2Ejuyxa18VuaYA1564Pzh1x0XVX+817aMSNt+bXs5fQp8o7X1R4Pv/ysqBawYk7FMoNlo2tWftYbPwCkeGllowlRkDmf33pi2xFqSeH2qId5LV5Yd7OqS1ruaEk8E/c4VQ+m3ZaloUn/r166+u38DTnu+34QEQDalqrD5Ri+AbuzVe7mEjtj5XLaFWgn1bSabGP3j/6bBfb8+cNbSc50PHRUKyzbqJWx71znB2oOyyQA4/Ab67mDrRYCg8oGuxJ4KwuyTnSC5JKu1KqthdFKOSIUSwTITEVmYDaP61RRhF6tujLowUp2CAoVuM4923Vt1s43jFQt2XL/Pn/JmlJFLMsxBTotd8oJNIO5VrXMPz69/SEDjWQW9KJm5XT9E5RamQIkupgfkbs3D9dCsbfXRPExf617BXtz8IEwgR99c/3xmy9vzGF3MSa3qUbSkHkBSAZATrbNkOUjqoEgJj8OoaPPPL1VmN1A8R9zA29dX0QXek6pa+nvhpPJSgE+b0Fv6Tb5t8gQ6RpQRyzoiAhrP2kDtvutvZksVg1VESoYBNzdVsoUbAMrZZcSj//5aCOuT5dsWA5Gna2bBvvE2B0GEafbru2/Pel8Yt3uw0KsdkVFTwqwlCHjP1T0Y5SFXMnzPVrwMllg+cG8k4v+Wf8rSlGA7LPhLWoNkZxThcFRa5MoKmVchS/IyNNO7xdedCSsGoIC2pLeUEpk46de5tXDc5cnWQNud8gWSC+OHJDPf2np3KLZK0pUP/IjiAAepl9HR7BToBZ1GtBU0PnAsD55/TU9fOt6nWHVjJYfSv1Bl3rxYPSd/J0eaFmheb8686qOXwi0rIf6Wlb36tIV7Wp81Qa/D42KCPkrrdNxMzuE3cscEWbHu9W1U/zMUXWGRUZgVNSCPke3Z1j2jsYttkYwfuzFtAkLfbEE/5xuYmeCF7Y8GtKHpzvVgyz0jV7n1Rjv634HmFT+ZmU+i+wOgYnohD6untiKE8RgwfPM3Rysr7MEAoWmPUlouQpKxoWam/0HWPPwUrO6k4ski2u2MgQ53BP/3nq4qtdO04x/2NKQc58fWelcigEuF0LgmXPzwcm8Hq9W+k4HluLF8wP0v0Bzsm4iS2fVwotK51J118StnXUbmHl5GrRFjcEP4G4Ps+4vZx6vzml1mbZk71gPGDcaSvhjNGm9FJBQrUnFKnpYMfMLdlvjJbmBXfb68eZc7O2Wpvz+7nPTV0pfX80FTshOT83H5/95TV8jZM7czg7VNVRiCBmMYT6+MezCVuAJWNA4L//ZrUVXAetTvjSjRmMFZQPFKZ39iCTj3K8GrMYfhwEceKmILNuxf/C3VlAhWSRPlHwjhIpXli1UOtxaW1906P89a1NxDqUYQ8Lsk12c4f3MlCv+waN9Zbb81RRlm7im3ACdZu/TNXj0kPojXYA75G5p+6BeuLmPwSLr88OTugvssrMcvZCHxB1/pKz2R+zxaaybGzk//tdomHXt+rdwmvv+P3+Vnb1aSdrYL+z5oWuP2cCrtlpNb5pk/OSp2IwHfFyHVlGxrR22aoAHWxF6lohgnXfrCasPu05K8298IKL4OfWIuwWy1KonQk88ICuoWVfaZnJA6P7y/gEbDJEYn/pd2VP08u8k25kNFpjlU0zgioUOC71dZbKPBpfrTyQX7ZGjebE+/bGvP4XM+WsjYrHI18k2zviWkPZtVcxU0y/nlUbPpH18gl3tVMWJxvBq0CmnFPca2oAqDTXk4pyP35LebIRcLSOSFVrx/m5SMXiU83mMw7Xy/VK7CmxH+rpz5oZxoqg+zmOwInwspBirGPdTbWjQhhaJccHlj603yWDFRVHFHAzEnCvnkofTFhfkG+W4Aq6zZ/MF3o4srH9xsw1t5AshS/mluN/Qh7rUxEklWfZqJ8Q7+bCtTUCqSjjoRX1VQIEnpkK2n90ojsrAblC5C1ndX/ozFFl5cuf03NiTBcu698dVpprhihgwesqXnsFGYWT89Toa1tyWD+6/WNFKdopwUaJVjAuNYnulA8YO+zsbzKS6Z/TYxcKGPtQ2tHW9ZY289RL5UVD2WyJGTgiVSqvMVLnabLn/ulJU7A+Lx5D06C8rxoafMpmQp9nIEGioV6wmujM4pqaJazHleMM85zLXEoQK8ZhiAEFScJ2J4Yt+9KtxOCNytILL5SekSw+3YyxhQNZmBAhS6xV/iBLiFFTrtPonF0aQ2B3xW5dRH7bxv5eMnPxnrL1uZYUCFrLY57kGNFSMKg0FskZNYmntbOJJifQNv81ehs1NsfcltJ+87MXznFO49MAHkUxTbEBngKgGWtTHxjoGXELGKTLPlQEbD7sS/SbuJm4bUfrZE02bC2unrTC2LMQJ4IhyPnRAvOhVosQQI2qhXG7o6PH7S8kpcyulmMzwtShfrZUwjhDWykH3uZfAtf+dBnIfPXvNjcgrTmv0z+svo/aCfgqv/mflVzLytHi0xcjZttqGhVGR81Atvqlv9UDTSrn9geOKzaEgy1akEoj2tH0cdnTt37yOcr4bQ1o03pF2R9JJLtnZC11h2nimjUXNHFm/RK+iSNkV40sKO9ymLLmz+F7qYB//YM+ASqmvKcS/fV4NqOJfBxO76T+6qSlXdjZMhr76PV/+QNC6jpIAEwqfVUD+uXULDvOGNdkdxLi3upkbVCVWVmp6bSTsJYe0MFssy+RWZEMDAFrxoYHHNXiKL1KHqMA7qo12XOzRXsP8jL3lVdZeQDgFWjwx1elLh4CoxRYiwgduhJ6KCaHCPwChQt3H79W3H3vttEuTn6iRB04Yowk1kMQ/whiufeWca8AuAcFk+APSJSh3ZklOVhc8G1y0dE9YotGEEdiieCaaCuUOLdiu3ViQcnkAe1cLyLGe0Q17Azq9shCiunZMhl/UC6JQYzRZ9PHCkZX5BtC6AFN/bKledOajOgSDBAfMer5s1/xaXE7NWPdt1RtPbm/0N4AC+cKKS/N8VV5VdIyiSYYB0MDOgPbQkRTD+q4fLorRHYHd3uSGzIzat0tcWH1cijVXU37RtTgfxdW5XVMprm02tlNdU03FBgJcPHVvynnn8OjB3ykIzO2XFIr0WBv/itvObahnSRcGHygbyhbIJR7sZpWR6f/2pUk8utNheU19MQdtRKyShBW2d6ffv3iuY2R2DjstVnIIy1PGgB2/AbGgeuVXmbf4uLmCVIEUUebtqfmjYv1ZnHOLuAurNUmH0ulwOVg8TZr+HkWwS1u92gF3XDE86kPszOmmNIKvvW6/9hKbu/au9a2YsUxeTkYYBsXV5Rp6ruqaeQ3e2TgYXkQC55hJVa1lKJGnTAltQBwSapXw9cGWoqUK6KFasr/XCJRDX+GV0web27g7XUjFG/EkxhjTLsVlUTxvgnxKG7BxsFaK0ZJ2VB4+6BTIN6rzvXmjsa5KVt3sGRdQHUN1mhdiNHa4osTzXQLM23TCa9ugvbHDUobBmJ53cCMdjK3QVCt4i7JVT1AFUBR+7X9fZzs/qpg41uGSS6uSyIpo11cyIcaaR6cZTTBFrd/veh1n7PfsUQKWp776Sh4b84QmRfZws794a/OqTGlUUF9Zaa1kb/xxMhRsWD1n01oDeXEx4eH+yGLGQnmU12s67mUObZ7dXNjaO2G4qMfozg7TELeeVpGH44vr2gYIKMe5LpwoRa0RCtslDkPQ3d+UqyzMQP7iU8uaVqcWy4Mz5uCUmXgMCAmMGZCaAhQKZuGRKVnWNaYwstZPg6+9dIxuLhf4hs0L9VUMxH0ucwHqHtb6SEPyb5mbej8T6OhbhTAg+tz8IbhWMpPqkbckOZx56ue/b08glmWyFYJolGUvP5gFmJFP4Mo5tni09XiWFGe/bh55ZVCXQbc4SvddgbCBWhdxpWF/qfNm8AeZmHXuCTTx7rpNOi/pSQv05IV0n02kIse113p7w9dNtC56ajV+/jkI8kjRC5KTRQ9+Yok1k7iOnifDlXmIdWsd55Nt7Mx3L+H2i+7O7IjHvRE7TNjVsJbQkToyJ/IYICLIQpEQN2oKBHkPe1dJ7K6psnPHroNrxxttU8vM+wo/Kjvlq1cumwwYCuevW7BaR9wI0SDnbsA9B2xMnuywP38HwkR5QSjU2uNC3xfmx1bQSe43zbWpGCFFlWJ9Wl0mUsYN2Qm2yfG2qwrhtPeZL5GgC+kY7hpB2w1cNh2Z4nKe6EE7qDdsBKLkABtmeHy7N/rhxyr1byNcDlylzfpVWoOzVG2welVFr0BEF1/6ZCPmOieAIOnev1wZvAEJfKiDTQlG5ETQe8J/xE7Z/Q8Wwm5j6Mp9kMFaDMulhOsibYq6Q14FuscUrcWUrEp6HnJs9A1UDE2BY1i8lqAMKctMHPY6MPezU36SZmp/Md+hMTZIGhjEMl41hW6/9859o4gte/kIyST8UdLcur9YrJcwYUpRTATkWMO69vWv3bP7eABKtXqCLxSgrqjUf8ttDFhkmAc55zngdPU1Ns0lBfObbxvN2iDXJTBgBSacZ2GDqlURYMQMnlbwnCI+k6pyRczJ1RsLL0DDfhCJsRTJwCIWiOVEykg8JtF6wLOsyKSQRXS0ORgSLX34rrwXskqXAg+DypZcp2roVnwGKrjbYchxfHUCk0+GxxsdxBquLXeXTw+5MEfkLX0jeMLr1HfPKy0gyy+5KsyxarygByDim094qFL2ekVU6xgTe64MWhBCFOg7IsZNMuuqWVUHMt+Hcx5cN3lb2ei4t795OABfTtHW+K68mc748oj09RSbfGHV1d2XrN7pF5fjiPDolJawXealsE4aKhOlg4/K2f/syLbOAL+dCD3xK2L7m/juw+5F5zdl2wu4eKLK06LeX6YoXiKDKsB7gftBZexTejoCdt+1Wn3CcqfxwS3GOH7stTfAxaLs5h42K92tzKbzyPk7652FwWnlajqAH2HaWCiryKDdodmr9kqyVF0B0G5kJlxMSldc8tb70BPlCUGqhezvTD3pSywUcjkA/JSFvUfUzQLJlGtYiMkLSfMT3mr2ArLtlIq5g8GneoiZtzv3wkUAArSmL5V1rj0ty1zYorv0P1Q9SzCTvU45PYgw380+euDRr7fwa/pMWgIdQXz3TqfTtlgw0guj4NG5Uklcc2r/W1aoAHpaGDYY8KmVNwtyUqeMMKiJ1wIJykWSHKNjleacvahOPwVTu3Wg6rPh9hqTzi0LOaoVGIzX4j/2FHCzyeIvms4Qv/YWo16pTQnXAP5j1WRx/cuQfXU6gsvXRfvvvhpvTP56berAAiTgVU/pwUPZ27lSHpAQTZOvAJbftlN+r2Rn3q+GRGsy/E2Cs/RYOErfzcdbxlcxu5i327BTETKNKCiF7GsN357iTir3VjEkSQam2fd2Sp85V6Z3sGnhnOHBxy3mzzhLsi3asvVLX58P5d1y0J2GnaTD0BfU9+SkFKCv8i6wJPGniB9b1mlwHTRY8GFLXK99mZ7OvYOt80y/08n6btB2fPfrRXNvpzfm/tnQUvs0sEBaBhZiQF2QyqOm+IRTiAD+Cu3NxsRHT959vkUn3c0FWP/i2X3MIco0nQAY45HShBWyL6y9gT3Y98pqG872626I7u5ZOTP9Q4wLUyP1ltdNv6M05TkY1u4ZFMa4HRQ59pjJIgVy2dRgufJzbSTKvMtBjpKpLhSW6gWVWDYoG2hxwYURo97WUcaUkCbL39HaessAyIVEHEsZE8FbL6qno4zfwIhlI5zBgJheJlhCX3lzC+Avah7pRJeQ/YVu0X+IMb70pMWkmAJj3dnaO197jw+M43hjkykoApog3ZbuBbkFJLawtpDfpMhD20CYqIw00S1kWyBBm/1eYQATQEIzlZPy9/eanv+qCuQ1zsH88FqmQZwlFEbAmFLKA36/1ELO2N3hXVKj4/YaEchNNtd3GPDqj6/nvhfQOWW3w3BfJNasuhTQd+0NoGZUy2TK/IEoIXtI69mAyI6Q2FcEQ6ZMwbGqSmlJBWSqHtimYth32fXb2HsI80rC6Zih3b5APidjVIzErw/ZcraQ7cwcoPVP1mCWoWoFWQ1CW13mn4mz6iHdZ+VEgDUWTpDJiJmBDlY9KED3x41nitnEwizO8rnAPdKtmDkhigzwTb07pXjr9i/di7fZzS0NTFbFLqyO2oeb6Lmh9yoNuPZy0XuMWvvCg+0bwIVHIsvoQsUKaKVx54rkLWPcB/DEFwwy9YnxMmKRqjQBZWnJF2AoqSzHAmjuWGmTsHFg/iczfWvkz37Hv59rsr7uYEWpAlQ/yHbgJfPuGGWeedHt7YNA5XdfiMJW2QzaqDHC/+/og/QjgA8HadYEv++vvpjJTlXPI5tnJ+v/ONuTdBQqgeNwJnF4o4cIHI956Av5qoQ+BO2oLaP41NE/bpMt1uLrwYz6m0iUOW53MzNprBzH8c9ufOUzszVZTCzdnchjJmnFp6BAKQkOekrN1ym4Uv6X1wfYpcHmaNQA1++JgFbqOi04je+ULmVsbW6NpoVsAG2bWLsICRCKndqPUNzUSZIWadaiA14yP+q6G9fQdfLsKaM9u7miWsyhJxHuSWXNKHk4g77SXmDuONv2AlYJJHtqn8YGw0/kGyZRCw5A05G1AE69l3pIagOBG8dNqNUrcgVVjsYb+wPwjT8pYY/ZsGIqY8/5pk5n/KLQk0WpNgMYVGSKVeqU2n/6RR07E7WGcvDgnrdPo8/9QTMzG69TmD48omvIuMmQ3fKnfqGRmx/5bSxZI+s9xP2tJ+b8W+zoKIHVmpeOMfeEmLsf/ROElQyy9GP6HfUR9vKu67ILa8f3O6jsWUw5qNcTojax25gywLBau5FDUiBaBUiS30X5seUhC+uq8ZqJOrqG+iQBlhGEDvy3XSTp4Z3EBTOrKY6AMHYWauCplufK8iF96EKKGIyrNyKR0hP2Akh0mfrPPeZKfx5WWVeXURYoi1z6PwHs1AclYFoOLr7qzuF/AJcyS8BSrN1aPnk57bYIG/2ILVzFLNQhXmeRyXHXstrWhRk3Tjrf9kXmRuB3HeA4P/LER/5hLkg73U5/pl/r+TEAIEdUwufgBmfpl8ja8rc7Q2uNSJu2wjrRpi83n3CiXouGZ6i9yrthJzgX+tZ3K/tK9INfwRVJbjuRQJYyCYgH2Yw0m1hsoAshASTOBxcjGH31B+2Z8jZ+hAHvCrC/+R+WUjsq6p3ffiA8QuKuUstWc3Fu7uLMyNCbm/890m8pc+i7u5cufXKd9YLcmqdXv7KGXyvZ/dQ3jrT/Zjr4Jzw69SoFqPw2wrVffRBXBF3Cgr3Sa0wIsEek9Q3GoEY4YsNBT3Kh4afERGeQ+OMXluk9oATrXvUY8IXVhIuY0teUq1rNKbCOztWuCiE8s2mkbpVGrkm+8in/3jP/aFlbOLegsSzubGobjfQwkXqg5Te5j0mXF6eD31gEv/ff/QHnoa9JrZsmV6wVH9DOY0ZuK5jST5e6NC/VgMuz0g2g+CAhy7GODuRfJl17olFbY4ADoUcDAI78QNiadb1+Ybn/dvhZ6TXI36M/BP+h73zjOUsRW+rBZz1M5SA4+t8GpfLXachaVDANamqE3Cq04Ti+pbDL2vLPibaCe8OY9z0AKlJ9lkYjs1wLy/+luOEYnGw+fB0Rs4Oi096XP7munwmq7juDtc7fAuPVB5HbPrVzsnN/LXDAfav3umNGgXKoWZqhb03WBsLtG99dOdwkglMtti2h9oEpi8qbFtxuOP951DmHXQxnYp066gKEjwAa7gIu5NzCHiiZrd4YuucELC2jwq5f3r4wBD57CzBD/jZugcXc3q37XUffpR9FK7kXvnff70LB3eMs6rIfCOnBD43LU8oXxUX0LKDXR7NyXmJ5RhH4jGqLG5gokzZh6o6U9Pv8cM9oFV47MIlvYnXkJ5TyAnSfNwhNsEKAwItYhEHUdhzECR+xgIkmDFQEnoo5jQGSpTg/QQY47++TnNigtmqHKhDUJzmjbBFDejIEb5nB8uIk6Bw1THM0mLHmxyv+e5okeKPcgq7JanoWXg8/KNaxL8ZutBkPGwkilCio0fugFGIyak96Li0W60jVQeHoJ3ckD61eVLHmYg8qLiYvT0mWRzWuiO6Tu/Lzpz9rfPA183vbuTB7pvH6Bqa13jr/s9jIfJYJYbpkTuGK7IROd+USN9A2/WFpX3wsqJWffTJc8F8dSzG+ZWrjVhzE7JuIpUIfTvcZl9rKtQ6UpOsD9XIRUnFavsZ+spNp5ytCTQAa3nBSb2gS+cIW0MDjrbflvdsgbXhppa+aDzvB0XS0WAHI5RUg0PYuN7A+RKduLm29hVFdgas/OTe/2kZUogNW4oHYf1AzYSo01kM0m5jcvQPJUSdvubr0qZFwfjGKjyn7ZbtHg9aIMUIcB1OFsm8ql20M9PGsDbpAwq3JvvN9UwP0rOWmjOzObkhV8LDp+WM7BrmjJmLpsaWPXn0aktGSpZx+QNaPQTvocIJDrf3K3R95A8GWC6zCSfWGRtvsTH9V2nrsRpu3DvY0Jk8mxM2Xsr56M0QAQn7Zddp5/gLMs52gey93Hc0WUzdEr3z+IHdDQ+S83s4nn8BDp/7TfGV7NWLyNLgXu9AxdxHVI0zM4ZoBIBa5RYOTaEw/lz3NfyXu7f0KKXE2EUjov7+1ENQqD64cWffaZeDAc30P9loq5Q76DULaROD0BERlfvetUg9euIXIa4fYYIe7MlCI310ODsCLpfsx+8ueiPDvRI+7WHl/Tt9DvLZQtQX/A/pyuo3xS1oaMzWxzb0ivViiRIyV/xtqphUBy0i93WM05O3U5XduN87CGZ8L4RKwRs3uSeT9gZdS/tnCN/kcxHU54P31HRPtqZO/osoIq1Qt/ZSos4adOfxZB5hgFPg8tkEoqA2fRmHkaagicYTrZyAxsQ7ht3I8KfV+RD1UjsGUjH/wcJ+cwpd3OUXyXC0HYKpWZYwFmT2i6f3CrideqREqBuS+W6UEru39QWUY3ZUBb8pZXu4ZvMsB38C78z6c7m+nO/M6o6HH/UzIBOc3LnTPFlGi3C192MnOuIafGnTeybfg6aGvTmrQ5lh6K3Y1+93arMZqYKdwpcIwbfH6zj7gDE1/RH5kJ9ARiM2/KQnVTsG8l23AjHMSBUbpSPpwBbNbdTxL7gjfpQ4Jpmqc3U/eY2t7om5rtZX2EAPYyRYz2uLJRzkpfhaWe+iXdZSznv7Gika0SXdSOgSWGpuXbdQChcEQybAlssh+53obzoWbMUa3EL+yd8Dsjb768i4Mzv0MkTtfxf5sXvoSxIL8UNHpPqzJXf6oFcg9f/F6BiY+xvfB2VzmgJfdeMtKhiw0bfUYGdBSLCTUYsjDyoHEz+XsmCshYoOWw8XL9lPxPYEhTVCW2mFAWnlQ6+sk9Wuod1Begtw/OO+kzAsQB8m/z4zCa02m6ciwyR7k8s9B+6FPvVPowC2VbtS6+Jb0dWm6F+wikjmIdsXvu3eBS7aTyCGH9y/xCQ683ITZZJpNS3UKGS6ee6Q2gyrGfydmR2VtNd9ozxV16pmgGxXjOqD20IWtL33YdLiO/8+sJv8kWqmMxGt2CJ7JOTjHM33RGzkxgfuxo/VjaIckLkBaXW41KSoeUhyFD5V/n4NmF4f5PFJbkmEtuYOrbjm7nwddnsMbW3DdP2gh+g4HQWv+HpgnHftj+ao4GwBzCkgG1Ft85Qamsi6ECpR0ahxKZusXlqZG5Nw5kG2yeXLBofm3Tjhba122sViC6ObbsOt4FM48p5aUi++9Hmvlv6UvW1i6KCUdnsgYtOffT81cAdGDdC4iba7nipcj6uNsIwRjlndCROxRmMQ4Tnz36ubP7/8u7HV8TmPjCkh5kyJ7/NwOvgoed2F2xoU+sklWEJE+HewIm3gbFJibgZzVYhsWkr+wrgy+LaoR7FdbHUaOLK8TAvhd33PPK1H+ilzIH7cZ6NCiq0KzzXb2zAPzCJGjXn/6J6tYzRq/9EJw7lkSIcc0h7xlNU+nFQ1vvJm+F8u+bF/Evjt4+uZX2fdI6hiFUIxTZhKXNchAFQpgHs/+FyhGX3KKeo6cxfc6/3hH4nyn1dlYX/IJVuVWVtJ7gBwlfWaDPBLnKzVunpMVN0UCTjF22sP/TnXkyufyK2PmQf8lI4spowOOLTL7n9OKIBfZRCZPaNWNghdLtl3ErJ/Cx4z3GLlyCbC2UNKu9OFG9tEQwNo/Aa2vf1dev4If6JU64WZuTo45q5lEbW4Pt2/vlIAlH5ZCWGO8mCg45R5MmzVv2h9Y8nXlUWlj4utqzFTVKTFHmzWElJgOXQX4z+2+dO4TkNw8U0QEjhYWJDAPP92yb+zsHD00ZcKzYavNShjlS3oXDr6NclqzFIWoVZ1QgRMkFTQ28sVESVnAho+SyQ7TtI+fYfFvK4aoyyj+un7hfNj5LOD7mD6C1I9b56B/ISQmeTmmYYTr/+jm3ZM2S4CptPnd65BL8a/nAn/Nt7QpSzn+o177Rf/nG1yi+fAxBUEx0n7ODH8ouVcaY1PXhKKRxryY02Lxg5ov+q0X4Gb9D22g2uFnH9jtz/75msL6NqbtbYsft0fGC9K37Ck9D2VV1KZJRKQjcnICTFUJDUZCPjt6BaGuw/pff/zRCF9CqAaDEJvKhaUR3opCQ4wXW/f+3pMRmla/+vid3fmb9dfc2v/aaVwnvRjeDGx8mWf6B4V1ZI3c1w/bgEl9pVavVhnyhuZF7UsvDlfGrVPak1PEfXf2vPnpPxN7gYWU1rnmDQ6wxlbWwYJKIEfuWXVq/e/5HyDwmJW9wP+tH4hntuG233wHRs1j7TaQC34vTZkZmVefC82SizCGqYEiQijDMvHgYAE5+R3+piYL5+6Yo49Nfu/+ToWF+2yhEZ37OmXIfePBMPy3SF1rH9jyRPiepHk1meL3Ec2i4Feavifkne9dYjJ6t24vfu2gopt22eVj+/dB+eoQWQgdArJi063ve6KrYk/06naIFo7cOXROADmzXzohWLyiThpxp7I5nSfKM5M0ZPFtedg8AfNm/dR/MFzO/Y54kvLtYUf2n5WVZ/SpONb4UPAAD89H/nmyDUXi0MqjlvNiXNkELf5Pn7qvS88hu6BRSMB9YhKrI0XdAaPlnS8F0+gVHIEB843gF3cO+fWVSqFam8F7J1xVJjxAhpKJbrCLp6aUM7j+yV/7Utkk1lvBO8igVAm5G1dLuzXWMgjYaGyz7/R+/S2AGj26D/Dzq/cTm75/ZWOphFIMXlw6pR+zHJ4+Pi6PpqDsl8uSzONftnnBRCjGT2Y1Q1kclMoggsbA7t7iREkBT9QJh5ru16t2mPTxuU8WVxdRVtb7r/U5L7ZxPU8JyVdybdjspqaBEHHYYiN03KIllk5l/q4fob2LKS2CffxYhFZjHSPxRJXkd9g3itNrJ8HjjGY5LtOP7N7tyFLRRXL6P+v5IF//xb4yqGM4TI66a2SdZWu/9NL0fJSCTiK8ciUzLSKsL3gb+6gPiH1AvX7jvTBzzLW5gWS46g9ADjA3OQ7+WBO+Z+oICNqbEkTGvP/rFch93we+h++d5sf+PNYaCGgTf3pbENX4RBi64RW3ILZqocwrrpFBUVtXhuyS4+2HtAh5seZ1vohDQqfbZ0XS1kSOLptnoBCQElGWFwwmQnpCx/UYkrHgR0AfrpYhFoklSkUfQYdxDzR8UmES3a7efM2lBKuH//3+SYo5GogQugatafm5Q1zE3RxufBhDhBAqyxdDpvUeeJcsu8ZPAhbBzxAH/cipEp/+i2J/Kpd5p6BuRJvPofhKkz15mXvezOXV00YTLy8z/NCFYwR6NKJ7LhfB7G5zuP9CLWigQDzYVvuA3KchfxYAaPPfyB3eQGLxw5qbglKG2wwC1L4tt2/5EJ1lxKNThZL1kWUw+3TtvYeIFP6l0L1BtphWpyS9vCBkEPAVr8I/lGvL8K8xlDqmD28DVBTLlsX1LSkPqHxrBkaxLkFi0deAZrbmOXuekCrQy+/Dkzp6aY+zzTWsHYGNhPPrDaYtX8vbzgl7vK+7/MoBDzpdH6yM4FLMHfKovV/D+Ai8C2clkpyFIBz14/dvyLP7qJ6gxcCpD+JHFlSFRJuTZU1BWioXttfgrOqfvzXKjgf8WgDvfrxaYDM4F8u1hX++BSrnGxHkUpM64q3LiOl8hRQTwIh1LKKSZ7LegEQTnVBevJZyEjD1HBArfDmJ66EsNld4QQDVdZOBr4brhw+WzMfdhupyQJVgUqNOVskmsCm1hcGMT5+5Ozgh1VMqbgxcDCOpxM40mswGps4ShyUr56Ttqyc9ULK8C/barhgqwg4UqAwcDu3vLdNrnOLnD0jY2e/bN1CZqRHex6VRK6Nh9IipOOAevpXKRSY8rMYBm7KTHTbBKN6lyhAB7eqfaLa4Balimb/zjAiOi1EhB21/WijZ8z87ysXvSlKcycrelsmUxXOkAjvaOofLCFMHesCAFzL7d/lIzC/W0YaykcpjMEtejBrQ7w3sZfT6bfHvsNfQrzkt8zBC63gVtQpRltCYPWj2d4CZprqJpE4Nt8l0+Q9Gls/dYepgsyvt1bfYT9GBMPiV57IzwJFnG+tUj00NlCEYqThF7nLw4SMwoKhptgV0WaPkwiKmkq4dTn21efBwaiYd5Ob/Y8NENiGrXF4iZbq1HXWvPQ06J0yrWwSgO635rF9dlTL1sgcWtnuKV4Av54O5x5d9eOd8ZHdc3NsS0QTn6hj78/9dWnLn8PEXKs+S3WwWklsbDZ5t+wODqu8Mk6c004U3C+dnYWnn8Pnm9+0gcD0d+WWrSjpIn3r1xVY2/kJ87vcLXMQe97/xCA/stWZu9lAWeSI28qnbuMNPG6y8WjyPLwCxmcoHSA8jyHHmPMRHO15iWZQlx0pqedmsjYutHBaI8vxEt7V10eu9wRM879fzm087XouoBbPY//ICZbfv60JcNAFjjLlvz6kBuKvrpBoRlv1o+uG32YQRwu4cx4rCU3L9QYQffMkiF+6OnBzkjfNiA6538FHbBpr/5oCDjY2ecRnVMUJ2hRoVPneMF1Sgljn6RagE89hUCfF0WPrwO3Pe/fRb16nyaK7zWVv56MFar8NOIuRouMyDyDTaGHjbcGgAjL0dTXd5RtSggdK7l0+x5KNyVwi/jHUurChydUV5Ckq/fLNJaBqea07SxXBAf6Gn8+/JMjSM5EXGtUY74oWvRJ2o9fedStGoTuT7jPMc/ttbp1qrO7BgOBp0t9z4Dag+VDy0vZ+MkhHC1bLqnbuMw8mcqbjVECv5lJjdn3b5qOfHZlj2ORdMGNHuwaiWYwmBB4GvwmjU0V71RhCO9y1C81De0gAKGZGPO+/CQB9P/l5bLfmFDQkVLk4TUlpFD3U4CdJXRB5f8UoSURWqTNMIKLwoCrwx2meTqz22h3PyXMpIc4hfgMaZ0nZmQiXGILW2pF/fo4mbWdrz1Xq0aNLK0Z3zYQIMOqgh2wnfbbs5AYn/KuO/me5v4BhalXgasNL7KwTWgD2peyoy8mvrhAEy9agGZBiZMJTPrQOextLFqnwRlqS6IqEgP8aLcl2BT11P8SJqz3KM+e86MM4J1So63rM1vkAip/ZVgk0W8R0dqbriovdNPXNnVp1ChaAHw4UjWHHkrG7t3Dz7sxqp/PRffqnM1B5ua6MwzrzKxzWkGriOVCMMiANEPpJU9rlH13eir4lBa2+0ZQdiWrCBSpdkF+4+1FqLZ9Zp7/LxyilLwvNA9+lYv9OiI2r21m6Nv5OPa2ozFRvAZqpaBY7FXC+PDm4085UiSAiiLLLx2JwiQy6wojWql1AkgmSIRSoD+X5tinJyIsWrVvY0GaULObDH/unIglJPXbpLAEk1BQx1kMn7DaTBgRel1cgKWuLaZz1kQsTK+ZVaiLRdjfNlpvFfdVKtNpYT1+UzdeAaI9XAFueh6OgL9k906t0OAylLwmwf8NH+tdImcI2sjVNOruXTigOw0OBbfhWw9sNNuea75lKcglbisSncOqVnH0MyOAHRzwvpvI0cBnykaOm8rC7+LradeGy72/epXj3tzbG04YguzI8HoYDAQD3gfFCE+tOtxcdMYAzsiPQbSxDe7pbkpizTlyG7WOMue6pPDXiB2ISsYDaWtQXnX3mq2SShXdKqO0sR2NTq2h7VWKl4i4wRGFCSrAogXVVrkOKr5V4Ne6sKzy4RvVy5VeU4g9LACsS4jMl2A/bgVSll7dDUamOAaFrZ/BEWhId+bPUQTpeoGaDm4GDHW0+Prh27h6Dlu/JeCTKMInMoJlx0DF2eWdSmrJI8fIPBlhPdbgStbVhBiNdtmC8rfar4FZqJhRor5ZswxHjaV9ig+cMLGuwWY19DNagqZSumoL8Kk+z5lgljEd8uTbdP/4nRuJpjagb58wc5RlGY+6BgOmxLrAwS+nA3YurQEbjeDXjpnS+zQ46TLIvl9usBTKecXn+pF4VI5FaoBwNk91nkGl9EDATj5xeM/8eBZh6Z/BI3lfjTBf1PU+LH/9iYrrbfmHlrL+nNW3UNu4BR8KmKfTj+/bujvOp/h5zgGb0m1riscpAxux16t19+Q0wxSct37a4Rxo2GGPEPjprLW5njc2bDfVC/Kb/0lJzQsc42X5giz8bWNm63tfuyNt8SLABJKAdtgpQqNbRnXzL7+Wo1XZfaSyAetmpTlRbuiPjwq5RjLdcsnnuKJ0hFL4OQeiS40MiqxESFJzePuiedrWSzu90cFPuD8fI6NhHGdF8aSJ8BFOsJ01UxTaJFIGOiYxUP1g43DlaAGgj5Bch/851LQl93sLDyYP2de93E/agJxpSBybGCpEl1JgPMIZsQZKiW35GqckWpKpWsoNQErqNkeBBmpnhRqSMGHGVtPpFqXIajFFxm/v2hKacSs6Hr1zMwv0+PqWJ7KBzN+WvzYyC5K15MnG2QeHzWidkBRYe7Oul1JGd0z3DB6uOz97mXfPeTYewpf7eziFRlc387vNOasSot2oLJoqwEqDpd61xrAhgt9GzoD8fxPGrFx1DwUNtDa/ac1yKVVzgw75wAKEHH4SclVzr9za65hxFhBrcFNtJvTgyQAULeCRKUKYpGcKTHqmNZQAYH0q99dhtOgdqpbyRlKF/s33UhlKey4j/fX0EP5u6uhLBeDmId0Z9zkdIHgB3L3b3BmPu7PdYL6sjpF0i0/VBPuqYDSRUMyixR7zES1iVXdRXieBKX0azforhLXpg/HejgcJECwaBtNSB7AB1Bqidr7SLAYcQYfCqEJ3KKkbSMrsE2Qu8OMO5kYX24EUq/7Z3YAW52EKBouryx4iwfXVq91gcTQl1W2JH9DqFx8SshIgF70C5zFRI6m8bQB9onM2NQlbepVAEEljjjamJj7rlrFHIj9c0uX1IpoI7cq9kwy3ALUeLDgildRTd5l1M5c23UkN48yJTOee9tYK06UPc7hzgdF3ZxCAnyafNO/iRuVbD+Faikd3VSYQFbBsgvKO77jhJdvZFIz6Dqc+cN2S5tbHXxLrq4WJl1el4nx6IUQgxLpUgArCqF9EkCO6Wdear/QHWEtKhePLJB8rVrqaquqX5ZaCFL06g0tZ2mfnS5MXXqN+okePFJpHyIv1RXYbWWmkM5y7kL+EuE07dmdC3VSDMmXU1cYsB1PAnGrPSlzKofe7KWz8mbg4n612uA7LCEp36WJtWns+kKYNVh3civQJLV022b/fiTRGFnt9pvrwiKo2toFBRmY0yQmvi2YYFFnr1KM14OLQ3dxYDho9geurtF0ubRlp7pHo4unQLkCw0Uiekw8wH9cnm74rOm2orSwQ7Xmz5aC253u7nejOocVmAYjs8Xwu655wF0sxeCoKJJqnrwFNweZ/9nxnnlfrO7NWPx2LIN176ajOooK4vkJdetVlmvoiJ4ZeHWVHe2oxiUX+34hbyiRmSZmdkDR9uT44dXqkDb+TqBsvqiYRLA8Een7vkY7kWxlsSZx224572LFzrcbwimLpOW+kqDLfOP9rJ2S+l/HFkQTVlhwnL9dXIJWgwpkYyzf+KX00fP/xLoGXFaCgsVipVIrYyrTpKUqXd6dSa0R8ltQq6imr3Bj82TJ7AZyIUh/347kBJ9vTE4TLUB0VInqftCIlPBlUhVu6laFEsLm8om4DQ50Jns8ZON9cHvXDs10mqtdzPmQNDMBuHgnMkLsZwZ5+Z+KnIVJXOso4FXaF7HSc4BHF2WKiilrIDeHbi+gWN7Xzm09slaxpZqjVXIYrVYRP6oS3GCeHnlyfhKUIc+xW1iO1PUJGDdActLkJGlAxKrfiaRb+l+oqqRrmLQbQd5IGhC+vB/gb7+T0zOBBfxn7V9PsKJOVYB5HsNfBCDCVMuzdCpyLCtOO3LgThZ2kwnlUfCApeURJlkuqknlODMeuH7uahBpPekYDr6Gu9f14rSxIPbeH8FCXGjJ6Hxwu4fxNVyJ9+3fF4zRG0O2ClalvpC4Yq+kp7ub/ZMZU/PzuQxWwiDpuAmDTps3Nc81OTpCvEAuSS2O6LWLrj0+6CECpz1wkOCVMPGreB2daPSr22qKwsO3CDbKz/9kErBi3YFKrmNsblubFN7zIGxu5+sx+PmHt4mt4w/Tu7nd4rR6NQ27HVMSNdsSQXj2Xm6yzS9X7i9DpsbXfUS+kj37pFBzmZtfnv5NgeYYxNxDKsT1BojoGp1Cqld0hS/Udq6+eILXhBBfteo00dEH6hUlVagP+o5RWSx8AGEw/3dVRTNc5POJv6b80cLUKF8EJCEQFJ+AYxzuyhPwGThl9avHNkKs+zqnyEDBOiNMGjU0LWDcHyq+XTB/WCH3NrgOyyPDIxWZ7ZYZ3ZmGxelM/L27FYT1CD2uOu51FMylx1iVScGcfFb3EEt0v4KT4FCMS4JAEBWVHCJAxxq71VR1DNVOvTlB1kolyVhDr1rXPLjhWmOf7+ft2uRKSE1qZRmeMYbO/Mb4RL21dOpe6rWV2d7gjFxJFDzjo7iwVYytgFU0kDpUFOTY3JOJv6HENPNY5/H71oHUIVzB5Qx/LQFKuHgrbz16g86I7XdjRSkm186qEClopMW0AZLYamBD5oZB3nHvcilFs78R2dH5p9trxJ7j6tflELcsFAEYtDvySjRc+kQg+Fw7K+g9fGlywdT3ouvNC3sKoinNfTHxZnj8QPsZ7PzWUlUJPZEe7uYjSgjVx9uO81u1LKBUAxpNFRQJzguCz4SpQ5e5hX4MoTz4yVgY8fZWA48QNS/Teamjqu4QbEPIemoYduuXrJ8T4z5jC1EZpzQXNj3xFOY/fPB1hvkxd67qwNEIwuqCg/VS5UM6otF/8Bt8wfSQiuLqOyfsrhkCUPPrgQesx7Bv85TopRjHWch25aa+6pa6gybgDhZjsbfT9ahRx3cbETeC9LX6AfUezCTTDBmebG22MuOxZkRz5q1Pphoa0F3xx2GE36v8JdTzsxXBZmriFz0zWh8cKdP29IdnD9WUDw8rtRqb3ZJTDrzetlcvKA5aVUu0TB1z60VKen0TB3/0OcIf9flj3fWGP/sx0cg0VrF4ABfjzc+iU+eCq5/X/U3uBVx5etTx211gFLA9nWOA22udo75ziMUfHTKOxkhZzbdvb/DUs14Z5blldgax3UO5Vesn1LQhJNW90D6N+KJYEM6JX2beuDrcXFMGgMaDKmUyuCdAR2oQ223daqdkqvb7kVeyG4mXf0nxTU5o+QBwwg1DmSsEOUBOVQ4GWyA6uReN2pe+PTZ9ojluqpaYyekdJmlpya/Ruz6t6+7c1Yx8JxWM3nAH8L1F6B2gQoI9mxs843K6xwlZr4+c9KgQ+QCjlH4y23wbbjB/P29BfjYEWtjgpyx5nCL+afK6BTbLzECLXTG8nX7Q7pnvQ+mLzoTB3jjcpoVr/GrMwJe7NhbQ3ZAIwmFd1LaUJv6sH0KdugVw71Z05frSdf4vFCVW0obMDwv1Bj0guyAKObXh12QmRr2kvb8C6t123IcQjmWkH0+hBUgq0qyft0BtSSGQxE+rd9VDmeKVLfeIIA1LcV9YRYVAhD3fa8LrmHs8IQiZY475Y/kd2C11PlGDMDm0+KpwMX7qVhRShHt/EiozS3cnxEz6oLyBL9NJunb+JNStTyih7ftT/6obxyQUXyq2UW5fwfYSxeQR8Twqc8D4gw0SIOcUBuS8bGB3hf9Dz2ioHnfdPQtdWS5lvP9kz/TmB0h4HybjbCnmuy0jKUn9tkp8vUJ1RbIwLyGYk2v8RwvwEGxIBM4CTNlBS+mDvKSVK8zfRvichXdGbjH8lNxCNmEN/GakcpjlZ5XmzJGn9AW/daJm8e8B7UkVU2UrwRtZ3jydrP0phUREyahzvJcw4swSTkv6CzhXdbkAAnb14iWEgIvwLrQmz9wAwlZqSuMHBAkIsLxmCzGKNNjYL7dwdrVgQHihberG7A+YyYFPPZG31G2lLEykHvynZunQUHvHsxO5lUTUiwlNsGZN2VnAxVgWagaVa5S42E/P0OqQjjMPI/wRlaUmNPE86nAi23j0GyV3d1Dsysd6uQ6pWWWPmFBu8GY0nNU86Dqo7rFE10PU7Vv/bP+f9eDESxA9OE/1ZS3kapRS/jouP241I963zy896dviI4VXVeFNzFI6lGYMmdqdI7FY+X5u+nPSOBFc1kxsaSre1873GdWjY+oKAtsdczwzql3niY5pveEXXld2piLCSzwIifUrq4+I+jaSF1lFXvLv0Ce68jlv56T823txp+bcOk4ykG/2hLkA9t1UaS9hs9BCh/V/wF92FVZoU9r+i0AT+3hPeiDoaxg8ZTT2Gs3RSwYiCkWqwXuZnZtAL4U4Ua4c1mZzho39TXZ0fY7VlzVWc4wHHi70bptfSHDMOSX7fMfh+qMlVqDHxu6Z7nci5AIwTkQTVpBXxBz8GnCqnxYbC7UjnzXlx4XkJ2xZQf//1/rK93mz7Wtgv/rT+8l8f+vPxub73/hxxXhoEjL2+C2sCuTZTnG6Hup0MlY5UGeHHUOF3pRpyCjD7Na0MaH9wiep69tr39fR85VxYkRr1t6/er33ggh9Kot93zc133RV33T9zziD/7uf0UkmxQVXZKlUGqlW2bKtbJQ1gpen6jTNWyV/Qt9si+O/8wkmAfHZ4GztsXJyWArZ2JmZ2N25+dQ8FurN3u8GfOm983uNye4037v/P2DBv9usLaB7/88a/CyAc3/wXnGeZJzT8qplFHnb/Rv6KfoW+vH6XfqSXwf/qH+pf4X7n+w4RDMwHZoBg2MgQXi8C+UQN0fowDjBkaxjB2cxm1YgS0oQj2OohS16EAfRjCDxfgCq/AdfoU/DVHZcGfkY5zsGoDMWUUPlnTapneCi81A69x5ay7SGgML8tOLzXaZWuXbAlFZibqXPgfmne2ALIPOhJYalhedfcXg6M98cO+SnxmfTwVqbkRpWmPSrWVqwMnHRlMFQKsTodidS1O12WsE4+gJfm2SYPRVMSDSnCXpgLado3Zva3ftex4Hbv0kor34hxe3eEcXsf6zWxtQqMI1ftgcjjUWTw1jHRFLTmOkT7Qtcofa1/Z200k6bO/SbfXI/cPMbnnbJNfD4j1Mm72i4RZgwUlKy4YUWM19u3cL7PcP9pdRbKA6eNQdbhxu3pYSeV10apukMNppg0bgxVsV2NnuyPPoTv/52XCWLNzeOqEBPWMgX705rZrrQMkS3cVPVzcKmaMXn/cDqVB1Ge4z2SNqCDSqQsvQ04TvFpEr3/mQM/XgaHWQ7zK+y0eruGws7UrEz8y3vUMFMdNMvIbcFoO07T3fcCtOzQK4zVDogfogyUojN2ad8SjL6vRqeH8MKVU6NgBTZf0rPag9MGeQdaSxe8g0LM0XrtoyFyIE0c+0Lnp7WJun27gfx0cA8VHT3yDwL6fQ8JOLPMBWNIVMFWlbmSx7pfN55bCuCTY9DMci4Mz3aAGkzx7Lx/D81RHxq/ete9uL7u+6wYWHndPcsN5m+z9+8mjzrbo782C6yj7CnpFhApRkN4dNUsIgzl5oFupGyGzRiNhg/GXagTqjysm8whk14/2XH7rGToH15sgksypbPo9/YnBMCizui1CpjL0Corn8NGSVlVrrho1rpYsy+W/mFt0p3pPPGx7OYhHrHOAZXEf8GyzYBb29eS7BJP3IZqkngM3lT81jc0AwN20DvqRPS8RwxH10WrpR+VYbDVnNl6Q8zRMgpmXHZQjc+unnxvU1Pmv0e04waf/LneTu+cnfbiR3x2TWsTyKuEXDwaDuwEZG7L933+dEXnpTo8K8KEn0bMhgSYugnpJF26g2r36ZTyt/rQ2JVU35JUBMd7qON/94onSudlKDliIOw3375YCL5fXCcNggA9louHE9EFNF5T2JVo2bpqW3u9h90gYGxtpP0XIzDutcifbePmM6frVh2b7bdXbSTCITx6vNeHQhZnGsZMZFLWd75x7admbVarpC1VutNM1QGnuBriQ313Hq7deP3QMyyQijw28/czpUztdiIZt3aI6vXeBHBhuXdVaWn6A3KC0TX7eaSaCnDjCz6oAmWvWba/nypIh7dFQgBq/4IlBK1s5q5rin0mJDjWzPjwI10QOzT9Sgu/16WbD3uZPp45X10qRwSLheTHwMFDWMqIaNeqwYxSHVETvb764UtgqANguDHSYNl565XExYNIO7UToEm9pgkZFEJZcvrFZLVEuub9JuUFFKpH5EPij66KNkfHVfgzStiDU0zOqX92XqUEGreuWHj0ktW+9juoyDelnhXr5AVIk6yymMTQDalxihAlSRyHWG8Dk1cvwoZ5E0ooH9pJqb1b8+M8Ep82z6JqfAs1e0VyZVhDh/YKf2tf0Z+5FjGx7u7N7uP+T7wPHGiWHI0Ws71YrhRQ4Asko4W4bM9Tyq48s4uw8GmUdUQbgF2HvKepTFeM2AbNFASNqiBRWDaCpSP/fQk3HuTdv5A7P4WndXs+PzkpxpaFUCGm/UVRnAkqrCkiGvCcnc2s6otp09Odr9tB4qmeF4as9DRou37VZdvk4mMpgHxGClHSST491Xzws91oa4beFijVFssb3UUlia0I2CBng0F+swGNq7Z5v3KUrlyHF7miTZWaZ6NqohtCt5iAytagGLi1PaCPwic7R9lrkJS2KiJtOQdYjzoNTykV6HHuhwkyzrRhfUMs90QgyXxZ2Y3k65ySfh4dYYTsuhn0RPzdZUT66xzIPqHxKrxZitc409VkxsbFpkdhouG5K6DotnhKYp06zAScq07/Z4KNPnWCAHPFUptiRb8m9xQS20r1qzrrMos16QNt+Ru/hdvpQomG684wxZ/0JlvbwSUdAtykq5vFDZzi8PWZnReCfUa3rVaNVnBFrZADPxjODJnpKEbTfc+Z0bWAFCqSjvIP/EO6UJbOaw6qKLzvRGAuPffMAxTVRsi7K0mis1WxXwxlymqnIBak2sjw9ggdoXq1cUmrNF+mQ/NRdYaIsnxwFiW9g6DU4D6u6St4osJm7NnamP66nYx4v4hbh9dhT3x7uTdeHnPB8TJ+WS/4/n085y/78RaReNMO/h0lyjXfSs+o8fM8OdqeS0QEi/Aa7/37dOiI4q6eMeQerrobOvvG56vHeLquSz3lpzLW13ZneCQHTqvtaLgnYw6LzfnM9UMfXE2iktPg+gMSExC0sUWwxINt4CfHfnxv3SAF+1L8S0kuE0pbTw5YDNnrbw0JoJRroRdf3S2unoUG3IR4Wv9eaDc6GaGKXIsqTUlGnOyQmktZFv2avsZfUyeRmIzEtyzI73HR1XLwCXtRynmJssksc1z2vlC2Ky04XadX86+8lG8ODZU4mJ+px/z4tCkVNB96UuGAa28wZbPtbWzQwM1peOMREfqAPhfnVlpcIeomPjUr87H3SISI1rMY3PeJMsueViVX3Pl4YS6NC+YSLyDj5pSTacLPf7KFY1A9IERbSHLTpGaAdRH6k7QjYUdRP9IPubjZGZrUdFqpaqFdPIMZ1WzGQxTwqQb8dtctW9LoqUksyr8Wa6kdnPHwvDoUmi9ePqwZAlGpwu8HoDTcOwkJidwUCIt81W+qAR3jhujLsPK1LzS2TL2vIxZ3HLBHnOln08spMobW09WE4200nL/b/jjxHSfwPwMSGqqre3M6JBNIyTce5uRWUl5bonlMtls9J86cz+azeu3t6aYMDQYN8ThkJD6SsBuJqWu4pJFaJxUcXONQxMy2bZvplmlxgpX1vBl0uraVq4lCSrC62yQYF5a51vG0TQuvLJyXkbnH/XH806ezGO5AiXsC1k7Hf6YApyuapRlUEc7Bd0pVXIWBU1u/sktXlWQfMzRq0GHMh6qrdixt8b/yli3XP7M/Df9dofo2myXGMiDRSmauLwnBHitBUZoXN2dMBR3/cdXkNN2bxZ0J6NrX3inrnZmJqKbYA0AZ91FRciqAaNz6RQ2tIk5BlZSiVNAhR8zmb2uKbOpi1rWP+itAfYWh4DwhPKvWVTUNaXQx+s+q2oysjS+ZKiAxrPhyYMR2zLw4cVhHdgnErbXvPQ04uyP8kLZ1tRP/kxhw4anFn9ljb09Ky51J8XN6Xxjo3/uXHHLM7DZmDMUzko9xyY/e8S/Th+GtEIfCU8o7QW5T8+2OBfnKA3/pGsivlOJyTXrEv8TLBgdaRL0oBh7w0hgovHTsei8tAkMa6namxlQ63vgzw4c3LQ7aPWgalnnMc/cZMdU0V/s0dG7Bx+NRkFzuhab8zIe19oERWwW/lUjRRtKNteXtaT3Qx1u1gCi5gebRIs6xODCp4u02swYPRArVYLfDl9ZzMNTtFMrTBrppoEtuwomeVFF5EXHZqKCnN77/iNRc9RUdulm7n6iaAxeSFJKBil5wKb2jTjab1ai8VKJcNoBAqq5INWr3KWKdJFJzV58mcfSHNLfSZ2QDxlCzZUvaqpI2e4McpOzHv+OEvoxqgRt+0ioF1M1mZ0s7aBd6T+wpYU+UMhph6tDbB7vb+wi6cg2fP8fhk2t4T2PFpZU685M4A5uEBkzO1/La/Sk2d2FlTvKMX4Qne4/mi9eSEEAl7ZqcCIVhYLaAuvjAlXAVATHzQRLVFSb9LcX496ZlBsyIfFdnllCbSrMRODZdLBfgHpXee2IhMQVcSBfYv7wYE0XLi917s9i6vvWzi1EhChg6cG0dvZ6sZ7Et+Q/MRaLgGU/TIm5xZSHb7b1DkKcpeiTdp/nx9E9Onodx4ZAntsm9d7KI38JjHKRNbBY52tXdaCRsGFdCSU2wzlazqyjcG8m8j5sVu9a2kKI1b/Y0OQOwZeleXthohhlZE64lKKZroHztCQ/mkwwzg1z2HZfp8Q2LjPy9YOrjhfmzqmhbN/3ZhVAPXhw8X8fX8NzOJnWmAuvBchdE57bpzl/fCjt5A9MDZ+nFARnEL1aDHZ5azQ7nARcePk7nATNPV2MGaB26JMxrt+w0Ba1AMXI2l1NYmE1RI2kG1bRD8GnMXnC1AodeR5YktX2g8FZj57/8ZNSyEfLS/nN/TKUE9HjbDaxc2KxbkF1VJW6jYGmp+Lz8NS8qrxospsHfe+UcKPP8cg+/d8ixw+5ilaOELT7ubxp0jkSGGeDpDDwovwPyLTZTcdsWmmVgiOQlGhW4GHDVMwoVGAhOq5ulYDivZRAK0wY9zfdb6m4G1KeqM23emKiHdDw9x7H7leTAAWMjY0CGIDl6HM8rxjjyFkLPJ0h0jvorxKjoVaEu8hmQquuLZN3yJ9iRlfmEyzGK9ZJtCmnWJN8PtdRUmfm04XEfWynpYeUubMZLtWTKWHLvECw9TD6U6yyRdYv7GrHJjT5GdSLbdwB6efZZFlYtpNw5UNidZMCS9SN7oZPyat8e0JPfHbQuRferR+iB9u2l7ba6pppqzwzeHedJsb+gAloo3k+QbNf5lMFy41BT+9EePOAITdab0NBeZePwc09WuKM31vP2ASbuEH0v7uLWZn044FxNPjKPvdfaCERad5I0MpyJwjYFqPKSGacMItIDhU3n7Qqn8XOL/jhK+ECFGa3sssuTrGPGMGXXEQ9Pnian+POTptoKlN7/S9wMKSczUSTJz/Jc+iNgN3UrLGcT+RJdAzTG9y8Bpesy6mzpbm6bPQgodPZIA2Pk0wAHxwMQAIkjo2AKCejh8AwGA7+O0ikH3wHQBM9CZyCpO0fIBvx5GC/yZsy1hJAQA4V/8oAD29XwBQgzFEtsPnQCqvKJohkPDoeTP0sX/eTzDM20YxAgoGDgEJDToMmLBgw4ELDz4UAoSIECNBigw5il9tGEExnCApmmE5XhAlWVE13TAt23E9PwijOEmzvCirumm7fhineVm3/XA8nS/X2/3xfL0/398fSI+qNYGafhKnzUar0+v2B6PheG19c2NrZ/tg//Do/gOAfFZJQfoEOuVq/k+cmgJl2gEwnViTNJGa1HAAQPsAyFJ4AAwYtEjyYve7INu8/ADAYsVawoRY0Pt8yJd8zafU2w2Af6MDoP9RYwdenpPP+RayRJUi1SrUqlOvRpNmADRaZrkO7/TLeHkZm4AEmpsJGRd/aGcAEMcHhgIAWBQZsNYmg3HpH0xF+QvLz+VO3/l0L1dmMlPZm+N5OH9CJhrGHIuscJdfcWRHuL7x/xmglqmVaiO1Rm2ndldvZsJQY4QTc/77t+N//58YxlHTZ431ttgbtwTn17mIsU/d7Za3GtXQ9g504wOhbDTimSZQTc/t5/nPYv1JLYwuxj8F5bVZt7pv3a6v6xDczd+1tbpW1tI+wnEOhd8nJHVsypGjYytG4dFrpB4+pkvn2+Uvyuh6J4ZvdFwnyhff+h7rFwVSVTqJAgUUKKJApc7sBQXG1c3dm8VHn1FQqJgRjmJQLDcnMwdOKrWG8/tWf0n7p9uKzJWrsmbz9RLWJn51K8qqbtputz8cr641noylfxqdwWSxOf/vH3+TUn9ESv//94jEkk/1aWhqaevo6ukbGBoZWzOxbsOmLdt27Nqz78ChI7ccO3H7Wemiu+6BEIygGE6QFM2wHC+Ikqyomm6xBnfsDqd5Bt3Pg2PD3KPyMI+Onyd5egyYCT3L84zkRV7mVV7nTd7mXRSwETAtn2Oh67fwiNieE/cjnNIMAQL4+Dhk889zj38AmRfi8neu9fzNePf5FPA37Z+NlqdQqxmG2RAgfT6UeyabsyZtz1LhFeEjCZgBfZVevMDL8WzGAEBOs51NrLvMP3SQkh50J+GGArKZ8gMBjasI5BubL3vXdnZyAIDvCb0NOBWjl82sddszGZN+i7skhnwCEwMD7+IhZVH7jMJr6GMq5ppwGppaGMEwAb9+4rz2+EJkZtQZEraApDTWReP2R/S9nFu3CGun6CiitaQloo/J7nWOXZiEXh2jopChgDsG++RbYne9+tkv2Kmxxn01Doq7VSh/EfsNOzdoxmQ+X61fsX1HyjCk42d2ZP9fECD45tN6nShLQt8SZ4TuVoFWYee04lyZxQC8kiCpI/AmDSwPwjz5W0/gB5yZgM9KHE9SDQvUH/p3SDqIvmm6rgbXf0lqx2g3drn4/YTaybMgLNdAzrWE4sDE9zupq6gM6I2blFThVoLRJUR3kGP7+ZA9WvXwYQGoJpnTNhk05gTxcLyHUQWxEMdgkkgkRyDXmGGCYQFet2jvGFo7XrZ7iAQzzcArDqepxGAFBVw3YzaHprhh7bDCvA+g53Mk5HbW4pSJUSd+BTDug+suxPBKg54u+RKaH4F/YBzE97HNMAEwv2OmcDNFDwJCjIQ8IWZCnK/8Kt0Ibo3i/OZs7w+M76vP7T2EAscLqWJfcCMSnv1LHG857uwv5DjN8Qors1X7bhuTz87wik2OQCpMdYNvxL5bquIwK+6TiVKPbBL3OH7h9Rb/3oqJagkQghHv0B7bLBjlHljo2I65fMtl7Dlvx0JotXkSr1/acoVwidDYcmaksjZSjgtwGWqQiM77iXGBPZuW3YzaXRHdGqZ2oqkE6ySn8sLG+yauHAeSeLqOAm6EBRWzmCiuxgFvbzht8SruT26xOwk3eCxb6v54fiHznH/yOhzYyaXxdScuTxcTpAcH467LwvxEK2M7+w/Ly8jYWDa5DMZrXwzelRoIUlfxb4CetniasVNW8J4pcrjWgEN7XGoUdhGZuoLrBBj+Cs8Lka7CPXChvy3H+zmI7Tc8d1KpmMUtqwkeU308ue8N4d3XYO4Aaf+62QPQKfQuE/oenB7O0BeYuzE9gmgDui24Lsb3P3WvT5FuIawKM2qnK0ZVSmrHyEsifbuPH2/0Gde/f69kOlu/AkVIuX9hOX4YMIrjhZbuDV/C3Y6b1gOKLDRcn8PSapQf9q0wVEYpPQE8ZBzpa+ojxc8DKyy2mSAABb/LhXGBNL1m3hHKydyX7hJFA3w/Eo529nz2LaXP9xHNB2SeZC/ajTXEqPXamyOmz7GiASgmwEKByENh5Omcx0Qg4u5HMA+VOgIJiqtns8d1B6F/sAcB5jXvwliy8CqRH4e2j0JLg/jZ0b8+Fwp7EeEZwDAkw+s85zrSAGQHcYi6lYFMxdFR/B63tdMod2Gu7ejTiO47ycPXKq+vQxlhEwTC4obFfvg/l1NnIs+2Bm/PhLVdMFUE3Vr4hgJW5+FxQiMfHnfcje0XXe0apeCGA1nWZHPo3Npf0fJi4jB4mnHKoGJ3InmPT/B8ddJ32xftb9UY4QHCwUcAoDT/N46ML7AvgmCVAhgMQShD0PKP5VgexcAwOCxlOAw6GYEj68pIhCDAoEU/MDq+wRgDmYEGM5rIWLRnwdZsZHEAArTRAAClZyEkdC66icGwO8kQou56DYpodBmGcrwlDtF4RpDZ2YzEcoYZDcsNY3RKH1vGwHHbIZkUfcZYWr6kD9m8svO/jVCv+dDs4tveUJ29kdsblexxWt/fbLTC/dG/XNZj8K/eadcz9tqJb4uUrT3t/yOpic9MnM3sAvRqUup243FfKT0MKOc+VMVHasvt2TUKIeqNECoWdF735WA8cqSRnZSgt74lse54Tu8shUjoPJmmQ81IjFGfIeZrjGFCidYcCQbl3RrJcRjhcYI1AYczHSLabJvWuq4PehMSXDI42U+xoLrGOgPh/ihbOMGFdQezf0tcfIqDAT5jmcBLvQAFFvT5VuZy7pIYaFzy7zqMUlT2QiIiTkhJf+BEUHOaNLchX9LGhTdILfMM8WGAjERslyaOObp19wMe3xvT5+e4L1yG94Vn1vtJ6E3BzqT0x/iQOGNgrggGEnmWJVLrza95BFYFdZr0pqn4xxnAz7ZPOrvxu9ln9GCHuHCq4qq2L77Vp8JuQPrgG8R7iwRQrDcvyLhL2T1n5FNGYDi1WhnLFfWeY4GC7lB0PGKUoMbrzuuEsy8uljTmTmNQAmvoaLLZQOmok1jKn+jxKpDVzuCgR3FYa2Y1qrBICINYO3L1m0h4Nbwz8Pagv0tjAAAA") format("woff2");}</style> ++ ++ </defs> ++ <rect x="0" y="0" width="711.9182835520442" height="573.5964413809206" fill="#ffffff"/><g stroke-linecap="round" transform="translate(117.27154915703676 238.02839846037386) rotate(0 125.6132919736506 62.51589262187065)"><path d="M31.26 0 C93.8 0.58, 154.08 0.67, 219.97 0 M31.26 0 C84.04 -1.13, 134.76 -1.21, 219.97 0 M219.97 0 C242.41 1.71, 250.55 10.05, 251.23 31.26 M219.97 0 C242.9 -0.4, 252.85 11.5, 251.23 31.26 M251.23 31.26 C252.2 53.7, 250.18 76.91, 251.23 93.77 M251.23 31.26 C251.63 54.31, 250.34 79.61, 251.23 93.77 M251.23 93.77 C249.84 113.22, 242.8 126.81, 219.97 125.03 M251.23 93.77 C252.52 116.18, 239.79 123.63, 219.97 125.03 M219.97 125.03 C156.21 125.09, 94.35 124.88, 31.26 125.03 M219.97 125.03 C166.39 125.29, 111.61 125.6, 31.26 125.03 M31.26 125.03 C9.54 123.71, 0.89 114.79, 0 93.77 M31.26 125.03 C10.68 127.23, -0.19 114.27, 0 93.77 M0 93.77 C-0.41 73.25, 1.57 55.04, 0 31.26 M0 93.77 C-0.32 73.57, 0.04 52.78, 0 31.26 M0 31.26 C1.85 9.36, 11.18 0.6, 31.26 0 M0 31.26 C0.96 10.27, 9.51 1.34, 31.26 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g stroke-linecap="round" transform="translate(186.70726707100698 10) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C40.5 0.38, 68.72 0.57, 123.42 0 M12.25 0 C55.6 0.03, 100.91 -0.01, 123.42 0 M123.42 0 C130.91 1.57, 136.88 5.9, 135.67 12.25 M123.42 0 C132.06 -1.24, 137.56 3.82, 135.67 12.25 M135.67 12.25 C135.51 17.2, 134.08 24.91, 135.67 36.75 M135.67 12.25 C134.61 19.88, 134.58 27.19, 135.67 36.75 M135.67 36.75 C137.48 46.76, 131.69 48.23, 123.42 49 M135.67 36.75 C137.06 43.82, 129.39 48.33, 123.42 49 M123.42 49 C98.21 47.23, 77.19 50.26, 12.25 49 M123.42 49 C91.01 48.47, 56.89 48.69, 12.25 49 M12.25 49 C3.53 48.6, 0.28 46.55, 0 36.75 M12.25 49 C5.16 50.53, -1.41 43.61, 0 36.75 M0 36.75 C-2.07 30.49, -0.06 21.95, 0 12.25 M0 36.75 C-0.15 28.95, -0.46 18.52, 0 12.25 M0 12.25 C-1.08 4.97, 3.28 -1.96, 12.25 0 M0 12.25 C-0.18 2.58, 5.05 2.17, 12.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(236.42063553159937 27.10232178526644) rotate(0 18.120773315429688 7.39767821473356)"><text x="18.120773315429688" y="10.368585785770504" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="alphabetic">Kublet</text></g><g stroke-linecap="round"><g transform="translate(248.10775797750728 63.908976948636564) rotate(0 0.33648643876290407 114.86137924595312)"><path d="M0 0 C0.11 38.29, 0.56 191.44, 0.67 229.72 M0 0 C0.11 38.29, 0.56 191.44, 0.67 229.72" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(248.10775797750728 63.908976948636564) rotate(0 0.33648643876290407 114.86137924595312)"><path d="M-7.95 206.26 C-5.24 213.62, -2.53 220.99, 0.67 229.72 M-7.95 206.26 C-5.83 212.03, -3.71 217.8, 0.67 229.72" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(248.10775797750728 63.908976948636564) rotate(0 0.33648643876290407 114.86137924595312)"><path d="M9.15 206.21 C6.49 213.59, 3.83 220.97, 0.67 229.72 M9.15 206.21 C7.07 211.99, 4.98 217.78, 0.67 229.72" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(10 144.1616587722251) rotate(0 345.9591417760221 209.71739130434776)"><path d="M32 0 C227.91 -2.73, 425.18 -2.08, 659.92 0 M659.92 0 C680.58 1.93, 690.18 10.06, 691.92 32 M691.92 32 C693.39 120.38, 693.37 207.8, 691.92 387.43 M691.92 387.43 C691.29 409.26, 680.31 419.89, 659.92 419.43 M659.92 419.43 C427.48 418.55, 196.22 418.73, 32 419.43 M32 419.43 C10.97 419.21, 1.58 409.18, 0 387.43 M0 387.43 C-0.16 309.85, 0.43 229.56, 0 32 M0 32 C1.75 12.18, 10.36 -1.82, 32 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g stroke-opacity="0.7" fill-opacity="0.7" transform="translate(261.64178340975064 78.9663359790759) rotate(0 78.24016571044922 20)"><text x="0" y="14.016" font-family="Virgil, Segoe UI Emoji" font-size="16px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="alphabetic">CRI request</text><text x="0" y="34.016" font-family="Virgil, Segoe UI Emoji" font-size="16px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="alphabetic">GetContainerEvents</text></g><g stroke-opacity="0.7" fill-opacity="0.7" transform="translate(31.502620036891585 163.87905007657264) rotate(0 23.152053833007812 10)"><text x="0" y="14.016" font-family="Virgil, Segoe UI Emoji" font-size="16px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="alphabetic">iSulad</text></g><g stroke-opacity="0.7" fill-opacity="0.7" transform="translate(206.6755449380774 253.0308946088519) rotate(0 45.07208251953125 10)"><text x="0" y="14.016" font-family="Virgil, Segoe UI Emoji" font-size="16px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="alphabetic">CRI Module</text></g><g stroke-linecap="round" transform="translate(297.8433794466403 435.23132280384584) rotate(0 65.17109829776129 30.02618577075077)"><path d="M15.01 0 C53 -1.33, 90.88 -0.47, 115.33 0 M15.01 0 C37.61 0.15, 61.85 0.33, 115.33 0 M115.33 0 C126.37 0.51, 132.33 5.12, 130.34 15.01 M115.33 0 C126.67 -0.83, 132.61 5.73, 130.34 15.01 M130.34 15.01 C130.04 24.37, 130.92 32.18, 130.34 45.04 M130.34 15.01 C129.89 21.8, 129.98 29.11, 130.34 45.04 M130.34 45.04 C129.59 53.88, 125.54 61.84, 115.33 60.05 M130.34 45.04 C129.89 54.56, 126.11 58.24, 115.33 60.05 M115.33 60.05 C93.54 58.25, 72.69 59.79, 15.01 60.05 M115.33 60.05 C78.31 59.22, 40.63 60.44, 15.01 60.05 M15.01 60.05 C6.79 60.04, 1.18 55.22, 0 45.04 M15.01 60.05 C4.45 59.29, -0.53 54.55, 0 45.04 M0 45.04 C-1.49 39.75, -1.38 29.36, 0 15.01 M0 45.04 C0.16 36.46, -0.86 29.76, 0 15.01 M0 15.01 C0.47 6.74, 6.84 0.34, 15.01 0 M0 15.01 C0.17 6.96, 6.65 -0.03, 15.01 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g stroke-opacity="0.7" fill-opacity="0.7" transform="translate(331.5344057229172 445.2575085745966) rotate(0 31.480072021484375 20)"><text x="31.480072021484375" y="14.016" font-family="Virgil, Segoe UI Emoji" font-size="16px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="alphabetic">Sandbox</text><text x="31.480072021484375" y="34.016" font-family="Virgil, Segoe UI Emoji" font-size="16px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="alphabetic">Module</text></g><g stroke-linecap="round" transform="translate(464.3025362318838 433.3645573229496) rotate(0 112.06915495125267 34.23649538866903)"><path d="M17.12 0 C67.56 -0.3, 112.74 -0.22, 207.02 0 M17.12 0 C84.25 -1.17, 152.52 0.32, 207.02 0 M207.02 0 C219.93 1.68, 223.74 3.87, 224.14 17.12 M207.02 0 C218.01 0.95, 222.61 3.94, 224.14 17.12 M224.14 17.12 C225.24 28.68, 224.02 39.75, 224.14 51.35 M224.14 17.12 C224.87 28.16, 223.65 41.46, 224.14 51.35 M224.14 51.35 C222.62 62.47, 217.1 70.44, 207.02 68.47 M224.14 51.35 C222.09 61.36, 219.37 67.26, 207.02 68.47 M207.02 68.47 C133.93 68.83, 59.98 69.31, 17.12 68.47 M207.02 68.47 C134.98 68.23, 64.49 67.21, 17.12 68.47 M17.12 68.47 C3.77 69.94, -1.05 61.77, 0 51.35 M17.12 68.47 C3.99 66.69, -0.86 63.22, 0 51.35 M0 51.35 C1.56 41.15, -1.01 29.56, 0 17.12 M0 51.35 C0.27 43.35, -1.09 35.56, 0 17.12 M0 17.12 C0.95 5.19, 5.15 1.11, 17.12 0 M0 17.12 C0.6 6.01, 7.79 1.23, 17.12 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g stroke-opacity="0.7" fill-opacity="0.7" transform="translate(476.33948750882007 457.60105271161865) rotate(0 100.0322036743164 10)"><text x="100.0322036743164" y="14.016" font-family="Virgil, Segoe UI Emoji" font-size="16px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="alphabetic">Container Runtime Module</text></g><g stroke-linecap="round" transform="translate(202.96133893280614 294.6317354405428) rotate(0 52.98359829776129 25.625)"><path d="M12.81 0 C37.27 0.29, 61.21 0.79, 93.15 0 M12.81 0 C43.76 0.44, 76.15 -0.31, 93.15 0 M93.15 0 C102.9 1.89, 107.43 5.16, 105.97 12.81 M93.15 0 C101.4 -0.21, 105.39 6.54, 105.97 12.81 M105.97 12.81 C104.95 20.98, 105.57 26.71, 105.97 38.44 M105.97 12.81 C105.44 18.29, 106.85 23.3, 105.97 38.44 M105.97 38.44 C106.33 46.07, 100.93 51.05, 93.15 51.25 M105.97 38.44 C107.52 45.41, 101.14 51.44, 93.15 51.25 M93.15 51.25 C75.48 52.35, 56.61 52.58, 12.81 51.25 M93.15 51.25 C69.64 50.85, 45.46 50.34, 12.81 51.25 M12.81 51.25 C2.3 49.7, -0.95 46.28, 0 38.44 M12.81 51.25 C5.02 52.66, 1.44 49.09, 0 38.44 M0 38.44 C0.07 29.87, 0.25 23.89, 0 12.81 M0 38.44 C0.11 29.84, -0.41 23.97, 0 12.81 M0 12.81 C0.86 3.4, 2.88 1.14, 12.81 0 M0 12.81 C1.34 5.63, 5.15 1.89, 12.81 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g stroke-opacity="0.7" fill-opacity="0.7" transform="translate(212.07284402988387 300.2567354405428) rotate(0 43.872093200683594 20)"><text x="43.872093200683594" y="14.016" font-family="Virgil, Segoe UI Emoji" font-size="16px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="alphabetic">GetContain</text><text x="43.872093200683594" y="34.016" font-family="Virgil, Segoe UI Emoji" font-size="16px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="alphabetic">erEvents</text></g><g stroke-linecap="round"><g transform="translate(340.1307113548334 434.23132280384584) rotate(0 118.95745710012142 -47.87031585660827)"><path d="M0 0 C39.65 -15.96, 198.26 -79.78, 237.91 -95.74 M0 0 C39.65 -15.96, 198.26 -79.78, 237.91 -95.74" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(340.1307113548334 434.23132280384584) rotate(0 118.95745710012142 -47.87031585660827)"><path d="M219.31 -79.04 C226.69 -85.66, 234.06 -92.28, 237.91 -95.74 M219.31 -79.04 C223.6 -82.88, 227.88 -86.73, 237.91 -95.74" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(340.1307113548334 434.23132280384584) rotate(0 118.95745710012142 -47.87031585660827)"><path d="M212.93 -94.9 C222.83 -95.23, 232.74 -95.57, 237.91 -95.74 M212.93 -94.9 C218.68 -95.1, 224.44 -95.29, 237.91 -95.74" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(446.2179677206851 323.87552036052466) rotate(0 -67.33072154104592 1.3508267927636552)"><path d="M0 0 C-22.44 0.45, -112.22 2.25, -134.66 2.7 M0 0 C-22.44 0.45, -112.22 2.25, -134.66 2.7" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(446.2179677206851 323.87552036052466) rotate(0 -67.33072154104592 1.3508267927636552)"><path d="M-111.35 -6.32 C-117.47 -3.95, -123.6 -1.58, -134.66 2.7 M-111.35 -6.32 C-116.63 -4.27, -121.91 -2.23, -134.66 2.7" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(446.2179677206851 323.87552036052466) rotate(0 -67.33072154104592 1.3508267927636552)"><path d="M-111 10.78 C-117.22 8.66, -123.44 6.53, -134.66 2.7 M-111 10.78 C-116.36 8.95, -121.72 7.12, -134.66 2.7" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(588.0527570054602 428.8174815081436) rotate(0 -0.8929182632952131 -45.627363327153034)"><path d="M0 0 C-0.3 -15.21, -1.49 -76.05, -1.79 -91.25 M0 0 C-0.3 -15.21, -1.49 -76.05, -1.79 -91.25" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(588.0527570054602 428.8174815081436) rotate(0 -0.8929182632952131 -45.627363327153034)"><path d="M7.22 -67.93 C5.15 -73.31, 3.07 -78.68, -1.79 -91.25 M7.22 -67.93 C3.77 -76.86, 0.32 -85.79, -1.79 -91.25" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(588.0527570054602 428.8174815081436) rotate(0 -0.8929182632952131 -45.627363327153034)"><path d="M-9.88 -67.6 C-8.01 -73.05, -6.15 -78.5, -1.79 -91.25 M-9.88 -67.6 C-6.78 -76.66, -3.68 -85.71, -1.79 -91.25" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(432.46706192358357 243.8230477655593) rotate(0 115.3102616706203 71.00074110671915)"><path d="M32 0 C70.74 -1.08, 108.83 -1.91, 198.62 0 M32 0 C76.76 1.11, 120.12 -0.12, 198.62 0 M198.62 0 C221.38 -0.05, 231.57 10.4, 230.62 32 M198.62 0 C218.43 -0.94, 232.8 12.94, 230.62 32 M230.62 32 C230.94 57.17, 231.6 86.69, 230.62 110 M230.62 32 C230.33 61.69, 231.36 93.77, 230.62 110 M230.62 110 C230.74 132.33, 220.4 143.92, 198.62 142 M230.62 110 C229.03 131.88, 221.1 141.35, 198.62 142 M198.62 142 C156.63 142.43, 115.38 143.73, 32 142 M198.62 142 C157.68 141.22, 114.68 141.93, 32 142 M32 142 C9.62 140.04, 1.37 129.56, 0 110 M32 142 C12.07 140.6, 2.29 131.94, 0 110 M0 110 C1.42 86.17, 1.64 60.5, 0 32 M0 110 C-0.55 86.29, -0.53 61.36, 0 32 M0 32 C0.95 8.83, 12.13 -0.09, 32 0 M0 32 C-0.4 10.51, 10.84 2.1, 32 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g stroke-opacity="0.7" fill-opacity="0.7" transform="translate(502.0197478366281 248.76318281167232) rotate(0 58.4481201171875 10)"><text x="0" y="14.016" font-family="Virgil, Segoe UI Emoji" font-size="16px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="alphabetic">Mailbox Module</text></g><g stroke-linecap="round" transform="translate(544.187664690382 297.68363735712705) rotate(0 41.46844678260974 19.261363636363512)"><path d="M9.63 0 C33.94 1.44, 56.41 -1.86, 73.31 0 M9.63 0 C31.38 0.34, 52.12 -1.12, 73.31 0 M73.31 0 C79.77 -1.9, 82.77 3.56, 82.94 9.63 M73.31 0 C78.98 0.73, 81.89 4.14, 82.94 9.63 M82.94 9.63 C82.49 14.97, 83.73 25.42, 82.94 28.89 M82.94 9.63 C82.98 14.3, 82.47 18.64, 82.94 28.89 M82.94 28.89 C82.11 34.47, 79.6 37.4, 73.31 38.52 M82.94 28.89 C82.97 35.77, 78.12 40.17, 73.31 38.52 M73.31 38.52 C49.98 38.99, 27.59 37.29, 9.63 38.52 M73.31 38.52 C51.54 37.58, 29.22 38.7, 9.63 38.52 M9.63 38.52 C5.13 37.91, -1.27 36.98, 0 28.89 M9.63 38.52 C3.94 38.94, 0.02 36.62, 0 28.89 M0 28.89 C-1.37 23.52, 1.52 18.82, 0 9.63 M0 28.89 C-0.15 24.06, 0.2 20.74, 0 9.63 M0 9.63 C1.22 2.19, 5.12 1.92, 9.63 0 M0 9.63 C1.79 4.92, 1.93 1.5, 9.63 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g stroke-opacity="0.7" fill-opacity="0.7" transform="translate(559.6080539168394 306.94500099349057) rotate(0 26.048057556152344 10)"><text x="26.048057556152344" y="14.016" font-family="Virgil, Segoe UI Emoji" font-size="16px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="alphabetic">Publish</text></g><g stroke-linecap="round" transform="translate(447.2179677206851 297.9866676601573) rotate(0 41.46844678260974 19.261363636363512)"><path d="M9.63 0 C29.49 2.05, 50.25 -1.45, 73.31 0 M9.63 0 C34.11 -0.1, 57.77 0.52, 73.31 0 M73.31 0 C81.51 -0.11, 82.9 4.24, 82.94 9.63 M73.31 0 C81.72 0.5, 82.42 4.13, 82.94 9.63 M82.94 9.63 C83.26 13.4, 82.27 18.62, 82.94 28.89 M82.94 9.63 C82.58 16.43, 82.48 23.74, 82.94 28.89 M82.94 28.89 C84.15 34.17, 80.61 39.22, 73.31 38.52 M82.94 28.89 C84.1 37.15, 77.43 39.1, 73.31 38.52 M73.31 38.52 C53.4 37.79, 34.39 37.28, 9.63 38.52 M73.31 38.52 C52.62 38.09, 30.49 38.09, 9.63 38.52 M9.63 38.52 C3.2 37.32, 0 35.88, 0 28.89 M9.63 38.52 C1.97 37.79, -0.99 36.78, 0 28.89 M0 28.89 C-1.8 23.78, 1.6 21.06, 0 9.63 M0 28.89 C-0.93 22.65, -0.76 16.28, 0 9.63 M0 9.63 C-0.2 2.86, 2.84 0.9, 9.63 0 M0 9.63 C-1.08 3.89, 1.53 1.08, 9.63 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g stroke-opacity="0.7" fill-opacity="0.7" transform="translate(453.22233369274795 307.2480312965208) rotate(0 35.464080810546875 10)"><text x="35.464080810546875" y="14.016" font-family="Virgil, Segoe UI Emoji" font-size="16px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="alphabetic">Subscribe</text></g></svg> +\ No newline at end of file +diff --git a/docs/images/cri_api_1.29_overview.svg b/docs/images/cri_api_1.29_overview.svg +new file mode 100644 +index 00000000..94911913 +--- /dev/null ++++ b/docs/images/cri_api_1.29_overview.svg +@@ -0,0 +1,5 @@ ++<svg version="1.1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1209.6444540126743 730.1247574550649" width="1209.6444540126743" height="730.1247574550649" filter="invert(93%) hue-rotate(180deg)" class="excalidraw-svg"> ++ <!-- svg-source:excalidraw --> ++ ++ <defs><style> @font-face {font-display: swap;font-family: "Virgil";src: url("data:application/font-woff;charset=utf-8;base64,d09GMk9UVE8AAO9AAAkAAAABO1AAAO73AAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAADYTJHQZgAIluATYCJAOQWAQGBY9lByBbpzpxQ1QZOitEWvl1ExHg7NqqxsS6RlCwSkciyuhqMfv//89KOsZwUAeGmJXV32EeijNbRUcYuVUTmWNvG5FWRBfkAW4YWVueXU6tq9d0pZufMcACcUzc7AmXTyBwYmw0WojBScItqLq853lRPMu/i21F/1VrveHFeyj+LdHE3AfsFiWacmNotmSIhpJesVGSFIFWJDgsMxNmRIsQQzGF4rvOX33MPtY+LeOF490dgR0G/l0zmGbP8ku9beGBOf2+EcKFm8Md59ESM5GZKYtFok/F14jmMBPlbsKEWTmjIcDMVCgM0RQv1VD4IZddn+x6g2Ff91p6BriTOHYqRgBPP/Z8O3Pn/WXBXUTBKoZoSbZg6N3qwPP/937uvc+bD5huHSziTSgio5mQNpAsr235cGq9P8/r5vc1H/KatrSSHj8fGwzDhavuia24cMyBvutkjYQtKFtwTcS1wVWcODY7KMSFOM69OS/+4fm59f7vRf5FsY0aMXLQQgsGrRwggmTYYCJVEmVBm4V5Z0ThidFYPeyrtRKEoeme3dvtgT6iOeJnlC8MsAKKT0UoFD469oV+l7DD8pn6lvkVc++0kSEfkCM0Vauqh+RnHNszHtYIqREKELpb3S0eAhMvIXmB7+tkb9/wpNO+5fOjmT/aU+6YYwu6wVlib1xrbGxcwNgduqmiNNOESimjUWmogISEAOOOe086XjvNccGlpjQnb8gjOY3Gyr1sLlmavf8vKSIJwtUoBiEBfY1jpaqrq1ET/z2/7f+/4/hmD/yU4cZxDvyseBGJ7/W9rVwMSlAkJUsk6hzqQCsqJVbd9NbifX/4r6pJ4LDfsD5vl0a3Nl1TSqk09SUxpviFT9KCpj3jFmScMz4t9f8VWXkadd4bSR35QqxNH33Ai5BewBI5hQAbQsZFe2nQZ84A787uLKHXduwYMHUawjI9wuP+7R+jZdW/WpL3hdof92mkcoac8YBwj3gJB7CB3aayqzgrsSIikYrAZYK2u3l6AJfgdm+OybOvB8+JTFAi4hU026JOz7fHCxQujUCxVVutY8Wgje+dK4CA+afpvl+9Nyecky85rLBnngvwxHEpBa9ATSI+MxnxjMcbadI6LaWIPXHlOeusTrrDBkAFCAzMc6UTOFEFKTY1Nm4GuAPasdAah4+DcVmNg/pfU5Nao+e7QykouG0KYR4ewkKQVmtP1n91ike6Kvt60/WySWeBpeOKr7QCg0hoKAwgPPOvqrl+UPI78CqVirROp/RhKm2YMy0C6CJQdAHdBNBOBOmKyGuC5AZe6X1LKQUflBN+2M7DB2VHoBs/KOcRvCZQfn6i0lsdM5Ypw5otoyCn0UlftixbMiaTk2XKy5Q1Y17G2zJOB7nKUm0/PkgM7J3K/X5azIxlS8eShkmeIYrIIuvN47j+8Xk5xmbzq/fd0AYlB/aifvZu/3or31eH2eCosZNM23gKUvMyo25K7fx8x2ZRqLM4/xxLbOST4x/7aOuym35y8olHctP4b8V/TX/YCP2p/1mult99+Gj0u488j59Q7/DTaWN4EkPx7itzRBhgghZ7XPAiQJAIcTLMtFCJKku1WqHHWlvsst9xLnKDYR4xwicQZ7tG+knSE5NnLF69LLqrR0X89vtHVudoGtiw59gDkletTr9hov04IcNsWAhCFGnUMYNrWMA6cqigBQoCbHTgQRhxDOMAjuIkzuIyzuAybuEuHmMEbxC773VEWIigqlrWqT4NbHCjmtiM5nd+y1vXlq5ob9d1c/f3eC93qA/7pk6NEihFn0Rmmnk61thin7O8wilOc4krXGeGedbYJEaSKhN6GOUQD/IKz/ACr/IW7/ERn/Il3/ITf/B1//9y0DjoHH5x+M2hIRCCxhEpTKlFmUZcYipGDGMmFmBNbIBtYYfYXewxq9h9hvn6/vr3thQeZ+KUArHScmqorznNakXbKqkpSqIM+SLlVlB9GtJ+HdUZXdZZXdGQ7uuZ3uizfnlBkASHEFk57pwrbnvKU17wunOuGjNrxbYj2+1zj/u92yM+4fO+5gu+7jt+6Od+40+uMZH/QTH0f6P/F2WjGegMdD6aieagxWg5WoU2oG0oH5WhWgfrEB2GI+UoOOqOGcc5+Rat6/9k/8+mUCCeJiIC5sZapHx9QEX2IMLTPuIbQjZoyoY9XgDCc1CSUcEfzRVFJcPJ4YfarTTXnEEp9Fll4bmCyF6huXD0c2NkyBrUclqFDS0fCcldmnBxq5ZzUdjQ0hBhra3noZkdrY2QIXdtwDXbzuJpombNK3hcpdld/aaHaGW4yOQ0Uq2KuSpqqHgkIlXMSxG5hZWvipLO5Yja+dGe2Ec0Xl7hJVeLriry+vKRHMnapHSSWh3Na/r0xosnKF8vBiOnHHm02DmrmKh7JKa/6Kx4A1EGB5L5gPZja3EC36Px8gVmpEMfpVo7eKal3dCDQk3OQWrtEfywxsP7kXKdc+xllnS10CSorh6UNHRFJNThKpI3Uj12NefH5EBdrYWSRruZkfa+i7mL+aR7b+irdu+/Svdr602Xpz3AJtUl38juYpi/pk0HYWKZtRHy//WcNGHEROmgcepGC9vOyzGRY1+qfJxX8Ka4G2gxbNKvLqrqBjSlQRMYHTrDDGsYBjAs71XrIXW0LYetKxhHqGHoNTTSyi1NsvSW9pYPZAY4gkxWtrU0z9vbRMU08fOJQxc+Syy+gtk9G54gA+xj+zrZX5Y0rjZrdBmGy68X/sBnzCF5AdMtKp4mqPbyQez0lZGifaq47Tf7SIZExoiy9kkO2JjFGs8VnQMdK9nMR0/nKEUbGADCEcjocG6DUGjz+CoAhQFwBBIDEwuFxo4bN+++CUBhAAhHIDEwsVBobBxhCwAAwIcDAomBiYVCY+OISUAQBH0SBIEgCIIgCIKgcvQdCMIRSAy0MCC+M6Y5BcVI/gkYkMn8sRZzoj6jjymGesj9+I2a22/ocz8NNPA2C+fgZtOvbe7ZbLKKuW1NchzK6mJ1Pf/1IF9PGNzuWk+fWr5AJLsGbXx6WqXtz7Sw5sK/2TRigdQEnKWCSwHOJDBxwFWcy/MuPpBExwGTydS++B38ygHYhoswSGJKao0cwRU+cxa87AMKDql645JDKYO+5m4QVzcJ3y7stzuYTaZTHAbNHXHpPfKmuHugSHKiUk/ZPC/TXrC0xqt/dsGQ57sgXd3hs/6nDbLRY/DknG/yxVziZVz+ldwy0Iy6cNfv9o1UVPeO7SRI/Eej3M5tWU/0ydP7mv8W+vb7QPcr3ytNdXF9bro22Y7bj7Zf+/5e7C/73n67/xrB1Iy9URj6uDPOjzvj1fgeZsnDRCrKMR3LsR2VaAcbarhhDXcEIhLDcSROx6U4E5fjZgzHoxiJt3PJ9GZwpV151l4c+73B+fGXExniT6JM1AELXkAM6XgMP5YTZEIWvN+8x96yP86/HmkfeR30BDeCq6FruBPGwn1qRw26oBG6Ql8N9qqpBupSDaqaqtbOeqAPtE9H9a6ua4yag2bZtBiT0RqTSZtKYzeKbdmu2Fors26rtyFbtJX2mzudd3CO+7zBVdzBKraziKWsZzf7OcUPuIzfccG8beSYgWdMypiWsTBjeUZ2xraMHRl1GW0ZwgxVhjlDyrAzchn1jJlUWRX48WpiZWxuhR6qmw57kBDolGzsgvY89rhb7XvJzwQfY4+20SX4OtWreKpuSFaXlECpiOVamcv4JA4/x8nKhM2qKCxtMjn4oyulp5+2+DzPxYtYzPSoPf9Yvlvd4vcuYU3142xsRkmAfbwfK6ODt6039/4JpW8VEqlHIZIunBL+6JmzfHyE3o3mF33fTPtyd+kt8NrwT5yY3tXcECqX0RZRjsN416y3bBO8IjDJcPLMjeX3UFc4USotsEQLOQvWwDz+tuofrKKIxg39EBCg4K6oeW4pSCLqm7ih+kAy7/Fydux5Ldh5nZZU05IuqzLvripqXcA6pZ2RPKH/d7Vhy+50yl+yFh/UIyLAjn8fv/fx2Z7SNsWc2qzj/U7z+cFG30qoPh7Z5qC3Ze5YhCSBZYH04b4xhJmKKLbQSuzNojJnYgVKYcrTV4jLFCCbX7QrsD8wOFpEHcVC6m7/T4/rDbOCMYvv9JxgP8P+5bnnhxblt29k79MERhrZfpNruPWEn9zyaWm9D6Yhlysalgwi4BGLoGwXLltLytTuk/Fts6n1vh2+DbWgbnrIDLRawWQ8nikM/EuPyg9/viTjuTpg3B1Y+hDUoC9ieivGQlh0cicgzznLfcQVZ3dVKCuRSmjBAmXfyhNu9pTGGImlCL53o0K42CnEDXXqbCl182f/Bf89gv5vwu2e7vJ4iWYg3CGwkKTThtD0GoseIwAWO58TjMhWCDTwf4n/MWnupK2x/YCzIig48kILRHftWSKYgi+uRo3xYmrCurpHP+wg/v01/53uutoVJiUQVDulJgTOfnkrR0ZI1k1tENAMdR8iv1SLeO3Yf8I/i8/b9WdwvtwVMZlpeDIIzYdmpyAtzZMJSHtCcxcsWTFrteeNzqfiMmfScU8kIho1b78owsx+mq4uQutyR6w+9Yo3PmDBKfNIugWybM4TIVcd0CWURQ0Sc9kHMwiRyovyick6zb2ikU77TH+AKiu2Gf+Ahqq76ShpCk2KlUBjcPCK11f9dj+F7vSs8H0W2JSabHDxn24YNFev43yHqeq6XfE4R3rTOnZgFh8dEP9Q12qAHDk022XQ1pUSyjS/MSNipVuDYUYp0cM2GyEIDULG3HlLVE2Auyq87HOQvZquGjVgYq9KECHaIH0yk/2vr37SWfBV8B39wF8uFIxNfTsoDBvADTTNVVQ7OCrfY6L0G9yaU8aGZUKY7KcewrBtVffi6ub8GXxbdpI97b8hBoFQUgYJ00i0awM2XFgyVBGcFpXLAuR+2yS9JkI/gib26ZWf37eeNzxCOI7hrUwsKtpDVjRC3jQBB+N9CokJWXc0xw4tO43puOVMxB3S5XQnpNSwKH7otNIEjQiM4uWWjq3nF5hOi5K7xSdq3G9gYjUCzpPda9tRcOW32Y1gO7B4VKSrcYl2suSDeHI+STDgD5mdKorFl+SBLm5wuaWCyjmlZRMIsqBi5kErzkGOJ4nBBsasiHSBSPGEh2GWK1XRkbr0idAO/blDD3UvNXmqJJmxpciGbqkJFkRNIUlDf+tr0yuVBmbNePeTRpvLyLH+s3osJPvlgTLqh3bYTHbiV3M6AGRNwjDRIuSterpZLzTWgcuy62rHrJUFzGaxIq+ArjzfVthhOorC0AiCppSU1uIeWScjvuX2saqaGLe8Fh3SttzX2jZ4L/t29aumKWwl4SBuLsVe6wvvnzl7f2Fj60x2nQvYoDXgEEtrWEGMMhekBIg0R6qwt5tu7aJsxFY80wzcmiOQL9SbhS37pegG7yUM0NERxJnuG9BGqdPXDMtrch5+Nbp6qOhNVG7d8BXcqZcV5coyFjc48CSzVlnjgNh2I9tCF1i0cmNaz0tWcdOWmnZQokq6pKW7cjrIj6FvBr7l+gH1AYTeF4etWFaoxef12lP4cFlvw7p8amqeUgDZqPYsC1MlUuNK8Mrdazc29gBZsdmRjPBrIuC6r/YdOIIzHVFYFJYrS6tbwdJCTvZNziE5kR7IqdUadOyGS3ESUHqK7/9NM3cbbyjUVkSsAOtQcaUIuezfbaVYT2V/XL5KOV9TQ9fG3WezS8MLr2cJBkVOfn9tH754Wao4QR+eb7wYvd2+107VAeA12lEZIBOssvLIFFCpkDpjRTY7Ovv0+7XIpQne3FTbfpFkF4sx2A8e2b/y6YM3OW4SCexYtGl0iDncNe3uxpa942A1SXQP5Cf61mjfYINvow1RZVm/4BbL9Xyj+fTag1N1gDdXtRHSP7w5j1qrmZ5xB3ApbsCAIKQ4zRDLkDfbE3FVSPJxnsxGk+P9wytWacTijZsSdBKq2ytb/prJ+O2mCchsWw3g0mDn9m7b4DN2d5seOaA066UNdPbUzuH6nQfXAunVgqo2ekuBJiumUfT9cxcjX6hdfdEzvjRfml7aBmEgqFVDoUAjb8kGGy6WB4uKKOG8vup5eFWweTsT9ItV3gQCp1IF8dX6Ok5ox9gPGraY2+n1iq7ZkiVZ5Dm9YDs7FyRqz+xpVVUolMUgZ/h2DpOmplbsrFN3TFtxNEArmqxBPWcSCxlJGisugPhwyHi/GDsfHF06QQKjxAP8pKgompEleE9YZHz32WddH0hyGDqskcSu9dgEuaqpMCjrBRHiIZeQsMiKnimZr0syQ+0SvxE9AFhxhQTs/AE/sanpzViT/b1wMD7ZvX1rBxCO8WcNLUCqHPuaarv1pOIBIHGalH2hk4EPA4jG3RO5MbMxXOhX7kx99f2ds1Vs7czum7f7DmvB5YZGY71ArWtIxnOaBSRdxAprUkE9EkEqH4Y6kuSWwazEblObCGPKRPDEqNqV1ZqTdw0xlgornRgwgOezRgQpEMGwA4+rQo3NbYvbG1gBuLaPrA9GzVMv9HbRsnav4EhDT3mZOlKNK+/e+Fx1s1pz/M9+UXf4Ja1DUcdd6aYJ8L++VzqQssWypCkghOsGqraGxEuQ2mgxXrGzpnFHzPRhhCT4J3DIYv5KjPnM8J3hYgyJRUFRERCiihSmv6uN1bjpzcsbV/NL2xfqs00vXq6mMiCiD4gbnSqNoxUtBqPV1/g+LIxqtIXWiofbm1thsPl6Tpp+Np3aKwM27ZYdKGpYoyiRVXwlWj4WDsx7Sn4iIWHC0NlwyGTXXr6wn4uzjyhq6JJ29+I1ABL4ss3q49ZkuZsSPlHwQmoZbRecMh/egNGDZJQi3KCtimV30adH2/PUB1KNGxptAPUUU7ajIqzWOquk8wrlY3tfbHumeo8VEOyryY9PbkV+J1BKHY2KWtbXAMeel6CQTZtybf3V627dbiPd71q46Ddo1HO0RRCRHyo+zBQ6YTv23devSKVABnQx4MJ/5Tlb/kCNO2giJf1rDycrANLWTQHC9x/Z272Dh6XbUfmlokgarA96u5NbV5DMzZEIEqnHX51L1IoVK7ndbzlCfP0n1UX1AtgGWfbL81d6m/JpVqBUQQS48gdViQ3PSvZD+VA4KO8s3VKOzdv7/ejA1bbLR/zvRoPib+DjXfcMJQd2I+MnyDoe/DePZDoN1QCCWscaqmlTQKd1AtMAB7bbbXfVsTEmNmOQ5EIvYqiRay296XeXArEJZJYnNeiPqoaL4seQk549GuJ/GTLKLduXFx+/SCHfYgMfIXRFrLHzo25iixS8ZaBbAuc7Cett9j01p1gzBodStOCUNZCCwIubdyN/utMxWO14HRr2m54XGI7nOpR6FsVAzvk0gLiVrFY4QZKKQSnXCC1RAq4xJbwHvP8Nyy0kkV7GC9pOVkK5M6vCwqa0z74TqahlwTSNmZKWbEHNZ6K6hPrG5FaXGRswXApnlnYWSWn6diVOZ79nOvEtn+Ox0RQ6xrJqA1cxLbjQJHiEZXWqYnb54dX2zEoxAQcYyxtX5JSdZTobF3KdImDFfgmUvYKII2bqGld+gzJ1Sapw1KJKJmbE8qJDxZTF08osFb/EMhMWs+mPDlXMCwmssLjEqUIZM9Ut8a2YS5QME1arzFoJ5lck07VmWSrXpid4zLMi0ww0pN7udPnlN1/hvUYAl62L039i1+t4moKRhtpjPKdhhJMXOPL0MpLpz0T2r4vrH581ODlaJd/z5uxALVOmys4gc4g4tnkm0yt9Yk0AMSu0KvP62glaZpuc/2JzG9suZErr5XUgDRhB1PifHrq0pvMFWDgK+c4wj14B5Om3O0g/F3lco3RR/aQgWEeRUE/slB77cjCLjTzqvCmGSrPM1EwiJDTNyALmpLmIgGrQ2pXdTLbk7vLBQQvjmUcktrqu72KCXYVU0+t1lYKg/gArCHwpVrmaQL0yckjHi1APcQhyykKTXX9tS/qqYwCZr16RoNHrtq9ceWLl/S0rn7jmlkW/waImOdjpXOn4bKbQpUscuG9fGb+xzmZMxYR6klAHLRZ/qJA2mjDlf+3hX0OMsUSsPQVhOPHsYmjyqGpQD4YJCCSMPCQv2q5IGDmI31OLvrQ01dvWVIanBA+Njw15M/XUSGnW/6I+tzKLo9Fj1+xXGPPUprENKxNidYwuiSiZbFutTvZlMVs5l9sqMQaR8LGJZeJUV3OWPWBkUfftVXC87aB2G39YDFYflLVYi5pIQDDNFBxKBlE1ZOZrWzjmgpJnu0o0AaR6neXUqQkJ19loSiYKkfHMwYnKfhMEUXOx02VkOGu+xrm6x6OvNc+9DajfWqqbU5OzpSUNwPN1zKNjWoHzF6kuw+EcRTkak/k0VudILpcSJXaPdFg+zHgJuWznOoTERhyYB5y6SCPWjrpkd39NaLQajfWd/uHWHoASlSRGba2322QquVondGQV20ZoNwLfO7l751szicl4a26KyEdv7bSPATtgJwUb6GRJtSGKRl4JhcO8qCVFZmPBKUkck2FO76rN17PktdM99RuLxKbn5gs3J3FHSuk4aqAk1ZACcNFGkJFaYxvBhvVyQ90LPqguF7fHJZ1+zh76WuBJPmLUuJoaNjBbPhdW4Abx+f51GmgAZIOYVBBv8i7WgQIt4w0bU4yN8UMzmMxcimVAllVR70rk+mJb1x1wRcsWr6SuCegtXmcFbZXI4pazvJwVljqqZGiqagmkGOQkoc5QGbD5S1rE/m9oYZeVhKXYlyqQFwoul/d5Nrp6SiKVmlThv1KiIog52ffqXvnT3xxQlwGT9OGGu9bWtn0+Dyqyyk9G51mb7sw/akWV73//dJNdcvmSQDNd/m6qUZ9XHRhWK0kRVetydUlKiYgFYr/8kEnFR2xXNiI/QK9CACmZ/+YXnf3LiOjMsS8ZnEgG63G1TZsNvzeGc4oeqmGeBLxogCYm2XWeAfNvJLvNO5S/onTtM7bnt9NTY6Rds26C4bbw/urcKix4S9t5hWLznuKpgW+FhAY3TMCEqVDWsJqeaX7XIyBw19HqWn68/u5qx/PVytZMTO8CGZaKfXMnuNX8WinQ8dW260lAKBdV2DWmsePLDauxHGxbE4d0lFBYwuzTG/gjJBPBQ0io/Z12b2eodtoTARgDpdIxSo5+/4l7YUDGSCgX6ha4JVZg4upSBZBjXQOimGEU1DcuUqbPWXsd/MczAA48M4B+0gpp9TEqiOV2NFhVIct11HPA3pfGoCQu+SIU8A0mREVOuSKXZNJys9d6O5kBWbq07svP1xb5aq4qn/nEGlSDh+O2g28bS6CG2abuNYKw7PjKDbQPhnlDt3AYZMLk87dDwAMyto8/2B+x8bvNlNB3R4DT6MZHUtVtlAr3xh0mpQ7aKnJAJSshCpAgdz2uiiAG/rD99o/XUODaTWr5BKoOdiMZ1Y0vug2o1ShyFVF2MZQNMMf84E66z0huVdwVpZcJeHZLG3DV0RwtJ1H6bzdWnZ87BkUEktYTxVcdByuZp1u352nwaA64cpHGX1aA2Pm3jjxjy9gGCbqilSAW9cM8wslgTjApO7u5Dfenl6eYOWrGoMI2wYTCoLyj7iub/VcZdDk6C4sywsci7sbhk8hs18B+32UvzeL7EfPPThkDuoYxqaeNzWhT9pdGnXHObIsHEEhWEMWpZYT7RoYhiGVN4sKs3B466Q88G8K/PDkP9IkN7Iz1rzCAmf77F10WCEy+W1mHfweg8htltqr2EqKSSNuco1kxc6ksA19TpZGFKQrTCvpJwff+YqgZbvttM8tTT5c0rLFVibKgWXl9dQLDI5NSoWLG85xuYziabFhAzljEhIlcrewjQnPG2JIPdCPjGA09JQRnNgLtVExzqGGWC3q32rimghfAFmMmuxwCFtd6EhrP7n49Pikn+aZp86ouacvA1mn3IePGfB5gDn3vkOwjCdNwINloGRy2xsRY2mQHAoyRCV2DVxhW5eUmnJwyej6yx6xUC4AFeVUyJfzK1gSvclTCoA8QxHkuhm18w3WELQ26hn0kcLfuCMexDQkAqbLKQ0aa4sHhljGG4RerELEjFJ+H9wGSr0eBevkxnn9Yj+AAEBlFqaAH6TGe5BUJUrcxwkWEAT4uJRw6HLZjXYjFmShkWEliKEhru1wMFQCNOrbRFVcPSBt1Aqe/GuP0BfzzGz9cW3I+s3JH6d7zPNHtRqA9bJS4IkTpohqK7cI6qzTpkji3uiYyGhZTZXFVE3ukSre9cPlO7S0ip6Bh7JQaNYUOwXblfCEELP1CJYAPUlOVjV22Q3ttPSpzFb4McMBEtXLzVyjNsgKRczcssaTDEjBDutnTgaqiTweDrT8SZi+58DlOCJA4XNTg6pG3U/rroamj7WTT52noPBolRUWR0KhujBe20ZBi/F4jVCOhWJwwKeXeqiF7lu/lHJE1eS3Fgno3UW1k+1QXR85btD3d3fid4lJjHUo3/rYpFQj5gWawRvTge6osbDe2V7rAwld3NSiXRX4eSTWdOV7Ob2nz9QKO9coZJknLzPQ6e/Jt+VZ4Mt1paA0aQ+thM1FdlCP1gXeqa7C620j7KLPb3JG2iqcmK8GUhSfWUbuXEtFXMZXtSe1Z4xlrg/nFGcjYOAWV63RehdtXnTvwHDIGuUElACLra3wCP9kk8z6Kp+485PnsZb+/93UWFVKh32aqSlfTEFNvkIBF4hRgVPUdi4tcKSxQDROKmCafwvL+3tp9tB7gt9AuTstW7Sf33LtyyzO4hcy2GMQ4HlBH3PWx/GOYBBg6pAoCzzBh+wGv0xsrLcud0OmyRYgILA719aT4ChbQnzhreVORUK1hc2QgKVp5pL8dMgHUrY2l5sZyFTOYl0Ie3AKZEO/arV6ApISCJVZEp2ShrijMQSgQTlGz9Ugrm2pkBKqe0ibpmTOQh/rAHv43T2BXA3BtpQT20woXrkiKQLjapGZ7h4vfq1XxZVLDHWtO2+bnFdWc4i8PrmBwRK1beJqSkhdouYDTZVGSrLkdCcwjdqTN9Ne9LNbR39Pzd4rTalZ93HbVhLdXXCCRZVqCgaAr8Gj87tPL7j3n7G9njhWADt4REmjnqRoioDdkDTLdubxDL66vjczXBWQu5CDUuenOC8ThncfkIbtmnAze4Cknur452dvk7onBc28W1MBtfGGvALu64uQCktJsnRNQHztGC4oC8VS1egptGIEFYKWtUI0AhPxo1jBcxcFaEp+t5ZUtEbD9N1IWtncO9q4hTrnohxYZcT1vNB3L5gxTN9tEM7meAeUFsOfSEBcB1tYFzCW0oOLCSvJiuKrd7NS6xIm6O/wtfvYGmRkRmJ6hTbcPrV96dZKYc0uxRmqTod/YN1QnUYBQWZV30MiB71ndge0w0Xx02zV7k1tlVu8PnhVE2BQEaXppK1/ZeMwaq2H9yW2jO5kVAjeIOQT+/dMN+LKlBy5ITd6ND6GVkyUH1cn9l0p1t/C0VvcJqUhZXU5qo/0JIKKDVEucSEAUUmwEouWEXvoJNWkLHblbNbhmvkXKnKyVmaxl+irxlHkwL/5Ra5I9SgxJVTBQwMcrFwu1tfhmdkFyAQSgwNckiY0Rl64mycf2XBdaxDnwBMstw5oUfzL21uQzxDz3RzFe8C1Mzaaqzi4Zh7UjbSzv1cbSQZQhONLcDC/ev/rS5iHAw5nPly+VltPTfd2eU7+qmzN5wlkJo6boBQ0yCh8oXmKKtEACgJtluJp6ALTs2NRoia7QexrLVMRfjfgkEFUqJmHYye/+FBiF7+HvZiIxE+IYVmdEJUI2HdQDDFyz1g8Z3yjfb5NpU4wr89e8J0aYqZCzVD1CY204dedr+ZHwA3nr3NjNrgs3e42TyL+kKQnl7yaf1wD+jxs3TCvVV1rHrbdjPpCNr9wSoULmayuo1SpVxEr1WkYulKtpEAt0bw7Hu4mSZkGtVhIqiLGUikkCcBAAcijrypeONwGX9CUfWlmqmUhN1zXFvviwT8YbCvtLxD3UHczdsNP5nHTQVGI/JR2+t0U7JK0P5LTmTAONui5V9bSYqPqsW6fzUlGcxfUq4IQyFqaxDSNnZi9TJQNrFMQgLI3JJoR78LIpAFfQe8H8NgFSaHZPhjiWRkW09lBpfpXIt4Gx/BH8ouFFgpmqEdCqrzRly+h5gK5+fVQAX6JqVtWopemdzmFeg0jjeBvZice2UQGpMHlDy2mBM91p0soCroBx1/GyD4tOEsKoFHT7ndb0n3iHBclEgKorBFnGKUFWMEzGd9xpIFOV4sTu+q6vH108hVZa1xwrwxOfZBVqPOVBm28UF7/OGhEQq7OKAZhmjNNcE+hyQTGEZTdRdZT1hul0ZzWEX3bNlsk7GdfaTeiWevtnKtqfMOVk1tegShNkeVSCRADUxkJMZlILhLjULCWm7Xl5ycQdqVJylBWKGpQRQvWGHBQH0bA2BLh3lzx1iAGLGksqHCV0bDyP0tzXVOUs3izGtqJIkq1NU70al0hAq+uiATV/cRFplJfYqQkspiM7UPViNi3jWxLH3z/JBY9ZuzfnIJFlu4uzTHYuY9ulEtHp5wS9QlvNZykyhU3FUrzVSjCNuRKm4rDMre2zU93UGNwL6RmwiyHvdrRl+bFONeLAdUsNvRUnTtXp/6mdEZrhQk8ypNZtbJYfS+mJ7TmOLQTYE43poPYIKwi//Qino7vU69FW3xQKX6mieISp2kuwQ8hwKOuRoRcfYcbSFzpdnVdPiNe4JHOgdMFTsco7JUmeJr9GX4pwBzsT+VMH3cF7JZoApHN8GUTX8mlDdsXuUj/tTm9ctLy9XEoYrUnanfZ1OvSVh2GrhngG7h2b22A8MMjgOB7bvhyRiFOuALGSfH9Fd1+DySdEwIph4gg9j4yBh/gIzpdNp8S/QXy07fKAqQpu/dOuGm1hZdocSc5cThCbiapoAv6jjzFDot/8ocOukugMCyVJPDrJtAopaTE4eaC+igv68rTrlyDpI/UFnyK0oBbUpttnn9b/mMxvM5kUNSWAmYENQVUB0A5kArSLFgF+v1Iq0AuUARuBOiAbdB1YAxQDfUAnsBrYACwGHQCzgAqgA2gE5gHdwCpgAFgIbAU2BWwHtgDbgG+CHWCQ2XlgP7PnwEpwABxiDgYcZv4D1oOl4ARzNHASdDHtwHnmjOAiOAcuME3AJXCNuRy4FX49sA4YAsPMbcFd5g6gB7SCR8z9wGPQr/0E6Mbv1Ay6+wnqFd4CevABGsLbQG+/gCPm7wEd/gCngH71h9OAp0D/3n8oAqoVXhJQDlSZXAJUgnerTQdqAF+ABMxVeBZQCBSAOcw7wFv35gfkBN4ErADfPLAWfOcbRAcgUAhoMFjobQdeXB7mKJfzlL+39BHAl+uXXvl/so5N/2RJKH9Sj7U5/fmhxseQzubqP3GyV3+yNnYcHb818XdREE6URRU0WPiFd/AeFsudUi8d0iU9clrOeDmvwh/o5/j3I+uDlACCuSAn/MtJnGbQK5KQllx0mWyGALVe/0NHdV5P6S3mr2ab+WTKzRv7k31i7dbLI3zgMT0qi2XGJmM/4qcTH02OJJk8mxxv7pOxZi5nVO+uXE++5Fv+Yx5b48U9xfcOczlXcPvO4MKlT5b+tQ+tuuv31P/sfKK5o+ltzbT/2rZ2/tWxdnO9W72S/o/9N5O/Tm6fLE0+TSV+8sNbXT16oe9i+pq/1lSu2V0rvLzm8quXT1xJy36Q8+DUZC7kKtf+3wTt3+9u8Jn+3Y2/nf3vQ3mz6dxffR1zIOSnYSML3sXCxcriT27//tLS8v/Ie5LPW21f/bWCduFHC+8WLhedXv89qlOx70Z3YzzaWNKxZPPmNyX/714rdS/tVPpqx8mMNptWbsr+FKedkpVby/m5k3l2ZbvKPpXh+ZdVn6iaVRAKSSFYeJ1wi/OFmlY1FyqFlX+te6L6/1M718+snaSNq7fqUuOPmz7YdKn1VvOXrFNtmL0QE7C72H5sPKdJ6yLiibarbQvtR9v7tP9DZkmW9Mkf8ydSN6ljtID+GS7VVeYxeC15vXhDuC2O5Eyuw33H/wX/C4KHBFHwBYfwSsgV6/t90ZD8jPxnknsURvk36eelRWpDlbVPyWq0P5V/Ux7X/2ZRT0WvsW7cNd5R/F8JytnK5UqbmZhr1suWaF232tahdU9VUq2orhW9J//ZTd2+2cZKRJ1qsiurIt89p76Ke843onpSvvxW/hFv4V/zNaJGxERCpIkMkSV5sn04J5nLOXN/cN58M77FhFwvs/VRpThMfU28hHnkBZWKr54RT6h3tX8vLZNeN/8snzE+biwZP2h+2XzTes76Bf1LesLJGDONa+af3H+2Br2u98D+tX8nmA+3w990RuJ/m5qXWpDKia/Hf5w+mv6lx8n9X1ea6unfujvZd2XLLaXdbnfb/fawPW5POjc6XEfoSJ2d7i91Z3vfVXq6PFeBDctGh6O36k+vbU6+0Jzf3LP+S+uP1rsba6eg81dTxVsT7i+5X7j/5+nxaDxaD7WzsPslb937ZW/K96Ozl2aUfmHgSOB/ULBUH7pywo/8Yu2X80+iumggmr9Tv2uIfyauENNH7xEqkRJbxJ+RWEG+YFCwST5PJol/UVgvfEg8SFSTv02+k/rctavUakpIM9fv03/Oo6irWTH3H4s+5HZz+7n7+W/dXimyd36VXOX5yn+8/302NfeT6t99+Hn+9/N7Cx97Ip8WycyaeM32n5fm6OW6L8pJ77/t1m/ojXI4q7U/EMXqXibzWQOct33KO7p78qQWcK2sTGv+kkDXjNWp2M49I1Q6L/q2WHP2BWDFnVwzBqFVqTVW48phjWgW1QgAq2uyBhveX4cDCDiwzY8u4tMErObflbyTvAPyR09/8Vrtj228o58s7AB/nZZNdnyWpFfGOpHuAfL1l505dxgxrAWyDzLMOTeGYxpvdRXF6aVDVqrJGoBWyAWyvl7CNcAdbjReNhsNNjaI79ewimqdXvT1XIQQYG6A9LGwTo9MAmeLkxvgZkmGleW0z8L4QB4biCACM6Q6Nwwi7OqW2+6F7+lOABJYISyTmeKVOsaMDKH829PqIqiLLw7X2FLIcZsqmBjpU8aYsnaVU9rmU2b++BFIPTHEGGm6HegMNZ4tcybgY4uhCgmKaDXkzOC76dpWsK5sS4DmV3nIm/OcndbuBmHXfJJvsTD+sdRuKFZ2T222Pgk0olNlHwdc6GaACrecfCB0VwvmHD5TsaSkXlNAr3mngRIfgBxbDY9VVFEFpwHK7VYqVwELH0G33Wss5cPV1bCslXILKfg2kHP+5RrUnzUSh50HK+9mhw+RztN6hTnR/xx32e/84W71GRAWuNBih/zhvlsGCENv0CWHUFbKHqtMJRP0DHYmvU23MNgfPIPzSIB66UhwB4TY6wYiwMKof9r1UcqjlqO+UvNJebyXL9okq0QcOxkmOrHV0NzS7zgb3ckuv976+mx7Y3LTnPnCfrNBJCWLbgHtXWurUmpJdvGoMwoqk+sm+BEgOUqR8UW62ZzHXHreFSGOx14FDUX7mkCJV7Nn0dVPNN/mu3d7xD7XEzdse8+Wzfjc37VU5IZ4LMe7kiuCMRhfJ17Q5pJA4mriTy9/nV5OQLhf20HsYMeyAzf19y6ZmcJig9+S6m8UNlRAndgQN+DZ05p5gshG6UbPSKFmKoKGvl2rA/fIHzQtcaKNkkSgCpWwBEaRe4I9m0dz/naSkFW/pwPUTGHwvETmjRclZYYI5U3VYsUcT8Q9XNxziuUgIGikhWj8/AIafZRmBY0jM7GUnbmFLFBkP5r6QLWpAdb272zUNGoBTDNqR3eN4RaowxPBagWjUW1Rcch/v7zyUuNtMhs23/gIkMQfsdI4V+a80am/guRmhiA/+R1MghTO3rpSA4EMWng7xLo+Y0MGC68IMN/7WpLVtlIHgT7t98OjKDb2GhTI4MyGe0s91U8bi+EWAX2ShzuIgKc4roCd7Mz33B7K+3X3PWnvXZu1EDjwb7N8QIvLJtjNUItzFCpqGs5IBcCC9itFn9xmXf4/dIhriZBnqyUZ1pFzGkHh1yudZljbjoBWtUMkXOaEX7xdbErxhW9KFBMPjPEyJ7HziGbku9OpneY97bc2JDJDIapthguGa/1MpaP+4q+szFdH5Op3URYsmnPCFAns1NnkU3GkZ09orWn62jd3Lu72nlLxexunYJG1Ya8AuDOBmsAZlKb4/XkH5Xpla0DjDlGIHJYzgkgUrKTZ74x0Kd61xe7T0lZrdvkrD3U59jYXwxBxrFCQIKJg9OMIyYq8+ilkjKGQk6lgVrxzpGDX1Sha1RJdSeo6jiKtrTbBO9oz/FZp3VAju6YpQBpSqYoSuhJgX+8vamqDCxuq3Aa8TChmHDlW/Youh4ZKXDf1lZhMVid1UuHblWYO0EHxUEIfIzZGHMoGdK9ay//aOOlXV4sikKS6wvapZIvQAU9hCQ5I5NyQkSWnpIo0tlaMxK5ZFVdtk3bYIoC4fJFzmvdNv9gpmmwt+dG/EdyAIpAFWqNSks3wOXndgtrphouiZyv3FAuQx31ihYZwBAvZMDSRVFm17TQTH/P/WHSwKtenDHx1uxjtpw6rVmEZ6ZUpPz9TRfahB/1nKNgXb+5ijOxxmdQGTwFtqiLtw7QoiRoC9fzx5xkXvEwt3EY6bKk0J20cyD6NVWjQlzEbBqtIfp3gmBgtwwvvaHv2qzDjxPTH3ITZWDUBpKukCof8jlRCyaAuyiJx+Lys5+vxdq0EXOSEWfW68pSK9UefUzldBRweqbbr8wMxwoBsnE733ZEKZkBOWKuktOnFjh+pUS1Y9g2jotCE08QyTwDdhrM21PkrQwq6GzGrk5RQkMRXUxGAwkogvxpfw86oxs00akFFXmWXXMUfqHhMRO15Rr/TPzXHsDfA6bN+XMFHmvXa683j6P1xLxiLyb4os9qZ01fSVCpjTriaUQj1chJ8+gmzMxT8Dake7+S+C9EbQjILkBE/bDlakqLfbu2v7gABfwBPTy2Zaf83aFabWmtZmf4ddn6M79SPrpbdU1GqSYnovxh+/gRbAib0/300uO+LdzdhW6vvXEZWi6qiGYRRVhoB94lK9b90rkT99iZQHjZ2K3PW86SXaE7bG8cuMMdG+gAu7135ygfo+hZxZDprXeEzeeufat9iVH+48h2ZNdkXj9K5Ao/W7n+jnXz9f7CmxDivEBeENMmyWUkXMBvNOJILtTShNjKyTlsKARilp8Z7FwgJBwesZEEOMYW6TRjUoYW8KWFWoGs4YanKfPYmWrdl9UkKB5TYXunDiVO+ZSKIKaoMMbvKo6bxjAuoI37cXIKhmt5P6Q9qMWL6iyOoAawnxEN6nV06fP8mQKfuCwOYKGCSopywHvvJcBJSb+OxRA3427Z/+Lzp2kXNNRO7thsKgX707IBog2dnyO6M/8PfLged30mDZafp5iYwA1eHTejpTeIraIVX87JfEWS3DYKgSm5K3d2OomhGDMgL9mGtQazxO+u810A4e1eqzq9+qe4OXz8QevXVkSPmcoXE62UxX90GFjFqfrg0li9ve2NoZ2XBQn58TAj1+BARufLyUqCKgEBIBLFTjx17Sm+a+CP8wcC1mnI6IaaVWHd9cHwsvk7PPx+qHeZ3Xtrczo+pQLABWjPv2OK3TVk5kaP/sX576na9XV0v1SOVU6vWqmDskY6cGpmhU4fmAXQKihyg1qms+B4ChJwrDZrLX/M1maLaLYacT+p7w8rveoc1rzNPOYmHpdMLRbIqMxogW16Ka7B+5/71L6L92VKnqDGlPi/jwHUvh3NaW7PN/APqrOrYEmAZcGj5ch5qgD6eiyTW/fHLiiY1JQ2DibypqaxqOqnD0FOloD8BDbXjjFhJ7lGNgtTgYYyoyuqaEb/ypl5pUpLECwWPdLPrBM+p7Fwq8FMvfcG3spNUq5IC2B7egfTBC0kTdnX3hxcQaSzbjjsXf/+Gloz27t4e7eoxF67aDBFqyvKVKxN9bdUtK99p8tUR1xIey7oAoEJH15ne3/y4BvnZoTNW6finnjU7vVvRo0zoVv9ucpd8dSAK3pr03AdPdt1KK6X/RMa1PVu2sE5TfNPo+68rGiAr/sCE/7LYQby4OOHH0AVfjzOwLg8YaVxtzPlyLGBaEEoTTPe5tA7r64a7zyMNze6ZMIRJgb8MBPZ8AZLDFVUSl6fixvWZhMF0RwdcMZZimKmqaoruhl/Hv5v2K3Z1eOUw713+Sr3UpEvn0ltUS89vAb0HP7wKu1X1+X1O9drnwsBSugt3LDV+GBXImeTD0bJeQTm1oOB1UhbFWFmH3MWPELhx/wuz+btnP6uoW5fZhY3o3uzeY6ulYeA2PX2rAIWi2F9GrVLFbGdv/Fxxpr+7e2/Nf/vW5nBPPgpBPk8WEsSXoH1aIKiWJXzqcoUhuDMr8zIbISoDPrmA6+MZeRQNuuvakN8qPLaf/HuDq/K0RAX0lx+8uogs1MJJSgHRQu99uq3NOUD0ItbUPTcS1uX01BdzigMHVitX9XNYx41CgIXKutwetHzSEVtAwhs7DPterpkpywo9NZNN48AK1N0wMsKExpVg3gVyeK6LwSioPpcnMwlR6zekdvXki1JDBDyvKDw6Ro7xrKTI3BMyrBn2cHgo3tT5+3SoAjITSsnmPaPcNQ0SgqcIOdB+cM6vsXFPNc1zVfX0QJwURMdUo/wT+vdb+dZTnyOeelHft+pTaAZKQNiwdvXrMWILoRLAxdNuaxe1xnbXtGNwfVJqXo/EjRUvtjHNvP7Fbu5gR4bXwKlmWqEhBiTKOybd5Jal7FxeyRSK/KcX/oYBB0ZzMd4NraWkZ4WDsT8IHlUj+ps0AnlJUhEcKxK4xAz7fA9Mqy/323BF+srclLgurTazBKF1iy69k9ktv6qPxFvzlrv6xa2NdAVY6snzvaGbr+iIO5McTvoAJLEKU2q3NMcNza5KRBKUSq59Bau5JQ0QeTXv6UEGWDw3AWXhAXvCnmcM1Cl1e7akO7pGvp4Gp9h3MWh2VAaptFa93NcY7rA1WO4ntOTF9Pm9S7VV6cvD16/b/Oy+fdHVoywQdFlT2JYnSo7lN+OzF3Imtbp92BRA4Oz8VJj2uZVMtHwwdBv777xsWcialW07ua7gzzOyoOuLhTB9cXnDLOwDgu5v5Zz5DF8CE4nFoGOsyZi2NGXxjj3ALeCwOdOAFCkJNTTg95sYB3bSf4hOveq2qdFSnBgklFaYMmKu3rNs3fMdxRcHfPu6DcaAR2bnHYlE6S7RFOXjejSTfh39E3i5pETJNRHqKWndh5Ztsx8haUZJL8F6kSqhkUvfIAksaq3w/R2psvFN2+nsulEwABYMPm+fKDnAC0hhadW2mAMNbTBm06h2oypKYjNAgli9s/XCSWsasY/88S5JrN3RQDty4D3ZGjeM6+Hdl7UIMKYtWdDpMWIVGXEp4VoLb8+dKFg721ycae/rnII38MQkaunht94ECG2qbqr1/SZeA3SBi7K12RVDyq0+NWP5v+bMb2/ONfcyKubxPeVB+24nxUeskxZPufbbbZKoFfSf+RLfwidfXWhkfCUFpvMjbgRlgLhvqq65bkZOmn3KaERyPUTG7ik3QS8QpKJ0/9IG2vMcpchSjPJSUYD8ijBy2G47AurTP4wprJKeMUFAORXuz2u/IZthDXyQb9EJmpAtyrisf85FRLpk+PBxkBDqMmZH9MWzB4CA8x0JPYDPecUq4MJ8DH8LDuF2uMW4Fs7B7Kc/XGdmwqo4dB0xci1No7t7mvZnr4k1p/Q7ifxMcrqRCgPRq4TnLD1kiOl/8WMIPfF8DQ0DEJS0KkrB9MOTaJ8GdPLx0WH4Wh4pua4p4hOGX0sO9AY2QooRGGEBe4Bwr121diEct8KBF7mRhE0Ibc1VVasmA1EDBmO5jHHywdz0EEj6H70vx+zZ6tbVLtgc/ZObl2H9Dj/8ElpayMitIIpFNA+st60L6B2GfqYi2SgZM0xR5pxnn3V8RRFEXnx2boLlRUmWn7lYbnVmwR+IDz043+zTIA67coeX1+Am3vtp0ejsLo6qNiBwMR9l0h8xHmL3GjctLz+ZebzYllD87A9xt2B9SrXH6HVH2a2pMi01LF9x7dEiBop/cUeEBxrc6PNoE3uRm8cLwRW//VobKtzhUCmbMY+FhOxbt+3xJjBBqzOpPXYu06YP3NxliCEI6ymusPZElicujdA2syBrOTSTwe/94PpSAf1g5GDbqIqrpHq9hFs+d8NKQNkJZQszmdGl8mK85b/Ve5jcHg2dra/IrqIgMnYbMDNVqQpDqr27GAk2VdFZ5vAUb9Q1zERi2Vh2QNuFVMAz88F+39rd6a8lXacZ9GsAqpKE0RFx0hDqJbqpEx1AVE6+PNj1CErlXtZk1zOdb6p9xVQt0Ve6O8ACCNtqs4MteqrFyPqTAMjh76caeccfpvfuX9o2CxnIpYw30PGMXUBQDgs4Z01AmsM89OKOUkWVvT361G6SGif39qXOsNR1fmfF+k7gZv1OTAu1XXjqCAeCFgUPAnYq0ljrVNGwEOEqzuHTJ6WOqUx1o3warTj3lKiSoNvVlIYUFJbMDcW78kM+Qm/pk+xhbIjhaONP/evFxgaxVVgcv4DFjCB03I0Tl6/EyU8+8+XAQrZFmSBWIoq2dOZpa/fGJSavrtCABcHQjUreTqTrWs9XJkoXKDByBL8FvcaxDtXYKQUIylakwrXC9xNxV2++C3V+GkvdeQ9IWH5Hh56+KJlDK+jywkwIWLaOa2i3NgYYvi5BtiD+ehK7gli5BgM1x4xwzNdshJCjFrbfogdqHVVWJYfLKJwpWSTUvYiPQCN5YZ2EGlMkBmzb6/U5dJCfLzWM5zz3cvcf7zFVbdAaFzMukJf40bt47edOoDqUIm7ZnrsYDgFMYQwnh7THjrNz3Pq+FlN2b+w3G4vjM2CZo9nzlmgAnqJUQqfHm2WMCOV4AbMiabvp8rq9tbOes/TA8KyYWSBYO6Nsylv74AFl6k7wLpOf8x/uv7UZjv2wgNJgujR5whx3NxJ3JG4V2lhCCZH8yooflYoTvHg5n6afuTTTKuan8xdn8YvNK4Pzh7NHK9/NN/bbfDlZXSFr49/1kaAwrlaaqE7uGvQFrZKj1YoldILCCnD1RRY73+fFbPyR1X9m1FIC71PVSPfMnBLA3dj2/JF8y7gz9K9DW9+uHkm/Gw8pP4oCnvSNws9A1SJ15rJyqnurNtKGhjEEZf3mkII2Hr9YTkn/RvjtD5Vv6K+IXXjiWKrttggIAEmURBH5A3FPuA76h7TEZE0xqFsYpC1x3ItdiiorOgI6X0dsQOUpJ5NkFBe3poSZgMTJBtvP4ZYMm8uldjUJ1Ax4wPFoKmI8H0tAEhc5LhCxGBtSQHK0deSKe1YTHGwdsYYAUvNPoyMQk7wXtqJqkjsLLqcOUXUFo2rtqMdBt15z2D60skWACbwX9TSR5ao1sfP8VvZevT4RsyN2O+309tod/3Z4p/SGGCidIk1fn3fKoWC5vgGDoSjKoh6c5vRGGhYuJPGXz+FMOUNi9PNKvTi788JMa+K+/bw+BdRgU27Cs2dMYwfV9dH6RAbtN1+Ey6fcdXxEQ5KYzgf/Ickt7xmtJA1KrX4TWLlQjaFb4cUYLUaOVwf5NB+t3Mx6ZkUsewupOVLamm+qqmyUtUIkKbRIRDwwo42AFlR7dScUlpeIPiUCNl7ulOHuhlfeeBa5ft0UK2tyPp14fJL8xDwmneLmu76Lapw1GtsCaA0ND5ok3KvWohN54VucbbF7xRofQACjsOdJ58foyJrkG1ybhpkRMw0M5T50y9hwkbdCVM66tbPViscnoP7k2sBiTNeB4/k7O3vUX9lw3KyvE1EyyoodrU5wu5Oi60uKxy/RBUEr0so3Qu1cQTJZWmDbYSMHWDIrydDXP9jPov3IyT8YRq2zNJQtlfAFx6y69QuSDUTu5SdYxAKisPcTI6m7dl6Tr10APJzqVPS4Dqkx6Ux4tTEfz42X0nn6QVbDirHOcI06oRv6LoD1ZYMGpkA0BhpTfO/m7b4XSef2b2UCmjneNU1MbQtj2637FbfUwQRe8aTVciiAkX675rSunp8fceYtU8ROc17lX/Ue+kzYH2lN+bKpcTtzhDeUoqkAF/otmfz5B9fzKkrVKtj7gZOGrr/3aHqpShWSuXolbZNzvuAuoIp6uuoAA88PLBiLhV4ZBYJjInMNa9k4yUbT14hN7dJp0ajijLgaOHw5mM0/3/uN1S3jTkuw0Vs/mXx0HFjsSb1uiB0lj8MzNXV13zXPktUNOKq/OJz2Jf984eQLJ1U6F6/SdXcDlrtICH6IffTDx1+6SNWNOqTOt6X+ir8LHFBS436tbWx/cdQmNanO1dWytEWINZCE5AXXjrVN77Bt73bSwsiL9XGmRXFPfcPQSqns0NRteq6ndpaStkZjX8kpsjKShv1m9KGRwbBE7LnIVGiZDoSIDNpGW1ww18wFbSNfSNerFb3dzAs40MerlmyBUsTfvhuPha64LTQJ4EZO4eQjkHDSUcon63FqderCH1YC3uFVJjBaAZYPwKPN5R296Q0yrF+0ZQKcyOq+AYVSXbRRPac107KHd4b/9eI2YFPwl59t0I18dwou3bOAWVNlFdqB1NUQ8MuaYANiLfhl0ahf6zypwIxlOI5djuSuMpGbVibLGK8PascMxUdFa4P2rUe9rd2jJtNeJw7dCWlX0nIKChCW+IYUA+FjFwxepmkvyyH6VWFc8ks20CPZvgy7O7z4KqqjN6sFVazVqv51TpEMR/O11GjtAflsosO88vuyXVYVr4Bj7We4y/yV7MjKhdxa2myVwyQvBspl7tsTLRIjs1XnBnAtr3HwgmTXCjS1qRG3D6QkE1t6Q+X35eZLk/ZJY4OaUF1qQ+wLRW6PWqQbLMvjvMAIAPXpqkikotSOfGAmedtBi08c3Dk+iUgaxcq3jpEj2uPFyIUthB5xDq7Zn2zLxXh2ynrNh7ydLGvY7rb0Vx1bVmji4dz0ym/9ZPN/E8On5zuoYf5LtN+0C6CCOfRKqEKGkQQeRckSFxeUT/ttIuyKxL/x9f0MlzpXbv04Ryr8sSNpRTd1WhZ66Ywv0N6qDtnyhg1I1QlH7N+tYAn6hLgsN2F0Sv3f9/Ce0nyCF8qdslgCTCKvqCxvsx12RK6bHHR48Y57FiKuJudd6eCZT7Eb/iBPx7NsrokQe75t9VwcSF87Htv7JiYkKGVAdQomKGGkYkhCmC5LYoqos/f9QdjKYuqinOy3hPRmC3cI+zlnLXkmjRsT4U+EttZSvOP1/eKad+9hg+o4J5+1XRDXrpCAVaOW2Rh2ymTT8ZRUS8qx4Cx3QvOn35q6+NRRhxhbGWjQ5w/kHBrtLq8PClK5XCBV0KxMr0ABxgxFicXOdZWvFe9pDk+V3BKNATHwkpny049pZnNn5tgfOtBlLF181v4axl9ImuWmCdWODqK9pjZ6efqRo7UD42ZtrT23HXL3dFdqPg5oDI8te7jaamnT1I5DvWvvLLb8IK8YmDOjjFAmn+q03xRTQIaIcTNzXCYm8bS21dZ7ipcSe3XsW0CKl3fKsLdXNTB0d+yw+0Q6gUt5VVbBmcTuDiMJB5LPspXKmyIWL+fXON7AgVQr4YrYsMvA3qGFMU2WDaUpmHOKQ+FaxNhqI0lo0/O6diuZCLeNTaCCJLosyGBIQdeB663BvWRxq69Yh7je28Y5/VX4oBgOQpJe9nUYJ4yn9Gi7Ssslr2kBK6P9XkgANE2VotqzrfiZBAZkTcbwLyCpFmjMyv5OkKpNy2lGKSaqphmbx5lTMFXDxHFu3zdHgWoU+B2kblXXulum7mpxHIqpSmISSRHvP17t5Lb7u8Ixd3xzX2/IJkPBCC7xhkw22TreDnYzf4vECuV2VuM0siTZOwqnFQ2TlAVRCO+Wu2oMiAgdkR8Tw0DqSawYuNcRsXqgD0oT/wUglYrBbDU6/4a47qpgqbjdUVrXPaCRgL4YBnizDcJJS5q6pn3htSqFPPR62MCOqFpyYEQlb+to83XvAp+GqVY4jI1AagF1Yr4tOECq+9zeBlybYxXn0FLRVuP6PM6k6ycCUEQfiC2YiakkRSuB443DJmiXXHgqNhSkHftxLpenRAQLomQg6BYUig5uu3b1JXMH33LCNUBAI3z4sUtzcY6ONQv7uPjzuAHEliSDB4YD2T7rDaB26Tgmapm1VtiOxE7QVUBq0FVDhNA8I1p715TXm7RRC+TNqwKH/VJNkcLaBRxc/pzff7YMOKqm1GBA9syyDYafHoBJzoLwWWPzgJHMSG/qKZ4R5HtG2L+rzuiKYVqW3mnQAc+TicdMIr8cJbioXv1v0dv/elwmJ4Wm6fCaLmrZdGTvwtCfq5Sqy8rp3l7PrlOaOZOPO2oYNiXXb+Bx2KN71BBIgfgAxNX5POQyV5fOnPw92vK8Y8sZt0JUqlSYY5upmRnWHv88y508uMB/3KN7KuDyjZIPKzo1KEqUa2m5c/Fo50i/h6llp1cbE1k0U0mWgWuSfkJImbVOOJTdD6goYbHsSu2KKdl531VV6Wwm1cMqwAr9ehPmRawm/tGosz05vH/DU1hoYXLe6ziMizY0ioRkRXXKV16w15kStlArbC24Xkm0aN42vmrYjMdWPslQyvmxxIFTUlhc0YArL/Zh+K7xnLU7bDbC56/HldpPTe7QU5TXY13Z8Q3OJ55sGI7VTpvDdmfQ60y2A1UxCJW3b6yXdzhQjbf32jJIG57ZZGC+6tdczlrAFg1oTk/opx9NSFpdw+pbt0q67fmyhymWhV8WF+bW2lUtU24IQGIUCbIcrqODh6Wj9mZg5KMEA4GpahykOVkQUZQqlJMy2APT4SEeqbGcGp640qzKLCXbiiH4VkXMeTZwhSCPOWuyoPmCWlc0VOOmeE2tY1hRDkv9gWmGoeAJjed93QLO5l87mIEtckVo2AmuWgpt9YbeIDzuf3fJn8ZhGKkdhWTiY8VQcD5v3ZC9FjomOk1HY8NKw3G4UJIsnk2F2Ny0XrV95IH9WuyEg/jdZwVhtRglP/n8zPL5miW5PboIPAbdc9hM/CAa1gARc+X/i3H+rFG9JoGRuI6w/m0gwUhwz4aX1WyPUS/+tfa+GFGlSIhhqqxZCbLTui9EAAoqZoZKvMY5VTs7kbf8Lu63GsPbh73x4WA8frfX4zdWVcuMybVmTEE856aKL+68HNJtPXL7UXp48HuVuGN39sxR0NKDRq1ZcVcMXQP3q5/f+7B3c/HSDv1TVvO43qKIb3M1jrt50/UWa6KBabJD7O4Mqm0nY6qWbPON0oZLll+ZXcWLa8ClmqcHzxz5oqPUmfc+KPbyFcKJrYE6iIeG6YdDp2WNlbaWZiM7as8akLGwpL+3Iy7HGwlhoAQa8YSjDa8MeRuIvEAEOIa6SRWxQrWkxJWfM1cqVzvjmFMfaz9KpWMfd3rxiNM40HuE7OywYtbGNiwOc+oQ1VOtsNFZ73C3uPWlA+bAO5Rj3PDdN5RI7nYf8BmPd+tX13YasgYmzWpXTcvNUggS3sWUgniwl5ZRXK2YnKmM+mvOSDvi+rlb5L31bMaRUE6v6sDEsrsKRNHYLaGg75hsiqG1q3ve/uV+yd0i3ItEKFdNPi+6y9fKkXRNbN4q6DWQbzJLg5Ymq6IkcvFN0SFARZf78KlQqU6TCiLcyJfOy0QPpB/3sTP9c5Zg4a5svueCabLzqstW+lK3y9gadS5rSxkRiU70QGhtuLa90OxkNigzQcIp4T4TAWib1jxodlZq9qUEiS2GS0NLUBZkkFwZT2T/C2RXz9ufCVCx+/+0MbGP74od/O859pzCf9NDClLEe68zaUTOfsNUCv2TmToicctpAMmxWXfgo6ZFDfCYzzANJN/bP34nYjKdrUqR4ZqICen3MFcgLdJGZq/+ggnSZQmy7YNK0sBiogH14ANABw6v2ScEvLzTpqmwbaZVyCLzenewu8qyQFrEn/+YAw593NbBnmtAjWVADV5t6JVZXjEEDGsgNeOrFC0OjP1NS/0kXZA2a2Fmt/N33VyovqABLyJqhVFCgcTzhIcRQpQ4dJwTX4pI0Me/fHUAI9wx8ht29Pe7vnIPKR/cNyWt19ZL652ZBACqSilyhlVZ6yu9+UjxfJKvewoYuhmTBVS/Jjr+IgojSlVsS21gSbfkezvPM8zZmuAhOFTS4PzU6452MbQ19IfSBmWZ66zLQyUCMuQLuIsDUNLHXApno4fCtmrGmVLItQcJyEyFTRUBpWkywpCYrASWp3A6KehFp9bIpVl/u0avFuyo5uS1b2v7AJWo4grbR+PlVgHw4WIC0+BJUuVioVzK8JjcyzGugsf+YlvKMoOicsc80R3sEocACgJMMSTjF9wJLiwk8A+ghXSElxgO7k//AKawQtraGgKnbGMLURD0aqn+VswRfaiPABhbJzGsgnfytJpjX9sgTJJ1LtgIpMuawFNcY+kWdlQDNKopURYQLJCA+38WzYzJCkyIbWlb03HG0l1V0xw3IxlamzC8WfQo25LPwCYBiAlNZfiRKSmhqgwAj101MatYsa9nbFfll1JsJekhPlHGIgtwTR3HZmIaqlM5nB8LWMqTmz+5TZU/6TRADEpyIBU2CpOeX8R+ivLP1s0Tb8DuYLTrIxkhRw2hbPVPdzDG9KBYCGQJwEweYjX3OuwyKcUY9SLWBK/owhNjyRRJ13mZIfbXbikAsSRayPVGA+tLm4wlHap9gLUl5syr69HcKdHhPrlENYsLU/v1JST3f64FITEsRM9w8LJAKcPZq1uyCgziiuXAng534eNoZnvxyEbhgHtEgXLf0QfZcaAAUCEyY1lvfPNTbV3dihG3hbq5HUUW9Of2VO2CfqPY/TEm80E69vu1H9pAyLQjjeQLYUiQ6Zh5KhRCAVl/m8AIKKB26HftmgsFk0aW/tLsXIIuXzGnJ9lp4K+8WR1AO080F03DX79IJjtccXA9bFVBEOtFF6/fYGAsFukraLOXLOyVwB+2MvXJdTpy3AW7cV0HaDjYyUJWLLjROiAsSCm19eyMdYVDEMs9+LwF57C+OrJd+8Bdw3uyosgEZZgpQaR1hRkYuaMQmAJ62rFQDfK6FEu05E50P7Fprm2Ck6meFzOFAf2Ny6xyWdd7rubPSKa/YTBlX8SzBDgdKokYyhSoGkQu5aPT/uizSwSofZLk4H9i667CftfWg06+vdCBy+GGHwhABlXDjpwJSEPVM+xe/qkPdpfZlvLYSLoc7DGsKpkKBha0Jt202wVQTdYpLI8HE9MEVsnwbEblnCplveAAKnPlcmBlPG2lzSAAsQ0vMp1OXoCI7vgRZ6K/TduWapAQ38CvXZXRPd2av+ulvUVeL1qJUujc3Seo3312OERUF/0nAizQOuL5PjDohB5keU82LUzYkdl+ZR8j3uSA51UHf4Y2b9ItwNingP6kCMSDC45rlfaOCxaIvWeKEEEJK0sHH3hF1uAFUoOROO/zaJDmBYNtb9hNd19IbtTMFtUaAnU4l7wFp0Xe3IMvvcjyaY18XoAYcuN+JbFuwVMBRHh83fdZt2/U3WBkRtbb3P4PaRL5pu3WtndX2S6v4VMtQIDJAqntwUkCNwRtOcaCjtxf+RjJbpxo0RV0NpE6xQlHigD+jyf7qlGm8j4pBUvwEyTV6M/KZmBBMPXWTQzz8lQ1uxchHTaY8RpjwKT+FrTANQBzvEKRCbJm1G0w+UDjooXEgdDxvzQJeo2xCfYwYVahIWStBHoBFcs76C6Yuy3tsmN4KxRhA+glqVUcCqldtcKmw1OMadZ9aFuCQ7qH5HWJuCQx7TQagEMTtJ2wNZD+WJZA8t7Raa8Fp19ODpsIqBirqJKagqmhZqgmF9dP9lxFhSEItN3mzV9uolKMu/Eyfkni63UYApyP1eWnr3M47M5x13gHUoDctYO5jkEMEAeWmqgLam1h7NTxuSCor3+/9v0cQqsv8x1YGjfMNgKKQqBYbRoGeo1hqGkwKGmYi0Z0TEUr+ydlkBOc71AYri7CQYoRIuQ1GSIdKg0NNiJt4TQAEum/IkHApsgojjCoELEq//IeATu7QmMVs9KS1ygfzmCVBNRW2HfOiG6um50PdpquZrkd4tXeaF3dB61tt4OAlRF3AvtKD+g972w0I9WuXzwoQ7FMkMXvE/o+mo5cJHAMCa+RvRusSt5x8wzPTX6Nzzqs6qSrjI6TMxCF5dyEgoJ9Gu9cHmLBc/ws9ktTVr7m+BxlxFhcDC7HVuU0hp16iExN2zwOnKHyMGRNRktT5PAY/K3WEQ3abAej2bT9LOVh1lSunF4kBFxyUls19FbwOfAORkI4aLWdZONY+cELzkxrgoDL+DblQ8uf8LjZDdPQBYOQd3QepgrcvxzAJaewkzK+u9SNjhLMBZsBqrJCynppqAAWnMDf42TzCeGMZz0ZH4E7321Z1Z3pYVB84N3y9X2SztzWkgI/kzjnPTruSyi3JpPLY8XdTsHh9IcvQ+/WrSBFNdsbzMf8tY1tBm2odi/Ff34e7Utjv1qE3QTHY8Kj7tjP7A8iYjSK82MHv1FcOH++3lrYfe8mEFJ1MYT5RLhpoxWLVfc6G4SyG46dCeKb6fx756NLr7pB/AaDAdAfi3aynwE4TPXtCTxUH8nyu2ij3rZUhwC0/SQoMay8IpnsLpsxP1q7mITcZTkE7b0BT8W1w7kYngYJzdXVULolhJPbbyIooOtXfhM5RevEZMDsE+au5fdAc1f+jTqMRDwZDa/9XjZRseAZSeYl5uT2EUmyjI3AhZp+YW7rh35Y+GI58R6K1hrPe50hWTxw2DQonPxuToOAanmUdjoZoNoBxnfuZOzycGAxamZ8oJn7+7e/3bA+On/sQ5PHQavQibYMDRkAtPDKdQeDu+815nHy+fOqWtrs7wIOUIKvK8iBgN121eJgyNSCvxuPsn3GmA/EuHhuDuqCpFQwZbbJekisy451w0iutQDdAsnILENknnVK3Vcm6uouLbHdTkmEtgZTnnoolQ1z24ohJ2cUFeM6hro8BUHxD9StgU9kXCpsegzUqIspqENvecyMyb0s7BA6r1xHJb78s7UAjJFWUYIJUdeQAfXBqMHkyMEFF+kHfVHKrkBUX5m/AgXHa3s0q87wHkwDoOBooUxySpAprjOgNOsEgqQkIgLo1pIkhTJX7j7hGZb/PTt+igOrm4r+F2hp+NG0C+WzD9d58ID9j5RjZAT5TatjfuPNTgPLj52KkEz8RYHrFOpASSoYTmVVap+ANEyH1bsvQBxwwmotkQT52uU4vjF/5kGCAKmAm/keHObWYvS3yLz/UU6AjIMw69XEK4CBXO6SxLhzH4/2viJCeyC/5udpGqHpSzeSBDztVZ0FRo5OCbM7D3TPj1naVLj5l/sRX6TSBJL95j2MoWA2LW5JJqdGMCinBVG0ugXLwK0oKrzod2JkwWeeWs2+D72DCBJVdDY6M0UZFBUzDX7yjIX0gWi07eMMBjnjphGwboh49tMgDt2zgiKKCmxTjoowq04/UNuptuk/DyHrbBD1xazXPXH2zlNDxtsVA256IpMhWn4NiCaTra2MO9gdq85Xbx8bOXx46Fn9rmXNOON+rwlTOcuwRwAkMEEAtmIBZSJeEOnNURQaj69sWVZoZ5UxnitWOzDa2R9ICAhYJuxIZkwQZaLAiLrY7DIyrExoXbUBZBh8pX5e02O8aEYUZBuu3tZgADAYH0B3NQnFFw2sn7L9dJlN0ppUY8LouUhn49BsUaUgArS+zUlUwxhOhwFdLjVjMLuZGcMLYQOKqCP1x/FkGGemwlhBCyAvFHQeSyyZU/HFuCdiwT2RQjI3MklP0FA6FcCYd6Ry/05v6DGFxuna22LmrtHLFSismVIAp7V4DdVgK7e7T/UAM8YsHvgboNoP1JSqUw/bTQWp+s2bz0J/tF9eQm3uqsC2afRvV9losn6nYoOcXzqchwezOzeHckzhWgodhmarnrkjAlHbzpYlk+1Sj9Qp1EBhyOhBCdRAayM0xG+GGJLWcfERSjOvJ2jbVKspL7gxq4Ck1rQDRpf7tPh9rwcI4nmtD9sr1VLoX0c+cWYxNn8I8Gb8KEfNlp7qXx2UC9CgStSAgo5wcB5VVHR2cmhK97wo/9RCDkDzozuV5aFoRT29n6yL3cfaxDuOZ1re6sfOI0ZmPuChTiuLHDotTRSp/CuNEWd+yxXA+lrkbz+1ZYGYBtyzZbsCR193ldnXtff55VYg7uuLfZgPKGtrRZm9i568IrR8yCR1y4YEO9ppwRQjtD30vQ+GC0Kzx74f2EKys8+2zpzmTOhFqmLMmU601O61CMgUD3T0GsKkL2951hdOVRGzpb8go4vCaxKG3LQJLGCe5oQssggKd4lr1gZ+oZ7j5he+vPbuO9aTcSZKV8pFxnqJu0BIYMy+A9wcXNzdJzErIBb2XHussr5Ufhp61kExc+c1W/NtMMTOxuao7b9/izU7FRq/OT9CfQ/T6M7HBVxeWv7m7gMrgKY/W28yl+Z+D4PmEyL5YXgtnkH0q4VSaeYjH1HcgSLJkym3899Ce355DvFaU/UnKPGpbhsVYfT0r34ztoup35dCYHRFfWa/JUq2xgRaKnmIzdoy7O1UeAvl5WIJ6hMyZZMNbuCLjctXRoYDbACzB8tmnmNaN6+vgloCrwbHR76sHwSe9pRiObYuwD5p7B3cfa2+vOQHRJnt/Sxceaa+bjF7DrM3d7jnwZ+/EEJbPlE/TVvYFsFI/9bFngX8lZcrHXj7/qWwBwlqkhIVR+pmlrbF9iVSBt4eqtGGWQbTGnq8oH21HGLDls+bbV56mtT4RrOcZEGzfH+1DQtyHaN0xvLMbjBsTjoWkJM6NaCeqpVs9P0/lX2d7krm+6ZY8mnzJr8c38DeMbffwUblm90UYwBX1TANKT/DA9tcz/2X136bl9GAnvZ3fWn3DSNw/AbqsFzNl9nW3O+n0RNsp4fh8SmpCcNW2gmA3a7lk2XwWM9LItQj6cg3jkGz5h1VQXv++NlBtxuji2pTPW0yZkCibTC3Hp5PixpVk+GwFikRoBiziSY6PSd63BQVE9WDevhr1cA4aPsWpbiEyQnrfIKbAmlYTMUKWK/AGSKSx880ndMvq9SSs+89GwBHbkzGvQ39gdQDwK4hBKetCJWf9L98O4GAdfYSEi5i4Fq/EEmYqcxCOUtZPb8Nuf7XF+1IwiMOL8MIHxbKKAUhY8zRCLQ19lroe5LQERkspdTtCWOPJc4GBNoRUbKRnALaQ5VA7fmy+BhWz234u9edKeS51+OH/eKszKihl7mFTFSSiBXKngrtxx+k7NAp1XMuGhjYURyYAGrR72hIhrFqfXdVB9lAdpCFDYBtyYYwy10Sg+oXiAtB4qgTy+aneAW3MNPInKp2JCH32L63DfW0IluIMw1Fh46+YVS7IB29u8JeL9zku8nnaAGSVMcSCtZm5RdqF9IU+SJJdJ3pTznxCmuwxtU8dB3UsT8BJnljUkJURJbuRnN1SWaImZ3BE5dP+uLae32wtiD/n7UEHS663cd/D9KFB3Sn3Rt9OAuEUTfdQIIyiDCcBm2GHo5nZKgJxWYC4+BIK6IaXt7rP1r8ZWC7dpPg/oPjkO9vG8faZGLMN+LO0KRxufGXvQgl/HlTOGK0ZAeYguiV4JLUxSsXYjC27XoX3UYEPYlMuKY8yMUwMFMFD3M1JLLuto7kyeuJGGyZFxY3eiDitwbzyP25TiQOXWblmLQdsWsDJ8ZdCYxqt7IRPFv8zgTbhCvkl61V6P7Tat6Tzy+PwF80q1gW6k132/Os+/eucynPMQTHZwI8C5V6PH3wx7EtiAC7aTvdI2ALJTajlfXKbx8jzoMPghkJb3twW0L/ypTKdXYRgV2JWBr/4zakFOXAdnxt3h6BU8cfrEzg7LDYvtmXd2e/szIAgu0mga7+bwLkNjZiW6rHB38pPS8dLL/WNS74A08h8M+tOoRCdkGBpVRCFgx85exllrAbKRIPioaTa+4YWjmZ99Co0ckmiqSXrAz7vVKPtVDe8EriZWOCu3dKwU+4KdcfvRJ3qKGsq0CaITqH4uB5WiMF5uBw3Kixq/x3pmkyLVdyavF3wrvSHgH3lGTzouhczo/wPeyvAtejRZyPY5Ft+puesfuzxFWrW2iBqb4UIQUAz240GujF/cS8RCVkr6kTYKitcFjBl5Eo1iU4LI0Fw5wAh0J4wM8D5WoskR0hbpoU0mZOU5kvdhLQ/LEMMtQustuB1m/y7QjMvZcVTxA069hCLzl7QNKoU9iYbnMj92tJinK/iwHNyWpbxTY4tr/PYvAK14JZEGe3Y4SstOcz3xNErGxMdbE+Fkvg+BKTX72Nxg1Sd2CVc+8zavPhr6UcBn3RgIY2TbFJH5sIdLg/lpzYm4P93YKLPg7JI0s2AgqKBpcFJ/iN++SUJFFk4VNp8d7YA3cwzU9xPbYKRYWXKlV4Z6RPC1FGrGaEOM660sZFLQz5E8c9sWYEL738W6GEZ4SfkVihfvfUcYfpH0EW0rjE+5bRq7bjS9S/MaD+n/qt+D8IWCzbdhAMxRi35IHe00AffN7gcTr0f3L8Q9gUt4Dmvz7gIIqmmYIZifZyhwPWFsovnNxOiOQ/a0PvFU5gwmS4phDSpp94QaomtXA50sySQhHAAphEEiipKFNFyD813AF+4v0lE7NYcw5IaFAryoBrDYNODuTA4shv/fPxdSgFltk/fGpmBwhPGNrWjcD1fqLoP6JtIb5qzuauYXMGCBGJnUPNWTe9qQPb35eqEuDGdQoroM2eRWZ3Cdgy+1YYz9V693bf2jSwZv56CSx3/YgJCtU0aOd/voAO24LcfW982yHJ3eIjWBjx3Bg8GnePRRy3yH3/GqicvEma5UBj5mKqa8dEAsotmc59qBmai79hF24oagXt+frQ8t3WaVzwrzGZIW/0Kubgy4cSjEUCs4j23dtvwyvuPnCD57rddzs0gZzHMq5eG++EAYN1sdB6YxMT6H0THMhEKTvzA9fAjqXLoezft63rN7U/bUOm6S1pE66c0/xdhNDjR08ev7YHKiMIIW0WGlo15Bwcj8IQZYOTgUmBMBYRgvgieNL9GBftlVTOkwvBBBgQsknBjPi9uJ0r+tllTABGPBtrItwXTV1uHvDLl3RkbUBf4yHJXgTinxZmrxVQYjDATTgEj5VP6Vs+Gm6c++2zW//5lW27wbqH7+Pu63po4EHlzK4/Qt89fraoQXn8ZAt8DVzaxDM++NzaUeEP+0S3Fzdc4Nv1NjeGn3xXiMFaYnQCQomvDedpQczPVTUgWVhYSx7W03dSRb8ICg8L3G0v5xBGQaBfqZoEmcOiGthpf7UwVsCmLPhc226kneypiQGkdBdlBfiqh9pLVaIJQVcXFX2GDuA/O6A1XWwyMr9SBG9JgGMvmCOWqKopwAQ5rdnyGAUROZNrsWPRsd59jbXjA21pwrV1dv1Cpx5FtHK3MAYF6JAC1+gi3NlhclU07pNlHWYOeY6hXR8a+QuUVk9TwMsFEr8KGAh4rahAFk6T0Q01nek9fLzigx25BYnx+309hxzaancoptvAUqQO4FtvjI+zshJgGaqAHyxzJiK+MMz6VyefmvvDpJGPOpCJOAraPqfTvrJVfn+EHk8CPaGn0BpIRo6+4FeLclrxkm1pCc74rc3LhwyADDUIrJC+DOL3PeNZ0fgdFnFpS3Hg3wGW5XmUPX9n+qAIBSFfHqptUoKTapNDDyy8CMWjp5FFH3trqhDVxpdjAZbn374lopNNpFNsAqTSHst+CwyzjJ3/mihTgrziKpcK9EtODmDt/Q8jbTvahD0Yv8U3hk+UmRt2FjpNF9MfvnNs1H5rrKrHg5ofpkBBRzqrYkjNmYW2BozgF5mFGv6/oHAmWcBOaGaGuq8/XeTaI7+4J8sW+wmxEVKfvvZuGLSph/zbstHbWj7u7IEcbPrXGAuRtGVSgcMSm4rQhbd7ILI6Q1S6+kKuYcSK3Hwqj30g2oxj/EkmAlq+tiAjp5PikoID1/65T0z65/em0GPuCA2oZDXdQXoqjb1erzOTHZzidnBfMsomfGtjbPjBqHbYNvQ2yMNA3rYz+qZbt6pbIM1Zq00kEc0NdYg26G107yr7VuafOLLtOq979sxlFsEGqbCX+bTaq39q8D8iSReKOcATtZLjgnjWrwZQ1jHVkWsGnhZO9h8umI+ACvnKsk/U2LJILd9YzukjgGyDh/4bkzBeo3WDZ5vEBf8PxP6/YonaXADWFvzCDQ9zSG0dfhiMemz0PUZBvoYA9VQL/fXGAr7CZbEPg368ewE2NBhMG+WFKVs8dYoYYstsqSPDQ6z4PCpw9MlMUwhegEqSf6UgvK3OuD/2CTHREVunW8oW8MYFkYKaqklXYAGEiiIhXq7sKq5dBsADO2FQRacsTt8BzUiONMMZwtZ273T3YKvn8+dW8KliFWUdcyJHNIEDnjVrl6mY52w+DGYY1VD99J8/+DCMXt356LNvo16XHhRGMk5N9MySSMaJx0PI/IwlEBlDBb1X2Kzwj119zAYENV/02h2XI2Ua5LwjcSO+KE7K6ObzqzCDHI7CCYNWRlNXjZz94bTIUczxUbxsVeCp64IHm7OTtRJiapcTacSBpUsfe5zMu94hH0bxyVOaiJXGy2qSXjrBEwcn10/BPJxd80jMIH0v+lP5fbdffoMZBSqzGnXgAyA8NTd27g0Zha3TbRWqHFxUmVagN+wKNbQPCnoRJ+zq7imDPZUSLXjBGNzV+7zNNPXln520kZdzEFdJpATOHvPH28jUuodJD7SL7j5OPVdQvbWB5DSpR/At9QMqKmyb/+oyolrL3Dv2AiLoor6P5ipehR33ULaqaDVqL7zNrfk0LokNtjSyN+ncl8c108wjdgyRdz/T2HqVQVOg6kRmJLRf2ZoYzCbjq1uATS3uH/Wz+x0mmhL0sgqeQV87lBD1qL7adPWZUPCgn8nrujqlFBRVZTX915fE2qnV03+UTrpjB3GzrwSODIBYl1ivzhRUJ5KeLkTpswsXq3QI9oJosJmi3ZAZKKoOS8G+ruX5yAcop/X8lo4VgUVYdioaWzEkgVqOkaoxHvTAuWwvZnInG9uHFrBeC54/QkP+HVFUgcsq2OynH/AY6Ev2xBSqGXV9rqxkEnmtbmsmU/wnHl2vmvCN7ZHhj3xTw7auNUAWBrO2nVO33IZRWwcOb6ymiB3NDQyINZgEun+TfSP77x3gMZ3xcL4SkY5BmsT3QM4/Wei4wFt+WG3BK581ZlxIUOIQUxyow1VKSPh1RgOhfpYgYJbFlEfP5IxXax7Bw1xRJwT1ZdqUCKydVEGHfLfQhAFcsz0nZTpGw++kg5YBJFMnGtSTfNVCKEHVkA7yo+nneK3RY91Rya/oW/UGAUI2DDlxRadaMNzYNGo+GmIMq5mFEqoTy3PsHjjN3KUKQpNVuYeTDsJIzJdqTXh+Cm2lvVI5U7CigT+kF1vE/urX8bI+qasd98VwT2mbI3Of2xxwXXFME72XaAmxig3feMLa9Zvu1IaZSRhbTJQW7945VLzh3vjortfBLbWVd8JALlqm2DmMFGDKVqBZZKxs4GK5Rw3amwV5xp26XNzeLDINdwawfs2z0IhFgg/8MKSNUzIwlPPDhQFjaob5AOk+b7uzr2rEMxP1BzcZ+sCFfrNEe/qlIQ06xlaOkOy1fKJprgLBNrKE3RhpobkTpb+VBO2ChM44NkRu7bry1ko+XBV7vfUbhu1CG6sBdsuC76KHGp6fkW0E+2vlRRTOVHVQVvHmSHtKC7o/vX2i1rX6H77qh/fpvIOA9EA9NsTY7aoRMn97f5b/UkX0ddQMa+deNVgqw29eMIZxLR0abJCBeNs04h+eaxKZS4fMW7LAe71Pw5IdNsmi2XUjyA7NmIuCIlTB8f1H5MA6FG++Ihtg+5xN+g3oR+RXzo/zWeKbTdNNB7vZmcf8NTztPZjVqeGGrxF53b7FXHRKALJeseC7m+OuTa9u0DW01h0IFmdLX3erZuXfYdIYkpStDnQor9QxdMcG++bWP0c9dnRz4pr6/4s6NlT8Qk+FuKULNv6ngJ+2KD+e+Nrr/OJk/pxqFBx6jO/XWYUfv6KmW69k5KRCYSqp9tA0zApCZcD2nTuSv/wI2PF8R4MoiHWJAnrx11voSQ+XbXge2MaWj24ayjexJAQaNrBtvP01OX2PUgW3ZhU04AMRzKyTsVDBGsNfWrHABTvSls6ct3S/wV+B6B3WNnqe7Uhu3Ohmzp59+3wiMlxR03wjlb7Z5SHDyqKA+onii3K0/FJPes4AoZAiSJB5aTMcyYg7boagzamjEMFk2GyHFjeGJn/gNEWbp9aB1GRnx/okn5KDmrQqSbKjLxFbaviVf2LSZNpB7pGdTp2f8oXa3XYgAir061vBD68+aBtak3zQsb+i5ZyW2RwAxhBVwvAhwcynwI7lhgZUcnQCrdhk38j9GxRMUCEvAgvYJAMXHmj87+waq0ZexFL3vQ6CTweXti17SqR1Qt+JnDiK4u/+oZA10Rg90UbJtD1hQAixNj/xnb79Rta4eUq4+IBJ/4+4DjMqHZLNu3bDcWynZaWkIQQFXZAVvj3Ha06Y8kj+ejxxPChHDV02ygXD8Qp1o+GNixN5IiwaoqZrtqTiRxpgueesWRJt8Gk2L1WMiYj5vGwKFGa4KPB8omdqf5gH9vP7yADFg/5IFLnTcqLPj/ba1CTeNv6twXVjNRJnH6RjAILcSbs/QaXDv1PSPMAU6KCBpkRKnX0gXqlPswAHmKpokfbcwcS5osF3PwUbwLXBA3rjaw4Te5MrXa32mUflLnTzdDhElbO7Cisow0AnbjpfITV39PS4HkiqfhmfwHGlrVK2jW2GIbBgmu4KMVujtKOrEBNKr9wSgJ2gDeNtF6JCQbmFA9ewORf2/+PWZHJDN/3Tw5APfQ++8vYHPESXJ8I63LgiVrvokwPWH8VWNYAHvfn5qQbXkEPjc2cdO2HTM/Jcllfn1kJzeTTR4sklHkBX4T6OWV3+/bX4BrskLD1Z2TOfCwpMMOU47C3jHT8nfB/DD4G/BYKk1DWDqcGMEaQpm63KI6uzxN9mpGHN+rPVOyv7LFu4TN0RjZeKKVBUNpIXHefCTd3lJOD0boa06rWHWcR6lwBTeYfxHuil8jwJ6bgFSzpRS9mR1LUaXGRe0u8OA1Dfhh9uTrh2vfpBz9A7Wv6gZW/oK27FLIqSRKVZHIK8PLlq8Qbr5oakgPd7DNIEEG6sg07H35nt20ASf6Xag3fvt5wJBZCgpBFWSUDHWdoSm9cIDQS6CawJswwhNfTogPpa0cf6DZczmoL0Am7weL0Zl0Cbcj/f8f+zu9pp2zFbQa817JlAEpzo0EjxFRUc1HZQ00gH6ad4x/kWo6dywtmTAF0Hlk7PF/+kDXKbm8iNgj38HDzRqzRFNtrqWnXIIacUs44NpASndAiyUyVoSq7pngrI5KR2yQuMjGrKaw480aG0UNhFpSD6sQNzgXqNIqRzBIZJ8XgMEGUew9va9gog5TplDYZQ9eMYLSA6kV8vQpmdqXJFYMLO+RzERzblz5DvsmiM5kVIjrZyCQ2nuQpoNzv6RK2o7gduFaGNVL589mYna4jNUUxr9KpDnhIdUP162YTK47eGSMoaluGAT88au3fRrevJRd7VMTe9VR69+8BUqy6QYbiy1OIE0DvQ/dSnUA6TrLLzv1SbWLVz41tbnhyFHN/ocBkcttEtocWesZbDMUt6w7ctoiFukHsa4bCiFWggyhBPR3AEUIzviwgtQgbkx/IvoHEo7/vH65ocAOrVDiPy1DtbqVOKFwRgYmjbillD37qL4YSJKUu0ztFzRO5DvkYJR69JcAzoWttB7LO/SlUdoB/spVe5DuLF7s991Uhgxy0HdQycqnujZPBavN8wW8CCoZlzLwtIcOTJI/txntcxmfWuenAOdnnaL1zOh0VVMEEEuYwmpp51LsB6pHwjKBTNH05PqJl8HYrAhEwpU26P7Jfi1ZccxFtvz7y8CZrGf2D+GNYzmNio8eEmr7E7amRZzVWb9lTgWsT+6zd+TEn/iGoQ0vdbDjwZ+kpq90aTtidsYmASEpgsdP0g6HlhwShNucqqp0GGkQUBTQiKb5JiqHpF9GM1leNA1/ZW0kFhlOPLUJtJFIS69IWjGeX8my+924Xcs3ycyDHcvKRMTLGrLzkSSQLJ5MV/e+5y2rQkLwgBPTCR+DylWH2TzXRd2V0wxW6VUkTiin4HdgM6fImssz5RtzS29OCzNtq1caO5/kPOo1jcLGb9IGkOlylCKVdq5EVPmHFtn1PY1JeZDZ6gZt90GrjNXmPGLVBcsjAy+GHAszUSSkCyx5hux/oK3d28+b2gx2TkIkgQZ4ZJoxV+zvWvLEUnwPwW2Ej6+s5TeZN3GolBtKYFxi1lJYEPgFnLFQMeMATDagTd7kpyQ9wTXc/m3usnbIfZFoNtA6XlRaJcabYbmIy6/omHE40qqqScu5ezn515t9PFC3ABsJ9bKYdU/IgEQqto4dMHsFr4D6Pltr3imj3xPAVBFwMNdxmP2KBPH6q3ZPXNQfWx+5Oj+ov3Sn3UasyL/pf70CFeUZkX/vjWTVcmzp+98Cf9NNrcBoAmpOVViJiONz/yPWNUhDW2ZJhFM/Ag2disRfNvTAZPj8cvf5oe4xlekSH9MC1cRwRgKEmqof6QGxd5TnAibbMMpLU1LoDylrNPJzw6O3UL4sUf/ujvEB2j3+JK/JOTwbJ944MUdLLaQed+49uLxKqqIzUNIr7jB19bTq6hCmywRdMsGr773Go0A1egmsdP/jiUTOzyJ8g6GtxveG1tfJiMC3wizEQgw4pg396ltPVqCGWgvaMKYQNg1O6uYxNdqSWAHT3xwMps3bMIOkX8gojeUArgF1dunLahHqRXGm+lV8Vc/7WFIjmnoPs3eX3F0d1FASSNLvXT6U/s+zpgE2O20RSGZmTqYT2f5U3gfBV4zGgWmXu206Gu623fRONkJxMh2z4ZqFu/pDc+88oCK9CoqEqaeA5L5Kr5DqfsGGbbAKAT5vhNIlz59Zz/FX0A7mCgnstDV/8kMXTjS/nIKwY1NFWPzqKfGYgkTsg6RiLBmvV+vOJ2rw21h0h1MAF/3h9Aw+Lo7yfhcpHro0/Tjar51tLArSF49qhPuvNFZYzlvT9qn0j3mv6dML6nkPUOiI/B0uPj7iAK+Npx0F1rzwxG+d8N3O6KDb8ZSrmuz9s3HOzPtfsRacj0L7nmgwVvRn1jcSQskgwUtmycjqN5PuatpZloAIzvig0a91H3E/QC6LAS/5dvwQQwrhvo2V2x4HPfjOuKxDAYdGDSoOlfdS8Git78GZsExZKgbHxH4d/mzVXmrIQHhpnKUneKYneMPGkLGZp+I+M0PvhqdqiK/AYsia8So+4G1UMVHEh5n1SMvcy9hPXbFhs1mzh3QD9ZJjHbJnI32IPsr0AR5GZ9qlcIoqAoiRSAbwGMy7JkfGsD7ODTA0v9rj/o31vTzp+DOPLJ4Q7pzs/rfzuxpAmW2h1m3uKAtO01vKkkdruTQpPk8fb6hnBJiizJfoXApzEzdV3orejfGZ1BYaE8TOcNJQLR1UUMA2nK3fpDEBnOhkSmuOb3zkGMD/dRO2niw3IVEZzpTrQ90Nh3H6j8B7TZb1f6Cd6EkwH2hEqS3FjVUDLcTKotfCf3NV1USIW46AHICliEXp245KHCRq/KbQAdFNEy4xiARTsUggIKx/BlxvxX97qJGht3A1CYHR7q5PGz2baPuJcQJvRMEp6z6k5hTVgH7iu2U2o/EDlF4dFetxVkBXPMejFZQCRgaUnm0emwH5cFSTlGIZ+oKwrLcV88ezkLabXPuVAiHkXoaF6fAoZWJ7AYnH0xRkMjd5pQOGlKpsmsyV9TtRNle7WowpWNcrlU86/k4xLMJwLss3QncwC55J7NID5qh/roG499ld+15Pq3QQF/CQKaSo4d+WtiiiO5QVMqvZCZ9UVm4IOI+3ycRr5tl2lHdKsldISIdvyGHzXUlIsWI8XM1XECWBHLEPUnRufy4gF5/nNcCHFVAQdqYKEvxOSuWkC+eF1gQFuy8Ng/d2SrUYwtN6haGMPx+gfBNF+2yqD00i2ZvvdiLlp4YzIE6LUFD2YeYfi9BRUI8okA6XRXMRSktLVqIOtu6huqCNZ2xVYNtL7mrD3bzmjn3OR84XUX6uczQK3upbLDjgVv+M4BysMyWzZKJUt5n8vHK3AjELqFGHoBs6nkw8xKMZlDxA+eCBWX5q8Ua5udq+RGnAdloLysCGE5OGteHqbPHjkLXWhyc6LYAEqEtbeBTO38tgb9/RZwMJL+tw7X8eVgCjOp2Ll5IXfWgnqQiKyfSOfEXP/l5aLx9w+/IeCCci+A4kOS6EKMSocELWtOzIidI4FlsUikuQ8pw+45EaL53q/vJJ5h/VblrpnShqyzDqRpCoycWcZrRuEd9MtPb7Umbb+qwkH22B+cRBNtuqKx1xIIf04LwkTxCa9MfNbIjDXA4sUkMGRJwiwypirRGEh8qpUmY3Wsu2PLzFC2SiIHNWW7f5pYn33qxKWIfoIUOoUOOlpjwB4MQA34SRutuGmg88Uzd3eKaPvmmSTzYpH5AUSqBjUnMM8mYgyUWaRCV/KO2No8G+8JWuPmTKL3XxIhfvjcL+28IjO8frbAtdhd9OQfE5Jlk6UbKbRwhHljdTHKNfbQTlimfdy2oRmOng6WJO8IbegLadv9uL9FB9jU5RJ9U06IDV8ModzXCvcOR4E2CIeW3QuSL3s4fBJfjr/SQ3XqOA2tgkHmreWpaP4f/Oact3mJH43bs1IC1IoQMEUiOYRfX79ywoFCSK+kr5FeFnPjU7PFI+4dv83lggKyuySCZFGVATadHm8PHj2YhDHcEcITLampmm0v7186KyMiP+5AJj035t7PQsq/W6Ln+01csGo/Zt6xH0/vsNv2VRC35LPHv8J1x7DJvmxcwQIZq/iqoiG522v+YAm40prMf+6cjtGvjh6Gfq3P8a+T8QSPgO69S3asn94CcHATU9Y5OqWmhdBjZKHuVaFPNoiZs1VlQzPYcreFKRAS/UaRJdOu6EIjo+o2ElXXJxS0C+02zZ2itdJ/hEhrSH3x9kTwzjdjqOZUzUVmshn4zVFjJrfwekrBjwtTg/2AYdA3xDP5s5a5pi07ZaPQobNJNFwpF6J7kmf+HVut62v/3QG2qI2ms0Ic+KSz+t6Z4xTdkOz1Hz776Ywa+PoEnB5pN3f4BKpp3XCQnolTb9Ttz7hDUxCIetmEr2WFd399WDNsm1oPlGGw4DhPGxm/adcOQIa3cx0kFcsNzcqFoVJHDzwYL+WfIDJuU70XT7mdP55Zc+JyH70oTSJpGldHdfmk7ksJY2qZXMsltmCYeSNws4WOAZUiDWn03KxDJRuKya4FfBsfwWEgsorCocdpE1YMn/xIioAL+gXQfbE8XBHgOODJ7SjdE5mU37JRXMMGFz7HlkYIaSKerFd4ZtYV7y+tZFBHeQ0iwjHSJwii/JVJflLb1jYBGNlTDUhk1ui1UuNLJAzDPveEuVPYdJpNgOnXZVdK3oQUOKPKVWu1uUr8EvQArpuoSEL36VkxQHjCAQErxAlk94jJCmIAx0iSiEbDYVwVZPLEeuHGlSZnQREArUFDDV5/mAndr8ZoMCH6EcHZGzvBRLYWDcMwEQcp1ilb5WQlFWIfhKqvcm147au0J4AUQ9c0ojszIKdNrhNKZgqCBWRwwM9rrMHune415abSBDwcz7le3mXdlsOrgqwXDdvxKrWiYKxOBIM8m1CgQ3lUGMC5lJc2hh03GqkGoKFiDSqmyJuoLVN/eupqVsHX/J+o1YLCWmWzEmZjMfRBcPB+rQ0vfJLxeJCsu058x4o2nKOcLBQkqylM4DHNVuGMk04CGtD1hl1xvYqb90QXtxVNksyi6XhVjisJxspEJKTwKQs65fv5LgwVhVJkpxzLSERYC2Y5aMQqgLqGVSjrkmCg8qVQL+R6pLa1DsLVgnkwD+M43vzUcmxS6UgQGJlfWI3n4SrAd9QiOLJ8bl9DT1HHg8GHmLA4EEhJrO/jhZzbguZmU9rkmHC7Z4bQpo62wEa7na0OThMrujroHueCUuIp/Igx/4HsbQ22la8alTKu1a6W4gLcDSSxrkjIy3FMgnpHEEicEgEd0pIgJ6Yz24VkCdGAzYFIQ0PBIK7wso2RxvA/8qgjVWfVadKdgDPGwzUIzAEhMENpkm3fnxZz/RTMEnSIjKb+9lMq5Dvg98bfLLHb5Nu/+gRWVt6/8bU+Lvjj0V9IKAzEOy5d4mOFlwg5++kPPClinJFFIpGqWwmKJqjGlLdX4NYj40Rcr2yemwjxQjozH6GX1l8DSEFarnvKEi06VOsLuMOXMBSdBRrVZJTsGy3X/dTSievHTQ6wMtRO6aU+CttESmgW55URe20DwAWPL2bd61lIqaslP55bNZl9fH6yrERaUw0yTL/+FMGGBHHpArTd8EZWrBh0Mo7YiOg6PfxE1fyth+T113gvA00y2TkDgjElNkafcNan5YZ8RxetKkgiA/5XyDhfePg2aaz6bc4PmzC8LV1DoGlc+DCBR1YcOHxsBjBOQ7VXvQmytohT6POv8ILLjyBkBjFL5iF3sr9D9RHy5vFhl2ygIjNVA5nRLfsAKqbu+0w+1l1XEZ+aWAZTNTbIgF3JuKddWTDlFMOdaLTDbmpBHR9TuorC8GB/wQVhTaYySwEnx+RyBcfbSBMX3mQpCtfggeKrtZdiX82cKtsuRcHBCFdEN5w58Vj4AixCodCA+SCy/zaWTbjYYRVjSkkmNIGpkiQ7yFS+2Idnz+we9+bbDLJxx4048mfL4b+n2ZDgeg0voi7gXm22Lhy3VuqpomRN9Lw6ZtBlUoJ+SrgsegQEwpX+degNKFz1OnGAZof0PuplVHgfAarOccyAyAksG9/ojGLWzWlr0L0oRF5fgu7tpFlJaoho5xyorwUqkjGlIrl2nNjlcTj02/S4WnBAN62RuU9u7oIsMNNSq+rK5OJmquVTdcCIlhpd9CXQ/99pqZOzIMWLiI//zkaePvxK4r4ijIN0XysMOqPWTCEGHnUbeDtuS7su+47FuQs24S2oN9UtwW43cOaym5LsQIXyUoNECzZbN1b/Azp14uFoXAWw+G9UfwY4V9MekzDPH4tQoZTmfBQEvqL8+OwpE27KWXJ7hWbaSKsuxihPwiKuFqzmr1gX5HAAzUfHlRwyu9uNYLg2mCnGnRBU+WqrKPf1uXFTP9TDfD2mu6TPBwU7M2rg7OWoCFgCIitWamjeCmVkKfEZ76U97Tw/AF9rMsz5LFRvm2ez6gqhMigPhykZyFbiRcSXE0WVXIly+rxpa2Sntm9tA16vB1jvr9+Kt1tvLobCd2Z8yf1JEwXwqK5Bx2w71ZbVv94RTECDcVmqGsI3eu7dtrDpIF1mCyJrNzk2MBVZcux8fGdNjcNd/fBKc5wR8mJzlYQ4L9aH+5YLJuOI+avEs/7V/vq1Z7EtEaaSa3aahHrDBlbG6GuR7mWcbItGIYB6pKrBBfMo0IPwPfhRYh+k6tGO3rox0Lp6bOr3DoeTq+BydBN2ecueXI3OXd4hv2PRXLlE14yrFIORHgQYZsGxRKwAg1KtHOjopu69XA/scGGiob5Qx7Zbb16I7g261M24960MtsyYlqpis5u5qPd8BUAfe1dPjM4+e7g5t3kbz/r83W7U6TXcP/LAXqRe+CkOivQFcpDpTcZCPm9dU8xXTvePuLdR/VuJQJkFHOKBuRxvMxSLNd94w2S1H98qE9YsBhS2MiDUjahrhpKo2FmoXmP6t9o15HdQpyHQYm2jkXMXD8NbQ64VZUu5C/YA/ns44f3i+IyyxlHNVTZDd0Pan6bM8t4LFRTbuOZ3oO95Cuw5BWniCD14kLq9AC1y33swsSKksJzULMWIQtcoD/24olootnyYXVoFCUkQGT68yuyr/7e/rdcILXGutdbB2AjXS1gD1JW64g/ApjaLWgTbD7o58gbqgjJ8gmbOjIbJ3wSlfoL41NzhnvyypBJngDemQ+wwMYlYG8ypspJsgDvaUsipsZ3gINu4iLqAbExKkxn5YFY/7E68iaNlXhs+2Lq9aw9ngCx7WJf1jEnUcRXG4/Xda0hiZEVCMYGXLV0qTbx90eMkSc6Vkg7ggXHYGhpDnvYWOVKu/hdvlhSnIz/Bxv8mxtK2K9ZkucaXV1+e0NApb/pWzYd2zsAqQvTTvxTA0E5e90PMXI/vqglyRfOKDcXpLGh8uncMEV6FJbGi8k/2de2UXeqiyI8vDo0qRsY/soc/0ewonCIpFzn5Oq2kaSmXpKslSZN1FZqqvmFRKFzNRr1nJqWEIjM8O8UdpuBtZyIDSJJ4/AzFS8DFYq2SSWJT0ex1wYWpgk9V9C8386emSIL/tLT2ccilzklQ9AnhZsLKqFPt1U9zdIexZJDGl4AfTjeCNhoatKpesCO7aqDeYMvVk3tnfdvtNvDLucGF3paOVHEsz5eKUfdiLbhL030mj9M3WC+IshYeUDnaVxtoOCW6cOiXtZdkOox9ZTXjy+mCBzECsiDEYHNoC8++EIVHRESXsrVJwEjmDoLvbiCVCvsQ1H9eJd9aQgvKiB0mwAn63wenqz/a9pr9XgHBR15/1sDW902ehHtpDyQwxVIJgGvuT3wzpk4q4zo6CmEJ5PB4aERnpAEIbvJby6l1VEWaTKBArfBrK2iB2mPRryh4ML2MtlxdGx/ShBe2pLkt1Q3qXUtNhvquIoLtCXTbEycfoS+vztT4TM8jeMOvwLm+VI7h558k/dMogTiBRvnTt5eWP52e03f93X4ueMDaXV6EpJbKgGFhLm1p4lHkxK+0MYx71YVIHu9NzILfPA2+3rJ7pJDy49eibtQjVhY8kERJqDMUximGoQwyAjs1ljoqBLBkEC3Hraai1GoBASaraSfp9NdHu40tAORAqYmGWWvOQ5ZV5BoawhhMpKwp4BIJLPmQ+ZIkyzIL1Fqe001pRuWmwkm77EPJJJr+P6027VbQCNOo2d+aHDbW9WamYlSShV6RkzjkI5V5WbD1id5Wuso3wCU3et0EuR/5PT0A64hc3Xj1Ys6SV8F8q6GkXUGOYwIhMdWt9gaeA1Zbw3VErO68eiFn/fnY3ORSglXocI6pbeEbWLp9U1V6InM2G9hbn148a0rt2yv97FOnV5ab53509ua+bCO+OvpzmTDqS5iRcsXWaL2HvP/YZBdN1zGH+CP1dY0Uzg5CMaXZmAGaZEsWAHns6do4moGwj7lCKd1UzRV533S4HESr9Bsg7R33DlLjTKnWjo+AjnNWzQJvjPjcsTs1h52aDRCtYiPahWCkv7MXLqVqx9pnD+yT/ptcLkmZDrLl9JugCrl0Ra/IIAaeExWlWl4p5/LrwyVxYYu7N1QpzVpTsw1bN4EEhGJB6SH/+FvFMWxmsOqgTWfiXBLj33zEOU+An0ikFVQfqzSwMnh1LlNVKCZsaWyytgtz1I7IXS5r87YcglPe5Po8kvml3MPwyxhyVkkmVB0ANXMNiDZg4ts9KW9XV1Suy6RzbecaD1cC252sI/NRnSOKDMvxIbpQDLvn1Y0W0ZkNONYiD3p0o7b/iKA3v2/ESE7X1tDQTgMafGnlxklQqM70+FH6dauAljFBeRj6yk6LOa+5u0FtUeP+A3fHdi7ytu/AvVXjNbojjj9MQg8Rg0zLXn0yCJ+lQx2/+pNsO1H3F/HJ0Bh/xroOjk+HnSl9oQf9Ic0IUMjfTcL+Rm9mOPZPiCRP2YGvA+66AVP2DceG0QGDiDisd8+6rwa3fZtHQZ/RWuhHeoXCocyWjNSa6Ob++q3Zyeiaid+DDB0+/wiehK+uI2unG6DD4wd8St1D3b5WBOjnTs4+G5Z2XbHjo81wHRdWi1zQuRHBbJ6DMDUMTdBx6p/kHDR/Ioj2DLBBAWve0udYF9j2U31lVSi4TxqvOA/9L6V//tkx2qfvtnn/yr163PFDGRyLDMCWwWvK2+G1SXeKlZnxJ4LWbkmT4arzoN+Rm8vF433yY+X7tX173BHfEGUtcpkXiD8qIOXYa7oyNxKW3hDzofr0L6KnAf/M1G2fiPYgZmFsybb9cjAZm989i+7snft6UQDXeGbDMBbwvvvgE6bXiBFi/z5hP63oadAH0cdmkQLR7pLVedJXyqKDWC/1E+K1FkexgjJxAjOazCqcfW3Uqa+GuEZGFy3ahX9E1CH/gTWl9fHHFynN6TeW+8xoEuEQn8Ty+Pa74TdO/ytKBepKMJd1jyBTkX1cHNcJw/sK4ACaCbK2BMQ5N0jmO7kfqNPu5CKw28XG7mJPb7NR6UmPBC1l0qDmG7kQcBiF5bEAR2h+wUOVpXFRWASJhIAFVoxyhGsA+40qj0aINHXsgYVRqa3Hk/C+B1hZM7tSJ1OgioYJIxG3U0S7OYU/XzsT+42XpbWE5FED1clZrVesS03Aeq75IaRuH2PbCBPHGEIUOXTUb0cGHwszgJoCQh0Dc4BVhT0xJjOKPAUNTCx01d8zL+wGJnBK+22b9znjvFKZURRG9xJsuL1bYpY1pnZz+38vmfcn+hsGZR0FZTRxKG7/IHHscuBg9Z59sZ3J0WW9BNGonTpoP63X5kpX7Ak3ISmC75MsPmDDkinEo7/2R94uTGgpiCAt/W9vn2fLE6xUEWvCtYUJki1yqYi5DMOPNaa4IQd0RpZN8l5aiGfqxtfTI/+5kekRIR9OuPbM6ZlkVYpFMO7fnx05YHToteoAvv64HikXMQQdjYikuFsYL9OaWLuOqUDUgmF12KBPzDP3QEh5vdxL+VWPMeq88GLLZLGK5i2DZvvDfC93C5bx2a7VjLqdYT+53pRBDGj6u6TQIy/adrCeOMhLsV8Z3uZM1wbivkfVNkz2qGaILuw3D289EIbi9MAbX1ulmIhdo9T2DgbDE+0byfHJkP5GrY/3255gYLzyAmUIWLvRyeFuvgWLDCdtuWbL7zWGbSvJHujZfHzFRu0o2tHinj1z0lqpTwvVcgx2Uzk9PD2bhu2wg/tHuEEqDQHHS9V878ALGuOinJSfWINbk10kCQ1esA8p6GMYNsc7w3KE5CbSUpxx0dWI1zO+u3rRJNBgwMWgqUUCn9SSuJC8aLGR+PPREHSXEuc7oJu/g3N9sRzDzz5OBbxbmE6wUWYLqVsf/wfi9feuv3+cCx60dpcC0s3NJdGO8li37vn4mfq0Gk5jqHP/S/JZUMN1ERiGu97LjlME+Y3j3EyB6RYL0yPO9L5ELx3j/33VUMA/JcqodCi4ITDu2IhgMKtK2pENjGm36nvI7GgGynXHqdk4ev1GyZMH4kzGXvVXoFGvaJ/FfjmDsyRU+c3aWi0D7PolT3F4WbP/djOvIZtJmIPXbpQpIaDSDDN2FfVFWYKvq0oUtYxP6pN59sLqcbFcsUjeD7pTrmw8upC1ZDEDVkIxVrVKARmsZUBe2dJQd4qV7c8DVgg3sRTjVaogkkk8g3dl+z11pQPViLCjxDL1EaLOw+d1CEitgkgC5jfc99kWqoncsyUllbsB7c+Isz9T077xmv+7Y7z7QJHspsZODXfWNgbf8wwct3bw2ZHdkfbSsw/Zor8OBSxhn5fzs+UJZwBMp39fBEXbuBE/5aCdby6PW5GFrt62gWukPY9/U3UT9XBBFlnuKWtTut+5AgDZ75+UXxzgsQRQNK7A2nRvq7ELlIHk1gNPJDzABS7qapynr04x9ni2s6E3DMACdeKCDw39hQBWJPesiOZA7LAHXyMM2sPWTKK30Zux21214OORSee+TD/o6Xozgni5oOVblyregnuwETaRpNJht8Mg0OJhwNcXooSp5/sQeeMFrMUgustU4YLCdUGuS3fHe3rpm3FflRgHtYvYJL7iCgUHcwcjXZWoY7JkI4YWDALKsjIbnQY7BhMFQliIqUpsG88JWYN2fX/9QPhLv4zzqszI/hObgFykB4BiE7CyRBTaJ/XAxo2vZklfRwugpVZzKKgZet3dzdQfDPszibhrlC+JreqfcO+L9MOevZ4KhsglbAXW3Suhsq94HMRMVnYA6e0w8klqA9Zblbn+Fpr2p0U3HY3RM3mm0UvN9UkU2bN6YL6YRMPC0aijZ3aHhbqhRJ4O83iXEUNfdX33z+IrXyYdXC2PpgvmM/StXbFMJ+OjE4RPO1RGQDZN2jR/AAqtIIVAego/6o5hkHqvFyoz0Djrc+gbg6QUhLrXvmThUfRivwRujzRFrLaLu6N1KNpt443Y6t5PBw5mx7HdngXnGFcj8I4DdcHZRDI+41XcwLwpfMr4UwNYZfPhta3ge/T6ynC3Us7JpYqUKwUuwTB+5uNXqEJ73LnJ46d2DjYOAH7So+iKISuwfCUjDQoxOnJ/vDnLnUbmsTxIbWzYr1aMmqSPZHMX747Ga1V8Nr+wmF0D6IpbJhHBoCx6qGVSK4tnZEkRFNE6snsiB4BNKIXjEtCZ5i1EXPjkdOnMLOpUgOFua3JA3HUyhv0018Pmb1ryZ6Kdui8qIlPAmZI8DEKgv1TGRGD95mQ6G4C54M+RMZNcAyQosB2xjFsreJJ1ZdIBLQinAfQAFkq76IkanpqgkXIiXDRD9cnDGgxcjoSCcOpOn1VaJiE4wEfcMSfDOM8lh9rEuKU7gRV0EIclzMdADS4c/4O7ggtcprbdbQFwchPlvBL/lX4e1Wh+Gy3qMoIxTKz2Hs/n83lsDFbRhKsgw0qN0Glqi1vnswBENOzGYHXw87H5yVzCVyRKDfAcvKBcopyuoIUx4LP6q7/lG5XTQHZCf8Pc4RuXqjZulk7uPOmYbIdInuwumE5zoHuRPVdfinX4r5/bgiuxdjeuJlyETtxx1H0tVK9GwT5zRlboStDKGAXGDnv5G6fYdHZVam08b5vLobyxwnPCjap+/ZiW1MsjODwKOOZuNg819NsjpKALNqy0fTJ+YBLYiEqoiu76hsO3OfRXn6fqr+UyObduZZRS43PjabbSehkjxHF66cMIHG/+p3sZF3M7s8Ct0OU4buCBqHfttN4tNZyvCuv+ql6cMbVFw4YaNW9W0XCHWsMtdcadNVbAN2o0tIfZdeRcQHL8MIFKr9wRV7r0g4rURaVDYnt+BES+Qoqwbq3Ro1C+ab2wIR39jNOUFTvRb8ghG2hG2omYYtcIZ4VOefFo78ZybihWB7vebxr+38zv5LZokeQKrgx05U3bx5Cx4zX6Igvh0CvZyQMo3zRS+BdByrjgBKyAZ1d0jZwSlDqGkQL70nwOCPUC12BEIbkZsyfXCC8pMlEWEOKOZ8j7fTK0V3ZqRRm+yyb65rih8w0ByM2PPtJj0UNT/I+EyampWLt9Z+64PNv12H+J3aoP1n/Ux7159ghUJOmEJ/wx2s3CWgBe1Z57K1ahSyEN2ZXKyxffWtQywaJ6WVkENfrp1RLUZj+otKAzoBkTxNYtjXjhTQD6cAx8gZLq6DTIUKEtLRKa+YB9pJ5rTWkFGVVTUh04pg6tGkrxLdymwo3mZpIDLAjxoxjgDEIsBBQrT53QfSMT5EEls+UeX5LqBA1KezsKCgCLzcA4E1jFRtzbJd0JDrAR3p5Mb4OlbsywcfOKK8B6Hb8Qoj26zVwN4oUru321Tg2WjGmfKGOntwWLCbVVl5wFAMfxqeDRGoU0AfkpJTWQ3jNlLwX6fXtphNAJMpqmoFlQpYqcCgoLKEVpbdvP2gUA+tXGdx6YHPr3W7Fo5/kLH91f0/JqvP60kz7hmLm006Frda1lxFIzF/KPddW+tG6kUhcHFlQoA2VbKMqzMIP018GJssDNp1QFUPmqDTBHD/THCdtNxif/dJrYZev4tW2EQpYvnkTuhLs+S2HipwGtBMtR7hZnkLoliAz+bJEmaHSiRmqylydqPp/rap2y5a/FY+UNbbjdhmdOvPZsEgtPzOB8okWm8/BvXzCw6VKRFtYvweFr/EmMFGQ7wuh4Yc+ELCvLAvLFYoKnuXVlQ85oBtViG9DFRlmTy8sJ1lRkZ7SEa+KHTprLqOdNycR6t26VC2MuKkBHtcdKGK6bdy82g3HUO+WHnuCOlh+TTmarlhYjACIvD0mIxj2Wpn9LzC+3GuWmoLVz7e3aiIjeUh2NT/RNFjxJn43vZUI5K7LYQXAjYbZaZH95tlwMxQa6ztPgOuJHB/slJEOJUEKXWVVuEpOVq8rriszvN38/a+vtN71ev92ZqFs+kFIBiWBxUqg00EKsUU8K7enFq+6USD73TbxPRoi6JDE0vchXY3v5dNBxejSiPC2yrKRRy0imja2kwVuwwA3wkAB7NPZNJJ+9OWzj25PoIP0h9v/zqh9PU1Yx7bTJGlYTI5DinC+ceK/3h2I1QytTSrzYt4sGHdnWY0GTMTkv3yhGVeCisMQ/3RpzoOvOSbs9kW5Z8NOraO63ZxyWuyV2Rm5x3tsy03AXVStDfBOy0FmVOSGbXnTrFil25n5nf5Edp+wK53zNxWLfq5vKrpKdyMEcpqmvAjNrPHnXvddVKipSjNTkW7JMfWFzE+qbUWLkJee549q6caf++mRnD2qb0uiV4Er+iviSW6JcLcsYWHhRQzyNZzURpUYPJd8Kljiv8Z8AeBa0CYDacGjlhmvIqzlupw1YK4kqLIsA40wfIQ9cSjEoIAGmlgF1svijIy/J0gjUwv88Nf0fG1kkg86Nyw34CsN/ZlovmwgRZR9BiFauwNQHRND4MERV5HsKzClrFEvZO6q1IQITyI20rmAb6QCax7S9pZ2PA+caFiP/Jsqx8bsPXT/xWI6cUhWdmNCO8p6GkMFwxyjTZ0QuzZy7bjkfrQYe4I9RP0C1Pa5vBNidOefoIQ8I1Ju5MZwJmYMAcX00whGACaoyIDaVhGR0hKx96uWYyoY9jggsMNXrCgMMNSzAoEHbnbPkgbKz7EKGwyS2y5CfRE8VUhHWNXxZru8KtIb2Ia17a2sIsRBbjFkQVkE4YIdYXtiIQYRhiR/zXrQzDbggph9k+3Bi3OoYSCxzjZIzJB07ou+WGpNlPW7E4xFPu5hxV1IXqht3+2A2uZsg4Ux+42gXlD19mm1WWhdq4Te5208t9thNe/UnsaVKD7yAfR2/3DIkP8AT+C4CwOnk7Yv7ejCwrb12wbIs9GoiTN92R6tWuSMAHlSZ7qjNMNMk1PqghkbYZUMQI+QBfIGfU4BiEt1m6sGAYeu58FNaFUQpp3yNqSFkBTZpurLHpi1xSLK9TRjgaXbZcTDE5FMM9oacJkp7ct3ONbD9Nd2JbFK1mYgF3HduTla6RsdkvosW9G4fKbCqqI+f6GsgxTmU5jmpKgEin9cRwHSraJdE36319KcAbf/t6hGhbb3TR/1veiOSUJ9UPtpI72Uc39dmb9TZgCSEKSDk76uWSt0qVF/Xf6n90nZCTsu33nCKQhEgcrx+grdNN53PyhAOeBvVkbVKjulyrdz9Ue2UdlZmNUR+lymBOYzxe/W6F7H3FThYDuUWrHzOpatPpyxSUno1FF/kC25FGdmHQtsUyHG3WDN7tNF/WdCDgGe/MIKtgWYrRhFHTSdse77Wyk3xkGqVai9BrDVEHZPhpJAd9jTAvvK7csBeY5sUFiOSIGX8BULOP+sbVQOueYy8qfcX5+ClRnbhp6QfKsn6Eg9iI6CXFVZj2rrsAl7B4nRI04RyaKjdaKCyL1QbbijvULUCommHa8PEMxrXgK18st5fmmAIBZgR9kyvpTXagUk6t29n5o6dnPJK0rYrIJHI3ZfZkakpMJU4ttfycbM88p3TiZE2960LtNuJAUWx7ZnFFdltH0vD6TLj4PI5uSgQJqQ4WUDqBt1ULZDWOlYQRz07EA68h1/3IP+x+e1bFdYpvylZuClEDESVTakgEWkDcnDdiqgcrKkTQCzBvt/Aef//ji5GemouQxrRmZwxmUJqhMcLS3lqMC5sIXZoSjHCUIdXGKrDJZM/LsYEMlinyJ8XP+VVPe38SNdoumta88Sf9G/nHx0LgGnjNr6/Q89rMmwXyVcoYfWcek5V4GVyrYoHBHjOXBSJnQy6P6lhbzBT7R7NiqyxmFVBTxffRoUI/vOq1e5RbQlAOy1nIPqubwZsbnzzp1f0NedRef6W0r3nWpaNhMTfMR60p13drjDbYVsE3+1d6zy3U2+QaiJgl0S7pjUiKV6wXlEI9JOeLnYYPUBtv2EKJZ6Nv7Ss3ek/rNhVceglYtDogYb2/804onJFZ0AC3wSsitctB8TSTsmDJWeeSSavF6nJ4N4iWUzEse2tRiwnFRVG44kCm+PLALjTS0jkRb7xei3C72uEoLRLazYY4VJbgsINx37xVTDD0MU/g6rxWGTLStiRijgdWvs3k5oywsNtE/JOW4JtDvzAwh149UrOOEFnIhvbo03Q7lr+lOPoKfI6T9ohwyXCABN8vvHKtVVYpti9UUSsbN1vbRndSYJDIh6cNXir6mFfb6hKCCwuKBvswLRyQK8FNOMzRT5YAZH0GIMtdrbIlAb2JnU4Fq87UUujZV4a5ZQjAlyzpzcYWOaTnEudqG+1Qp6W0cbL14i1eArPE4XXYyw24GP5gUK8gr63n4/B4e7EH6Q8rrJt47G79BHwG5xqqMZW21aaOK5GOT92jH2vcTqmGQO14OU0/o1/2nkhPsgNlIergw10NiQ7E/rLrTAyNj8ClVVgPZPM9FzwxpbhtP4nPnHp+LefRWpJWTPYt+fuOD1swL+4zENhnBki/ABiGLUoXqM4gQHD/NTW3O/PX4CBBMNqgWwvoI8Z4+3mg+6zIAw+JurEsBasAeKjiMUa0hhacv/dF9NJho2C1cLak6FWKEmmCSEsTPzyJ26PHsnBKPglemnPOwNE+jnggX7ftqKnfLT+WN2NdMXSIhoiGw6lIXcO4uRErCXQIg6IEoHhMuTLx/0XEzi7a5VWH6KXyS4c2geSUzSXxEiW+2JrD/dC1ClALo1rnkivvgpH7/zttW024GC3sfTF6w6TIzlo8GEZOTVRplrv4juxnfG+BKLzvzQWCspyVAVRo801YKpHj2WU3zmmVcSpfeagDFfnCNw6WtBU2ZiDln5RVW5nkVRSmu8oosif/oagLDsFcd20S7DgNDrAhRzhm/+nHVV3kTYFq1T93RLafkh5FExg+uHj7i3IxloEFmf4iW4j6DGpBols83q147QljyXoBej/A+Yf/L4D8kY+HuXz8P6rNCA9Ixu8CyfHf8iAGmXnAXL2pLB5yG+DKeViA4NCka+U0bAmL2lLwlCH+5BEZ8t8xaOvFl9tbaksxIfWfdEOeclNdLEvtOZPkdTixwFLJ9O/0L6Y2TB4XcprMCBv4zowNr4PC5GRMq72W6EPeIAYNaeAvwdoTICaDcaT40H0lMB1C0itkwRVWFsULtA1aV7ZDK/JCadU5bkay++94OOD3TbBnF0iHPgPfTSEXU6tAa9T/zpFzmd/+WxvLorQrajjH+lbdd4rls3/5ILKgKUi5xv+Q8sJrsylLf+q95sIXZvSYaev2cRWXP1bZlOrUacXM20zKzMgbNQHdsfYbkGuHO0E6zgYqzZgRsyHPOCz36MUx0vSCZxxEZiCgW1D0aGIFGXWBxduLYHlZy/+GuiNbgDxjffZWsuyI0Juu8YmaLzzeRPvjGibD5xylj7uqJhqfxj5+aXt9XkWkaNiidMaH7Umx19RWk+YRBu+qfRBOKlD3LDf/sHvTgFypDmOFMWipMxtj4eAI+SsBY29o548OY1vyx7dHnkNdgEJXLWurboBfaGdgM/D0E8CTdgHFlRshyRiAeiJ6s5Gk+9fuOk4BQ71Dsm/RUqqp3RzQHAFR781fHHJAProluJ01L9GQjThFqjMRj0Gl8I5sriTNEEq5RoOdCNz52JUzbKepdc90c6CkDEKlZGdbv3QB6UxZwOB/pztGuCdy67SPGvO7+V2g2CW4GIgOEc6A8bgY4gOP0E69U/6dfTOL23BM/EHCwq09PPrkr28afKOjk9AkVmUWwUIBY5aQVlTTZCkf3j/9Mz1EFr4UAJY9w9QPlkiFlw3NTNPjfJ65M7babeTsOlQ0EghARXkqYAM5eN7MLdaaTQFp+h/1O8ikbwQpLaRy9T57/MucJPdsnSj2s/ZOMwX7j0sorNI7cPXZQtJp/669vJjdAq/ucdDjUaWK2ik9ejevU2hX0eLLgCPwczWmixb7chdrSRgb/HD4+cr2Ejuyxa18VuaYA1564Pzh1x0XVX+817aMSNt+bXs5fQp8o7X1R4Pv/ysqBawYk7FMoNlo2tWftYbPwCkeGllowlRkDmf33pi2xFqSeH2qId5LV5Yd7OqS1ruaEk8E/c4VQ+m3ZaloUn/r166+u38DTnu+34QEQDalqrD5Ri+AbuzVe7mEjtj5XLaFWgn1bSabGP3j/6bBfb8+cNbSc50PHRUKyzbqJWx71znB2oOyyQA4/Ab67mDrRYCg8oGuxJ4KwuyTnSC5JKu1KqthdFKOSIUSwTITEVmYDaP61RRhF6tujLowUp2CAoVuM4923Vt1s43jFQt2XL/Pn/JmlJFLMsxBTotd8oJNIO5VrXMPz69/SEDjWQW9KJm5XT9E5RamQIkupgfkbs3D9dCsbfXRPExf617BXtz8IEwgR99c/3xmy9vzGF3MSa3qUbSkHkBSAZATrbNkOUjqoEgJj8OoaPPPL1VmN1A8R9zA29dX0QXek6pa+nvhpPJSgE+b0Fv6Tb5t8gQ6RpQRyzoiAhrP2kDtvutvZksVg1VESoYBNzdVsoUbAMrZZcSj//5aCOuT5dsWA5Gna2bBvvE2B0GEafbru2/Pel8Yt3uw0KsdkVFTwqwlCHjP1T0Y5SFXMnzPVrwMllg+cG8k4v+Wf8rSlGA7LPhLWoNkZxThcFRa5MoKmVchS/IyNNO7xdedCSsGoIC2pLeUEpk46de5tXDc5cnWQNud8gWSC+OHJDPf2np3KLZK0pUP/IjiAAepl9HR7BToBZ1GtBU0PnAsD55/TU9fOt6nWHVjJYfSv1Bl3rxYPSd/J0eaFmheb8686qOXwi0rIf6Wlb36tIV7Wp81Qa/D42KCPkrrdNxMzuE3cscEWbHu9W1U/zMUXWGRUZgVNSCPke3Z1j2jsYttkYwfuzFtAkLfbEE/5xuYmeCF7Y8GtKHpzvVgyz0jV7n1Rjv634HmFT+ZmU+i+wOgYnohD6untiKE8RgwfPM3Rysr7MEAoWmPUlouQpKxoWam/0HWPPwUrO6k4ski2u2MgQ53BP/3nq4qtdO04x/2NKQc58fWelcigEuF0LgmXPzwcm8Hq9W+k4HluLF8wP0v0Bzsm4iS2fVwotK51J118StnXUbmHl5GrRFjcEP4G4Ps+4vZx6vzml1mbZk71gPGDcaSvhjNGm9FJBQrUnFKnpYMfMLdlvjJbmBXfb68eZc7O2Wpvz+7nPTV0pfX80FTshOT83H5/95TV8jZM7czg7VNVRiCBmMYT6+MezCVuAJWNA4L//ZrUVXAetTvjSjRmMFZQPFKZ39iCTj3K8GrMYfhwEceKmILNuxf/C3VlAhWSRPlHwjhIpXli1UOtxaW1906P89a1NxDqUYQ8Lsk12c4f3MlCv+waN9Zbb81RRlm7im3ACdZu/TNXj0kPojXYA75G5p+6BeuLmPwSLr88OTugvssrMcvZCHxB1/pKz2R+zxaaybGzk//tdomHXt+rdwmvv+P3+Vnb1aSdrYL+z5oWuP2cCrtlpNb5pk/OSp2IwHfFyHVlGxrR22aoAHWxF6lohgnXfrCasPu05K8298IKL4OfWIuwWy1KonQk88ICuoWVfaZnJA6P7y/gEbDJEYn/pd2VP08u8k25kNFpjlU0zgioUOC71dZbKPBpfrTyQX7ZGjebE+/bGvP4XM+WsjYrHI18k2zviWkPZtVcxU0y/nlUbPpH18gl3tVMWJxvBq0CmnFPca2oAqDTXk4pyP35LebIRcLSOSFVrx/m5SMXiU83mMw7Xy/VK7CmxH+rpz5oZxoqg+zmOwInwspBirGPdTbWjQhhaJccHlj603yWDFRVHFHAzEnCvnkofTFhfkG+W4Aq6zZ/MF3o4srH9xsw1t5AshS/mluN/Qh7rUxEklWfZqJ8Q7+bCtTUCqSjjoRX1VQIEnpkK2n90ojsrAblC5C1ndX/ozFFl5cuf03NiTBcu698dVpprhihgwesqXnsFGYWT89Toa1tyWD+6/WNFKdopwUaJVjAuNYnulA8YO+zsbzKS6Z/TYxcKGPtQ2tHW9ZY289RL5UVD2WyJGTgiVSqvMVLnabLn/ulJU7A+Lx5D06C8rxoafMpmQp9nIEGioV6wmujM4pqaJazHleMM85zLXEoQK8ZhiAEFScJ2J4Yt+9KtxOCNytILL5SekSw+3YyxhQNZmBAhS6xV/iBLiFFTrtPonF0aQ2B3xW5dRH7bxv5eMnPxnrL1uZYUCFrLY57kGNFSMKg0FskZNYmntbOJJifQNv81ehs1NsfcltJ+87MXznFO49MAHkUxTbEBngKgGWtTHxjoGXELGKTLPlQEbD7sS/SbuJm4bUfrZE02bC2unrTC2LMQJ4IhyPnRAvOhVosQQI2qhXG7o6PH7S8kpcyulmMzwtShfrZUwjhDWykH3uZfAtf+dBnIfPXvNjcgrTmv0z+svo/aCfgqv/mflVzLytHi0xcjZttqGhVGR81Atvqlv9UDTSrn9geOKzaEgy1akEoj2tH0cdnTt37yOcr4bQ1o03pF2R9JJLtnZC11h2nimjUXNHFm/RK+iSNkV40sKO9ymLLmz+F7qYB//YM+ASqmvKcS/fV4NqOJfBxO76T+6qSlXdjZMhr76PV/+QNC6jpIAEwqfVUD+uXULDvOGNdkdxLi3upkbVCVWVmp6bSTsJYe0MFssy+RWZEMDAFrxoYHHNXiKL1KHqMA7qo12XOzRXsP8jL3lVdZeQDgFWjwx1elLh4CoxRYiwgduhJ6KCaHCPwChQt3H79W3H3vttEuTn6iRB04Yowk1kMQ/whiufeWca8AuAcFk+APSJSh3ZklOVhc8G1y0dE9YotGEEdiieCaaCuUOLdiu3ViQcnkAe1cLyLGe0Q17Azq9shCiunZMhl/UC6JQYzRZ9PHCkZX5BtC6AFN/bKledOajOgSDBAfMer5s1/xaXE7NWPdt1RtPbm/0N4AC+cKKS/N8VV5VdIyiSYYB0MDOgPbQkRTD+q4fLorRHYHd3uSGzIzat0tcWH1cijVXU37RtTgfxdW5XVMprm02tlNdU03FBgJcPHVvynnn8OjB3ykIzO2XFIr0WBv/itvObahnSRcGHygbyhbIJR7sZpWR6f/2pUk8utNheU19MQdtRKyShBW2d6ffv3iuY2R2DjstVnIIy1PGgB2/AbGgeuVXmbf4uLmCVIEUUebtqfmjYv1ZnHOLuAurNUmH0ulwOVg8TZr+HkWwS1u92gF3XDE86kPszOmmNIKvvW6/9hKbu/au9a2YsUxeTkYYBsXV5Rp6ruqaeQ3e2TgYXkQC55hJVa1lKJGnTAltQBwSapXw9cGWoqUK6KFasr/XCJRDX+GV0web27g7XUjFG/EkxhjTLsVlUTxvgnxKG7BxsFaK0ZJ2VB4+6BTIN6rzvXmjsa5KVt3sGRdQHUN1mhdiNHa4osTzXQLM23TCa9ugvbHDUobBmJ53cCMdjK3QVCt4i7JVT1AFUBR+7X9fZzs/qpg41uGSS6uSyIpo11cyIcaaR6cZTTBFrd/veh1n7PfsUQKWp776Sh4b84QmRfZws794a/OqTGlUUF9Zaa1kb/xxMhRsWD1n01oDeXEx4eH+yGLGQnmU12s67mUObZ7dXNjaO2G4qMfozg7TELeeVpGH44vr2gYIKMe5LpwoRa0RCtslDkPQ3d+UqyzMQP7iU8uaVqcWy4Mz5uCUmXgMCAmMGZCaAhQKZuGRKVnWNaYwstZPg6+9dIxuLhf4hs0L9VUMxH0ucwHqHtb6SEPyb5mbej8T6OhbhTAg+tz8IbhWMpPqkbckOZx56ue/b08glmWyFYJolGUvP5gFmJFP4Mo5tni09XiWFGe/bh55ZVCXQbc4SvddgbCBWhdxpWF/qfNm8AeZmHXuCTTx7rpNOi/pSQv05IV0n02kIse113p7w9dNtC56ajV+/jkI8kjRC5KTRQ9+Yok1k7iOnifDlXmIdWsd55Nt7Mx3L+H2i+7O7IjHvRE7TNjVsJbQkToyJ/IYICLIQpEQN2oKBHkPe1dJ7K6psnPHroNrxxttU8vM+wo/Kjvlq1cumwwYCuevW7BaR9wI0SDnbsA9B2xMnuywP38HwkR5QSjU2uNC3xfmx1bQSe43zbWpGCFFlWJ9Wl0mUsYN2Qm2yfG2qwrhtPeZL5GgC+kY7hpB2w1cNh2Z4nKe6EE7qDdsBKLkABtmeHy7N/rhxyr1byNcDlylzfpVWoOzVG2welVFr0BEF1/6ZCPmOieAIOnev1wZvAEJfKiDTQlG5ETQe8J/xE7Z/Q8Wwm5j6Mp9kMFaDMulhOsibYq6Q14FuscUrcWUrEp6HnJs9A1UDE2BY1i8lqAMKctMHPY6MPezU36SZmp/Md+hMTZIGhjEMl41hW6/9859o4gte/kIyST8UdLcur9YrJcwYUpRTATkWMO69vWv3bP7eABKtXqCLxSgrqjUf8ttDFhkmAc55zngdPU1Ns0lBfObbxvN2iDXJTBgBSacZ2GDqlURYMQMnlbwnCI+k6pyRczJ1RsLL0DDfhCJsRTJwCIWiOVEykg8JtF6wLOsyKSQRXS0ORgSLX34rrwXskqXAg+DypZcp2roVnwGKrjbYchxfHUCk0+GxxsdxBquLXeXTw+5MEfkLX0jeMLr1HfPKy0gyy+5KsyxarygByDim094qFL2ekVU6xgTe64MWhBCFOg7IsZNMuuqWVUHMt+Hcx5cN3lb2ei4t795OABfTtHW+K68mc748oj09RSbfGHV1d2XrN7pF5fjiPDolJawXealsE4aKhOlg4/K2f/syLbOAL+dCD3xK2L7m/juw+5F5zdl2wu4eKLK06LeX6YoXiKDKsB7gftBZexTejoCdt+1Wn3CcqfxwS3GOH7stTfAxaLs5h42K92tzKbzyPk7652FwWnlajqAH2HaWCiryKDdodmr9kqyVF0B0G5kJlxMSldc8tb70BPlCUGqhezvTD3pSywUcjkA/JSFvUfUzQLJlGtYiMkLSfMT3mr2ArLtlIq5g8GneoiZtzv3wkUAArSmL5V1rj0ty1zYorv0P1Q9SzCTvU45PYgw380+euDRr7fwa/pMWgIdQXz3TqfTtlgw0guj4NG5Uklcc2r/W1aoAHpaGDYY8KmVNwtyUqeMMKiJ1wIJykWSHKNjleacvahOPwVTu3Wg6rPh9hqTzi0LOaoVGIzX4j/2FHCzyeIvms4Qv/YWo16pTQnXAP5j1WRx/cuQfXU6gsvXRfvvvhpvTP56berAAiTgVU/pwUPZ27lSHpAQTZOvAJbftlN+r2Rn3q+GRGsy/E2Cs/RYOErfzcdbxlcxu5i327BTETKNKCiF7GsN357iTir3VjEkSQam2fd2Sp85V6Z3sGnhnOHBxy3mzzhLsi3asvVLX58P5d1y0J2GnaTD0BfU9+SkFKCv8i6wJPGniB9b1mlwHTRY8GFLXK99mZ7OvYOt80y/08n6btB2fPfrRXNvpzfm/tnQUvs0sEBaBhZiQF2QyqOm+IRTiAD+Cu3NxsRHT959vkUn3c0FWP/i2X3MIco0nQAY45HShBWyL6y9gT3Y98pqG872626I7u5ZOTP9Q4wLUyP1ltdNv6M05TkY1u4ZFMa4HRQ59pjJIgVy2dRgufJzbSTKvMtBjpKpLhSW6gWVWDYoG2hxwYURo97WUcaUkCbL39HaessAyIVEHEsZE8FbL6qno4zfwIhlI5zBgJheJlhCX3lzC+Avah7pRJeQ/YVu0X+IMb70pMWkmAJj3dnaO197jw+M43hjkykoApog3ZbuBbkFJLawtpDfpMhD20CYqIw00S1kWyBBm/1eYQATQEIzlZPy9/eanv+qCuQ1zsH88FqmQZwlFEbAmFLKA36/1ELO2N3hXVKj4/YaEchNNtd3GPDqj6/nvhfQOWW3w3BfJNasuhTQd+0NoGZUy2TK/IEoIXtI69mAyI6Q2FcEQ6ZMwbGqSmlJBWSqHtimYth32fXb2HsI80rC6Zih3b5APidjVIzErw/ZcraQ7cwcoPVP1mCWoWoFWQ1CW13mn4mz6iHdZ+VEgDUWTpDJiJmBDlY9KED3x41nitnEwizO8rnAPdKtmDkhigzwTb07pXjr9i/di7fZzS0NTFbFLqyO2oeb6Lmh9yoNuPZy0XuMWvvCg+0bwIVHIsvoQsUKaKVx54rkLWPcB/DEFwwy9YnxMmKRqjQBZWnJF2AoqSzHAmjuWGmTsHFg/iczfWvkz37Hv59rsr7uYEWpAlQ/yHbgJfPuGGWeedHt7YNA5XdfiMJW2QzaqDHC/+/og/QjgA8HadYEv++vvpjJTlXPI5tnJ+v/ONuTdBQqgeNwJnF4o4cIHI956Av5qoQ+BO2oLaP41NE/bpMt1uLrwYz6m0iUOW53MzNprBzH8c9ufOUzszVZTCzdnchjJmnFp6BAKQkOekrN1ym4Uv6X1wfYpcHmaNQA1++JgFbqOi04je+ULmVsbW6NpoVsAG2bWLsICRCKndqPUNzUSZIWadaiA14yP+q6G9fQdfLsKaM9u7miWsyhJxHuSWXNKHk4g77SXmDuONv2AlYJJHtqn8YGw0/kGyZRCw5A05G1AE69l3pIagOBG8dNqNUrcgVVjsYb+wPwjT8pYY/ZsGIqY8/5pk5n/KLQk0WpNgMYVGSKVeqU2n/6RR07E7WGcvDgnrdPo8/9QTMzG69TmD48omvIuMmQ3fKnfqGRmx/5bSxZI+s9xP2tJ+b8W+zoKIHVmpeOMfeEmLsf/ROElQyy9GP6HfUR9vKu67ILa8f3O6jsWUw5qNcTojax25gywLBau5FDUiBaBUiS30X5seUhC+uq8ZqJOrqG+iQBlhGEDvy3XSTp4Z3EBTOrKY6AMHYWauCplufK8iF96EKKGIyrNyKR0hP2Akh0mfrPPeZKfx5WWVeXURYoi1z6PwHs1AclYFoOLr7qzuF/AJcyS8BSrN1aPnk57bYIG/2ILVzFLNQhXmeRyXHXstrWhRk3Tjrf9kXmRuB3HeA4P/LER/5hLkg73U5/pl/r+TEAIEdUwufgBmfpl8ja8rc7Q2uNSJu2wjrRpi83n3CiXouGZ6i9yrthJzgX+tZ3K/tK9INfwRVJbjuRQJYyCYgH2Yw0m1hsoAshASTOBxcjGH31B+2Z8jZ+hAHvCrC/+R+WUjsq6p3ffiA8QuKuUstWc3Fu7uLMyNCbm/890m8pc+i7u5cufXKd9YLcmqdXv7KGXyvZ/dQ3jrT/Zjr4Jzw69SoFqPw2wrVffRBXBF3Cgr3Sa0wIsEek9Q3GoEY4YsNBT3Kh4afERGeQ+OMXluk9oATrXvUY8IXVhIuY0teUq1rNKbCOztWuCiE8s2mkbpVGrkm+8in/3jP/aFlbOLegsSzubGobjfQwkXqg5Te5j0mXF6eD31gEv/ff/QHnoa9JrZsmV6wVH9DOY0ZuK5jST5e6NC/VgMuz0g2g+CAhy7GODuRfJl17olFbY4ADoUcDAI78QNiadb1+Ybn/dvhZ6TXI36M/BP+h73zjOUsRW+rBZz1M5SA4+t8GpfLXachaVDANamqE3Cq04Ti+pbDL2vLPibaCe8OY9z0AKlJ9lkYjs1wLy/+luOEYnGw+fB0Rs4Oi096XP7munwmq7juDtc7fAuPVB5HbPrVzsnN/LXDAfav3umNGgXKoWZqhb03WBsLtG99dOdwkglMtti2h9oEpi8qbFtxuOP951DmHXQxnYp066gKEjwAa7gIu5NzCHiiZrd4YuucELC2jwq5f3r4wBD57CzBD/jZugcXc3q37XUffpR9FK7kXvnff70LB3eMs6rIfCOnBD43LU8oXxUX0LKDXR7NyXmJ5RhH4jGqLG5gokzZh6o6U9Pv8cM9oFV47MIlvYnXkJ5TyAnSfNwhNsEKAwItYhEHUdhzECR+xgIkmDFQEnoo5jQGSpTg/QQY47++TnNigtmqHKhDUJzmjbBFDejIEb5nB8uIk6Bw1THM0mLHmxyv+e5okeKPcgq7JanoWXg8/KNaxL8ZutBkPGwkilCio0fugFGIyak96Li0W60jVQeHoJ3ckD61eVLHmYg8qLiYvT0mWRzWuiO6Tu/Lzpz9rfPA183vbuTB7pvH6Bqa13jr/s9jIfJYJYbpkTuGK7IROd+USN9A2/WFpX3wsqJWffTJc8F8dSzG+ZWrjVhzE7JuIpUIfTvcZl9rKtQ6UpOsD9XIRUnFavsZ+spNp5ytCTQAa3nBSb2gS+cIW0MDjrbflvdsgbXhppa+aDzvB0XS0WAHI5RUg0PYuN7A+RKduLm29hVFdgas/OTe/2kZUogNW4oHYf1AzYSo01kM0m5jcvQPJUSdvubr0qZFwfjGKjyn7ZbtHg9aIMUIcB1OFsm8ql20M9PGsDbpAwq3JvvN9UwP0rOWmjOzObkhV8LDp+WM7BrmjJmLpsaWPXn0aktGSpZx+QNaPQTvocIJDrf3K3R95A8GWC6zCSfWGRtvsTH9V2nrsRpu3DvY0Jk8mxM2Xsr56M0QAQn7Zddp5/gLMs52gey93Hc0WUzdEr3z+IHdDQ+S83s4nn8BDp/7TfGV7NWLyNLgXu9AxdxHVI0zM4ZoBIBa5RYOTaEw/lz3NfyXu7f0KKXE2EUjov7+1ENQqD64cWffaZeDAc30P9loq5Q76DULaROD0BERlfvetUg9euIXIa4fYYIe7MlCI310ODsCLpfsx+8ueiPDvRI+7WHl/Tt9DvLZQtQX/A/pyuo3xS1oaMzWxzb0ivViiRIyV/xtqphUBy0i93WM05O3U5XduN87CGZ8L4RKwRs3uSeT9gZdS/tnCN/kcxHU54P31HRPtqZO/osoIq1Qt/ZSos4adOfxZB5hgFPg8tkEoqA2fRmHkaagicYTrZyAxsQ7ht3I8KfV+RD1UjsGUjH/wcJ+cwpd3OUXyXC0HYKpWZYwFmT2i6f3CrideqREqBuS+W6UEru39QWUY3ZUBb8pZXu4ZvMsB38C78z6c7m+nO/M6o6HH/UzIBOc3LnTPFlGi3C192MnOuIafGnTeybfg6aGvTmrQ5lh6K3Y1+93arMZqYKdwpcIwbfH6zj7gDE1/RH5kJ9ARiM2/KQnVTsG8l23AjHMSBUbpSPpwBbNbdTxL7gjfpQ4Jpmqc3U/eY2t7om5rtZX2EAPYyRYz2uLJRzkpfhaWe+iXdZSznv7Gika0SXdSOgSWGpuXbdQChcEQybAlssh+53obzoWbMUa3EL+yd8Dsjb768i4Mzv0MkTtfxf5sXvoSxIL8UNHpPqzJXf6oFcg9f/F6BiY+xvfB2VzmgJfdeMtKhiw0bfUYGdBSLCTUYsjDyoHEz+XsmCshYoOWw8XL9lPxPYEhTVCW2mFAWnlQ6+sk9Wuod1Begtw/OO+kzAsQB8m/z4zCa02m6ciwyR7k8s9B+6FPvVPowC2VbtS6+Jb0dWm6F+wikjmIdsXvu3eBS7aTyCGH9y/xCQ683ITZZJpNS3UKGS6ee6Q2gyrGfydmR2VtNd9ozxV16pmgGxXjOqD20IWtL33YdLiO/8+sJv8kWqmMxGt2CJ7JOTjHM33RGzkxgfuxo/VjaIckLkBaXW41KSoeUhyFD5V/n4NmF4f5PFJbkmEtuYOrbjm7nwddnsMbW3DdP2gh+g4HQWv+HpgnHftj+ao4GwBzCkgG1Ft85Qamsi6ECpR0ahxKZusXlqZG5Nw5kG2yeXLBofm3Tjhba122sViC6ObbsOt4FM48p5aUi++9Hmvlv6UvW1i6KCUdnsgYtOffT81cAdGDdC4iba7nipcj6uNsIwRjlndCROxRmMQ4Tnz36ubP7/8u7HV8TmPjCkh5kyJ7/NwOvgoed2F2xoU+sklWEJE+HewIm3gbFJibgZzVYhsWkr+wrgy+LaoR7FdbHUaOLK8TAvhd33PPK1H+ilzIH7cZ6NCiq0KzzXb2zAPzCJGjXn/6J6tYzRq/9EJw7lkSIcc0h7xlNU+nFQ1vvJm+F8u+bF/Evjt4+uZX2fdI6hiFUIxTZhKXNchAFQpgHs/+FyhGX3KKeo6cxfc6/3hH4nyn1dlYX/IJVuVWVtJ7gBwlfWaDPBLnKzVunpMVN0UCTjF22sP/TnXkyufyK2PmQf8lI4spowOOLTL7n9OKIBfZRCZPaNWNghdLtl3ErJ/Cx4z3GLlyCbC2UNKu9OFG9tEQwNo/Aa2vf1dev4If6JU64WZuTo45q5lEbW4Pt2/vlIAlH5ZCWGO8mCg45R5MmzVv2h9Y8nXlUWlj4utqzFTVKTFHmzWElJgOXQX4z+2+dO4TkNw8U0QEjhYWJDAPP92yb+zsHD00ZcKzYavNShjlS3oXDr6NclqzFIWoVZ1QgRMkFTQ28sVESVnAho+SyQ7TtI+fYfFvK4aoyyj+un7hfNj5LOD7mD6C1I9b56B/ISQmeTmmYYTr/+jm3ZM2S4CptPnd65BL8a/nAn/Nt7QpSzn+o177Rf/nG1yi+fAxBUEx0n7ODH8ouVcaY1PXhKKRxryY02Lxg5ov+q0X4Gb9D22g2uFnH9jtz/75msL6NqbtbYsft0fGC9K37Ck9D2VV1KZJRKQjcnICTFUJDUZCPjt6BaGuw/pff/zRCF9CqAaDEJvKhaUR3opCQ4wXW/f+3pMRmla/+vid3fmb9dfc2v/aaVwnvRjeDGx8mWf6B4V1ZI3c1w/bgEl9pVavVhnyhuZF7UsvDlfGrVPak1PEfXf2vPnpPxN7gYWU1rnmDQ6wxlbWwYJKIEfuWXVq/e/5HyDwmJW9wP+tH4hntuG233wHRs1j7TaQC34vTZkZmVefC82SizCGqYEiQijDMvHgYAE5+R3+piYL5+6Yo49Nfu/+ToWF+2yhEZ37OmXIfePBMPy3SF1rH9jyRPiepHk1meL3Ec2i4Feavifkne9dYjJ6t24vfu2gopt22eVj+/dB+eoQWQgdArJi063ve6KrYk/06naIFo7cOXROADmzXzohWLyiThpxp7I5nSfKM5M0ZPFtedg8AfNm/dR/MFzO/Y54kvLtYUf2n5WVZ/SpONb4UPAAD89H/nmyDUXi0MqjlvNiXNkELf5Pn7qvS88hu6BRSMB9YhKrI0XdAaPlnS8F0+gVHIEB843gF3cO+fWVSqFam8F7J1xVJjxAhpKJbrCLp6aUM7j+yV/7Utkk1lvBO8igVAm5G1dLuzXWMgjYaGyz7/R+/S2AGj26D/Dzq/cTm75/ZWOphFIMXlw6pR+zHJ4+Pi6PpqDsl8uSzONftnnBRCjGT2Y1Q1kclMoggsbA7t7iREkBT9QJh5ru16t2mPTxuU8WVxdRVtb7r/U5L7ZxPU8JyVdybdjspqaBEHHYYiN03KIllk5l/q4fob2LKS2CffxYhFZjHSPxRJXkd9g3itNrJ8HjjGY5LtOP7N7tyFLRRXL6P+v5IF//xb4yqGM4TI66a2SdZWu/9NL0fJSCTiK8ciUzLSKsL3gb+6gPiH1AvX7jvTBzzLW5gWS46g9ADjA3OQ7+WBO+Z+oICNqbEkTGvP/rFch93we+h++d5sf+PNYaCGgTf3pbENX4RBi64RW3ILZqocwrrpFBUVtXhuyS4+2HtAh5seZ1vohDQqfbZ0XS1kSOLptnoBCQElGWFwwmQnpCx/UYkrHgR0AfrpYhFoklSkUfQYdxDzR8UmES3a7efM2lBKuH//3+SYo5GogQugatafm5Q1zE3RxufBhDhBAqyxdDpvUeeJcsu8ZPAhbBzxAH/cipEp/+i2J/Kpd5p6BuRJvPofhKkz15mXvezOXV00YTLy8z/NCFYwR6NKJ7LhfB7G5zuP9CLWigQDzYVvuA3KchfxYAaPPfyB3eQGLxw5qbglKG2wwC1L4tt2/5EJ1lxKNThZL1kWUw+3TtvYeIFP6l0L1BtphWpyS9vCBkEPAVr8I/lGvL8K8xlDqmD28DVBTLlsX1LSkPqHxrBkaxLkFi0deAZrbmOXuekCrQy+/Dkzp6aY+zzTWsHYGNhPPrDaYtX8vbzgl7vK+7/MoBDzpdH6yM4FLMHfKovV/D+Ai8C2clkpyFIBz14/dvyLP7qJ6gxcCpD+JHFlSFRJuTZU1BWioXttfgrOqfvzXKjgf8WgDvfrxaYDM4F8u1hX++BSrnGxHkUpM64q3LiOl8hRQTwIh1LKKSZ7LegEQTnVBevJZyEjD1HBArfDmJ66EsNld4QQDVdZOBr4brhw+WzMfdhupyQJVgUqNOVskmsCm1hcGMT5+5Ozgh1VMqbgxcDCOpxM40mswGps4ShyUr56Ttqyc9ULK8C/barhgqwg4UqAwcDu3vLdNrnOLnD0jY2e/bN1CZqRHex6VRK6Nh9IipOOAevpXKRSY8rMYBm7KTHTbBKN6lyhAB7eqfaLa4Balimb/zjAiOi1EhB21/WijZ8z87ysXvSlKcycrelsmUxXOkAjvaOofLCFMHesCAFzL7d/lIzC/W0YaykcpjMEtejBrQ7w3sZfT6bfHvsNfQrzkt8zBC63gVtQpRltCYPWj2d4CZprqJpE4Nt8l0+Q9Gls/dYepgsyvt1bfYT9GBMPiV57IzwJFnG+tUj00NlCEYqThF7nLw4SMwoKhptgV0WaPkwiKmkq4dTn21efBwaiYd5Ob/Y8NENiGrXF4iZbq1HXWvPQ06J0yrWwSgO635rF9dlTL1sgcWtnuKV4Av54O5x5d9eOd8ZHdc3NsS0QTn6hj78/9dWnLn8PEXKs+S3WwWklsbDZ5t+wODqu8Mk6c004U3C+dnYWnn8Pnm9+0gcD0d+WWrSjpIn3r1xVY2/kJ87vcLXMQe97/xCA/stWZu9lAWeSI28qnbuMNPG6y8WjyPLwCxmcoHSA8jyHHmPMRHO15iWZQlx0pqedmsjYutHBaI8vxEt7V10eu9wRM879fzm087XouoBbPY//ICZbfv60JcNAFjjLlvz6kBuKvrpBoRlv1o+uG32YQRwu4cx4rCU3L9QYQffMkiF+6OnBzkjfNiA6538FHbBpr/5oCDjY2ecRnVMUJ2hRoVPneMF1Sgljn6RagE89hUCfF0WPrwO3Pe/fRb16nyaK7zWVv56MFar8NOIuRouMyDyDTaGHjbcGgAjL0dTXd5RtSggdK7l0+x5KNyVwi/jHUurChydUV5Ckq/fLNJaBqea07SxXBAf6Gn8+/JMjSM5EXGtUY74oWvRJ2o9fedStGoTuT7jPMc/ttbp1qrO7BgOBp0t9z4Dag+VDy0vZ+MkhHC1bLqnbuMw8mcqbjVECv5lJjdn3b5qOfHZlj2ORdMGNHuwaiWYwmBB4GvwmjU0V71RhCO9y1C81De0gAKGZGPO+/CQB9P/l5bLfmFDQkVLk4TUlpFD3U4CdJXRB5f8UoSURWqTNMIKLwoCrwx2meTqz22h3PyXMpIc4hfgMaZ0nZmQiXGILW2pF/fo4mbWdrz1Xq0aNLK0Z3zYQIMOqgh2wnfbbs5AYn/KuO/me5v4BhalXgasNL7KwTWgD2peyoy8mvrhAEy9agGZBiZMJTPrQOextLFqnwRlqS6IqEgP8aLcl2BT11P8SJqz3KM+e86MM4J1So63rM1vkAip/ZVgk0W8R0dqbriovdNPXNnVp1ChaAHw4UjWHHkrG7t3Dz7sxqp/PRffqnM1B5ua6MwzrzKxzWkGriOVCMMiANEPpJU9rlH13eir4lBa2+0ZQdiWrCBSpdkF+4+1FqLZ9Zp7/LxyilLwvNA9+lYv9OiI2r21m6Nv5OPa2ozFRvAZqpaBY7FXC+PDm4085UiSAiiLLLx2JwiQy6wojWql1AkgmSIRSoD+X5tinJyIsWrVvY0GaULObDH/unIglJPXbpLAEk1BQx1kMn7DaTBgRel1cgKWuLaZz1kQsTK+ZVaiLRdjfNlpvFfdVKtNpYT1+UzdeAaI9XAFueh6OgL9k906t0OAylLwmwf8NH+tdImcI2sjVNOruXTigOw0OBbfhWw9sNNuea75lKcglbisSncOqVnH0MyOAHRzwvpvI0cBnykaOm8rC7+LradeGy72/epXj3tzbG04YguzI8HoYDAQD3gfFCE+tOtxcdMYAzsiPQbSxDe7pbkpizTlyG7WOMue6pPDXiB2ISsYDaWtQXnX3mq2SShXdKqO0sR2NTq2h7VWKl4i4wRGFCSrAogXVVrkOKr5V4Ne6sKzy4RvVy5VeU4g9LACsS4jMl2A/bgVSll7dDUamOAaFrZ/BEWhId+bPUQTpeoGaDm4GDHW0+Prh27h6Dlu/JeCTKMInMoJlx0DF2eWdSmrJI8fIPBlhPdbgStbVhBiNdtmC8rfar4FZqJhRor5ZswxHjaV9ig+cMLGuwWY19DNagqZSumoL8Kk+z5lgljEd8uTbdP/4nRuJpjagb58wc5RlGY+6BgOmxLrAwS+nA3YurQEbjeDXjpnS+zQ46TLIvl9usBTKecXn+pF4VI5FaoBwNk91nkGl9EDATj5xeM/8eBZh6Z/BI3lfjTBf1PU+LH/9iYrrbfmHlrL+nNW3UNu4BR8KmKfTj+/bujvOp/h5zgGb0m1riscpAxux16t19+Q0wxSct37a4Rxo2GGPEPjprLW5njc2bDfVC/Kb/0lJzQsc42X5giz8bWNm63tfuyNt8SLABJKAdtgpQqNbRnXzL7+Wo1XZfaSyAetmpTlRbuiPjwq5RjLdcsnnuKJ0hFL4OQeiS40MiqxESFJzePuiedrWSzu90cFPuD8fI6NhHGdF8aSJ8BFOsJ01UxTaJFIGOiYxUP1g43DlaAGgj5Bch/851LQl93sLDyYP2de93E/agJxpSBybGCpEl1JgPMIZsQZKiW35GqckWpKpWsoNQErqNkeBBmpnhRqSMGHGVtPpFqXIajFFxm/v2hKacSs6Hr1zMwv0+PqWJ7KBzN+WvzYyC5K15MnG2QeHzWidkBRYe7Oul1JGd0z3DB6uOz97mXfPeTYewpf7eziFRlc387vNOasSot2oLJoqwEqDpd61xrAhgt9GzoD8fxPGrFx1DwUNtDa/ac1yKVVzgw75wAKEHH4SclVzr9za65hxFhBrcFNtJvTgyQAULeCRKUKYpGcKTHqmNZQAYH0q99dhtOgdqpbyRlKF/s33UhlKey4j/fX0EP5u6uhLBeDmId0Z9zkdIHgB3L3b3BmPu7PdYL6sjpF0i0/VBPuqYDSRUMyixR7zES1iVXdRXieBKX0azforhLXpg/HejgcJECwaBtNSB7AB1Bqidr7SLAYcQYfCqEJ3KKkbSMrsE2Qu8OMO5kYX24EUq/7Z3YAW52EKBouryx4iwfXVq91gcTQl1W2JH9DqFx8SshIgF70C5zFRI6m8bQB9onM2NQlbepVAEEljjjamJj7rlrFHIj9c0uX1IpoI7cq9kwy3ALUeLDgildRTd5l1M5c23UkN48yJTOee9tYK06UPc7hzgdF3ZxCAnyafNO/iRuVbD+Faikd3VSYQFbBsgvKO77jhJdvZFIz6Dqc+cN2S5tbHXxLrq4WJl1el4nx6IUQgxLpUgArCqF9EkCO6Wdear/QHWEtKhePLJB8rVrqaquqX5ZaCFL06g0tZ2mfnS5MXXqN+okePFJpHyIv1RXYbWWmkM5y7kL+EuE07dmdC3VSDMmXU1cYsB1PAnGrPSlzKofe7KWz8mbg4n612uA7LCEp36WJtWns+kKYNVh3civQJLV022b/fiTRGFnt9pvrwiKo2toFBRmY0yQmvi2YYFFnr1KM14OLQ3dxYDho9geurtF0ubRlp7pHo4unQLkCw0Uiekw8wH9cnm74rOm2orSwQ7Xmz5aC253u7nejOocVmAYjs8Xwu655wF0sxeCoKJJqnrwFNweZ/9nxnnlfrO7NWPx2LIN176ajOooK4vkJdetVlmvoiJ4ZeHWVHe2oxiUX+34hbyiRmSZmdkDR9uT44dXqkDb+TqBsvqiYRLA8Een7vkY7kWxlsSZx224572LFzrcbwimLpOW+kqDLfOP9rJ2S+l/HFkQTVlhwnL9dXIJWgwpkYyzf+KX00fP/xLoGXFaCgsVipVIrYyrTpKUqXd6dSa0R8ltQq6imr3Bj82TJ7AZyIUh/347kBJ9vTE4TLUB0VInqftCIlPBlUhVu6laFEsLm8om4DQ50Jns8ZON9cHvXDs10mqtdzPmQNDMBuHgnMkLsZwZ5+Z+KnIVJXOso4FXaF7HSc4BHF2WKiilrIDeHbi+gWN7Xzm09slaxpZqjVXIYrVYRP6oS3GCeHnlyfhKUIc+xW1iO1PUJGDdActLkJGlAxKrfiaRb+l+oqqRrmLQbQd5IGhC+vB/gb7+T0zOBBfxn7V9PsKJOVYB5HsNfBCDCVMuzdCpyLCtOO3LgThZ2kwnlUfCApeURJlkuqknlODMeuH7uahBpPekYDr6Gu9f14rSxIPbeH8FCXGjJ6Hxwu4fxNVyJ9+3fF4zRG0O2ClalvpC4Yq+kp7ub/ZMZU/PzuQxWwiDpuAmDTps3Nc81OTpCvEAuSS2O6LWLrj0+6CECpz1wkOCVMPGreB2daPSr22qKwsO3CDbKz/9kErBi3YFKrmNsblubFN7zIGxu5+sx+PmHt4mt4w/Tu7nd4rR6NQ27HVMSNdsSQXj2Xm6yzS9X7i9DpsbXfUS+kj37pFBzmZtfnv5NgeYYxNxDKsT1BojoGp1Cqld0hS/Udq6+eILXhBBfteo00dEH6hUlVagP+o5RWSx8AGEw/3dVRTNc5POJv6b80cLUKF8EJCEQFJ+AYxzuyhPwGThl9avHNkKs+zqnyEDBOiNMGjU0LWDcHyq+XTB/WCH3NrgOyyPDIxWZ7ZYZ3ZmGxelM/L27FYT1CD2uOu51FMylx1iVScGcfFb3EEt0v4KT4FCMS4JAEBWVHCJAxxq71VR1DNVOvTlB1kolyVhDr1rXPLjhWmOf7+ft2uRKSE1qZRmeMYbO/Mb4RL21dOpe6rWV2d7gjFxJFDzjo7iwVYytgFU0kDpUFOTY3JOJv6HENPNY5/H71oHUIVzB5Qx/LQFKuHgrbz16g86I7XdjRSkm186qEClopMW0AZLYamBD5oZB3nHvcilFs78R2dH5p9trxJ7j6tflELcsFAEYtDvySjRc+kQg+Fw7K+g9fGlywdT3ouvNC3sKoinNfTHxZnj8QPsZ7PzWUlUJPZEe7uYjSgjVx9uO81u1LKBUAxpNFRQJzguCz4SpQ5e5hX4MoTz4yVgY8fZWA48QNS/Teamjqu4QbEPIemoYduuXrJ8T4z5jC1EZpzQXNj3xFOY/fPB1hvkxd67qwNEIwuqCg/VS5UM6otF/8Bt8wfSQiuLqOyfsrhkCUPPrgQesx7Bv85TopRjHWch25aa+6pa6gybgDhZjsbfT9ahRx3cbETeC9LX6AfUezCTTDBmebG22MuOxZkRz5q1Pphoa0F3xx2GE36v8JdTzsxXBZmriFz0zWh8cKdP29IdnD9WUDw8rtRqb3ZJTDrzetlcvKA5aVUu0TB1z60VKen0TB3/0OcIf9flj3fWGP/sx0cg0VrF4ABfjzc+iU+eCq5/X/U3uBVx5etTx211gFLA9nWOA22udo75ziMUfHTKOxkhZzbdvb/DUs14Z5blldgax3UO5Vesn1LQhJNW90D6N+KJYEM6JX2beuDrcXFMGgMaDKmUyuCdAR2oQ223daqdkqvb7kVeyG4mXf0nxTU5o+QBwwg1DmSsEOUBOVQ4GWyA6uReN2pe+PTZ9ojluqpaYyekdJmlpya/Ruz6t6+7c1Yx8JxWM3nAH8L1F6B2gQoI9mxs843K6xwlZr4+c9KgQ+QCjlH4y23wbbjB/P29BfjYEWtjgpyx5nCL+afK6BTbLzECLXTG8nX7Q7pnvQ+mLzoTB3jjcpoVr/GrMwJe7NhbQ3ZAIwmFd1LaUJv6sH0KdugVw71Z05frSdf4vFCVW0obMDwv1Bj0guyAKObXh12QmRr2kvb8C6t123IcQjmWkH0+hBUgq0qyft0BtSSGQxE+rd9VDmeKVLfeIIA1LcV9YRYVAhD3fa8LrmHs8IQiZY475Y/kd2C11PlGDMDm0+KpwMX7qVhRShHt/EiozS3cnxEz6oLyBL9NJunb+JNStTyih7ftT/6obxyQUXyq2UW5fwfYSxeQR8Twqc8D4gw0SIOcUBuS8bGB3hf9Dz2ioHnfdPQtdWS5lvP9kz/TmB0h4HybjbCnmuy0jKUn9tkp8vUJ1RbIwLyGYk2v8RwvwEGxIBM4CTNlBS+mDvKSVK8zfRvichXdGbjH8lNxCNmEN/GakcpjlZ5XmzJGn9AW/daJm8e8B7UkVU2UrwRtZ3jydrP0phUREyahzvJcw4swSTkv6CzhXdbkAAnb14iWEgIvwLrQmz9wAwlZqSuMHBAkIsLxmCzGKNNjYL7dwdrVgQHihberG7A+YyYFPPZG31G2lLEykHvynZunQUHvHsxO5lUTUiwlNsGZN2VnAxVgWagaVa5S42E/P0OqQjjMPI/wRlaUmNPE86nAi23j0GyV3d1Dsysd6uQ6pWWWPmFBu8GY0nNU86Dqo7rFE10PU7Vv/bP+f9eDESxA9OE/1ZS3kapRS/jouP241I963zy896dviI4VXVeFNzFI6lGYMmdqdI7FY+X5u+nPSOBFc1kxsaSre1873GdWjY+oKAtsdczwzql3niY5pveEXXld2piLCSzwIifUrq4+I+jaSF1lFXvLv0Ce68jlv56T823txp+bcOk4ykG/2hLkA9t1UaS9hs9BCh/V/wF92FVZoU9r+i0AT+3hPeiDoaxg8ZTT2Gs3RSwYiCkWqwXuZnZtAL4U4Ua4c1mZzho39TXZ0fY7VlzVWc4wHHi70bptfSHDMOSX7fMfh+qMlVqDHxu6Z7nci5AIwTkQTVpBXxBz8GnCqnxYbC7UjnzXlx4XkJ2xZQf//1/rK93mz7Wtgv/rT+8l8f+vPxub73/hxxXhoEjL2+C2sCuTZTnG6Hup0MlY5UGeHHUOF3pRpyCjD7Na0MaH9wiep69tr39fR85VxYkRr1t6/er33ggh9Kot93zc133RV33T9zziD/7uf0UkmxQVXZKlUGqlW2bKtbJQ1gpen6jTNWyV/Qt9si+O/8wkmAfHZ4GztsXJyWArZ2JmZ2N25+dQ8FurN3u8GfOm983uNye4037v/P2DBv9usLaB7/88a/CyAc3/wXnGeZJzT8qplFHnb/Rv6KfoW+vH6XfqSXwf/qH+pf4X7n+w4RDMwHZoBg2MgQXi8C+UQN0fowDjBkaxjB2cxm1YgS0oQj2OohS16EAfRjCDxfgCq/AdfoU/DVHZcGfkY5zsGoDMWUUPlnTapneCi81A69x5ay7SGgML8tOLzXaZWuXbAlFZibqXPgfmne2ALIPOhJYalhedfcXg6M98cO+SnxmfTwVqbkRpWmPSrWVqwMnHRlMFQKsTodidS1O12WsE4+gJfm2SYPRVMSDSnCXpgLado3Zva3ftex4Hbv0kor34hxe3eEcXsf6zWxtQqMI1ftgcjjUWTw1jHRFLTmOkT7Qtcofa1/Z200k6bO/SbfXI/cPMbnnbJNfD4j1Mm72i4RZgwUlKy4YUWM19u3cL7PcP9pdRbKA6eNQdbhxu3pYSeV10apukMNppg0bgxVsV2NnuyPPoTv/52XCWLNzeOqEBPWMgX705rZrrQMkS3cVPVzcKmaMXn/cDqVB1Ge4z2SNqCDSqQsvQ04TvFpEr3/mQM/XgaHWQ7zK+y0eruGws7UrEz8y3vUMFMdNMvIbcFoO07T3fcCtOzQK4zVDogfogyUojN2ad8SjL6vRqeH8MKVU6NgBTZf0rPag9MGeQdaSxe8g0LM0XrtoyFyIE0c+0Lnp7WJun27gfx0cA8VHT3yDwL6fQ8JOLPMBWNIVMFWlbmSx7pfN55bCuCTY9DMci4Mz3aAGkzx7Lx/D81RHxq/ete9uL7u+6wYWHndPcsN5m+z9+8mjzrbo782C6yj7CnpFhApRkN4dNUsIgzl5oFupGyGzRiNhg/GXagTqjysm8whk14/2XH7rGToH15sgksypbPo9/YnBMCizui1CpjL0Corn8NGSVlVrrho1rpYsy+W/mFt0p3pPPGx7OYhHrHOAZXEf8GyzYBb29eS7BJP3IZqkngM3lT81jc0AwN20DvqRPS8RwxH10WrpR+VYbDVnNl6Q8zRMgpmXHZQjc+unnxvU1Pmv0e04waf/LneTu+cnfbiR3x2TWsTyKuEXDwaDuwEZG7L933+dEXnpTo8K8KEn0bMhgSYugnpJF26g2r36ZTyt/rQ2JVU35JUBMd7qON/94onSudlKDliIOw3375YCL5fXCcNggA9louHE9EFNF5T2JVo2bpqW3u9h90gYGxtpP0XIzDutcifbePmM6frVh2b7bdXbSTCITx6vNeHQhZnGsZMZFLWd75x7admbVarpC1VutNM1QGnuBriQ313Hq7deP3QMyyQijw28/czpUztdiIZt3aI6vXeBHBhuXdVaWn6A3KC0TX7eaSaCnDjCz6oAmWvWba/nypIh7dFQgBq/4IlBK1s5q5rin0mJDjWzPjwI10QOzT9Sgu/16WbD3uZPp45X10qRwSLheTHwMFDWMqIaNeqwYxSHVETvb764UtgqANguDHSYNl565XExYNIO7UToEm9pgkZFEJZcvrFZLVEuub9JuUFFKpH5EPij66KNkfHVfgzStiDU0zOqX92XqUEGreuWHj0ktW+9juoyDelnhXr5AVIk6yymMTQDalxihAlSRyHWG8Dk1cvwoZ5E0ooH9pJqb1b8+M8Ep82z6JqfAs1e0VyZVhDh/YKf2tf0Z+5FjGx7u7N7uP+T7wPHGiWHI0Ws71YrhRQ4Asko4W4bM9Tyq48s4uw8GmUdUQbgF2HvKepTFeM2AbNFASNqiBRWDaCpSP/fQk3HuTdv5A7P4WndXs+PzkpxpaFUCGm/UVRnAkqrCkiGvCcnc2s6otp09Odr9tB4qmeF4as9DRou37VZdvk4mMpgHxGClHSST491Xzws91oa4beFijVFssb3UUlia0I2CBng0F+swGNq7Z5v3KUrlyHF7miTZWaZ6NqohtCt5iAytagGLi1PaCPwic7R9lrkJS2KiJtOQdYjzoNTykV6HHuhwkyzrRhfUMs90QgyXxZ2Y3k65ySfh4dYYTsuhn0RPzdZUT66xzIPqHxKrxZitc409VkxsbFpkdhouG5K6DotnhKYp06zAScq07/Z4KNPnWCAHPFUptiRb8m9xQS20r1qzrrMos16QNt+Ru/hdvpQomG684wxZ/0JlvbwSUdAtykq5vFDZzi8PWZnReCfUa3rVaNVnBFrZADPxjODJnpKEbTfc+Z0bWAFCqSjvIP/EO6UJbOaw6qKLzvRGAuPffMAxTVRsi7K0mis1WxXwxlymqnIBak2sjw9ggdoXq1cUmrNF+mQ/NRdYaIsnxwFiW9g6DU4D6u6St4osJm7NnamP66nYx4v4hbh9dhT3x7uTdeHnPB8TJ+WS/4/n085y/78RaReNMO/h0lyjXfSs+o8fM8OdqeS0QEi/Aa7/37dOiI4q6eMeQerrobOvvG56vHeLquSz3lpzLW13ZneCQHTqvtaLgnYw6LzfnM9UMfXE2iktPg+gMSExC0sUWwxINt4CfHfnxv3SAF+1L8S0kuE0pbTw5YDNnrbw0JoJRroRdf3S2unoUG3IR4Wv9eaDc6GaGKXIsqTUlGnOyQmktZFv2avsZfUyeRmIzEtyzI73HR1XLwCXtRynmJssksc1z2vlC2Ky04XadX86+8lG8ODZU4mJ+px/z4tCkVNB96UuGAa28wZbPtbWzQwM1peOMREfqAPhfnVlpcIeomPjUr87H3SISI1rMY3PeJMsueViVX3Pl4YS6NC+YSLyDj5pSTacLPf7KFY1A9IERbSHLTpGaAdRH6k7QjYUdRP9IPubjZGZrUdFqpaqFdPIMZ1WzGQxTwqQb8dtctW9LoqUksyr8Wa6kdnPHwvDoUmi9ePqwZAlGpwu8HoDTcOwkJidwUCIt81W+qAR3jhujLsPK1LzS2TL2vIxZ3HLBHnOln08spMobW09WE4200nL/b/jjxHSfwPwMSGqqre3M6JBNIyTce5uRWUl5bonlMtls9J86cz+azeu3t6aYMDQYN8ThkJD6SsBuJqWu4pJFaJxUcXONQxMy2bZvplmlxgpX1vBl0uraVq4lCSrC62yQYF5a51vG0TQuvLJyXkbnH/XH806ezGO5AiXsC1k7Hf6YApyuapRlUEc7Bd0pVXIWBU1u/sktXlWQfMzRq0GHMh6qrdixt8b/yli3XP7M/Df9dofo2myXGMiDRSmauLwnBHitBUZoXN2dMBR3/cdXkNN2bxZ0J6NrX3inrnZmJqKbYA0AZ91FRciqAaNz6RQ2tIk5BlZSiVNAhR8zmb2uKbOpi1rWP+itAfYWh4DwhPKvWVTUNaXQx+s+q2oysjS+ZKiAxrPhyYMR2zLw4cVhHdgnErbXvPQ04uyP8kLZ1tRP/kxhw4anFn9ljb09Ky51J8XN6Xxjo3/uXHHLM7DZmDMUzko9xyY/e8S/Th+GtEIfCU8o7QW5T8+2OBfnKA3/pGsivlOJyTXrEv8TLBgdaRL0oBh7w0hgovHTsei8tAkMa6namxlQ63vgzw4c3LQ7aPWgalnnMc/cZMdU0V/s0dG7Bx+NRkFzuhab8zIe19oERWwW/lUjRRtKNteXtaT3Qx1u1gCi5gebRIs6xODCp4u02swYPRArVYLfDl9ZzMNTtFMrTBrppoEtuwomeVFF5EXHZqKCnN77/iNRc9RUdulm7n6iaAxeSFJKBil5wKb2jTjab1ai8VKJcNoBAqq5INWr3KWKdJFJzV58mcfSHNLfSZ2QDxlCzZUvaqpI2e4McpOzHv+OEvoxqgRt+0ioF1M1mZ0s7aBd6T+wpYU+UMhph6tDbB7vb+wi6cg2fP8fhk2t4T2PFpZU685M4A5uEBkzO1/La/Sk2d2FlTvKMX4Qne4/mi9eSEEAl7ZqcCIVhYLaAuvjAlXAVATHzQRLVFSb9LcX496ZlBsyIfFdnllCbSrMRODZdLBfgHpXee2IhMQVcSBfYv7wYE0XLi917s9i6vvWzi1EhChg6cG0dvZ6sZ7Et+Q/MRaLgGU/TIm5xZSHb7b1DkKcpeiTdp/nx9E9Onodx4ZAntsm9d7KI38JjHKRNbBY52tXdaCRsGFdCSU2wzlazqyjcG8m8j5sVu9a2kKI1b/Y0OQOwZeleXthohhlZE64lKKZroHztCQ/mkwwzg1z2HZfp8Q2LjPy9YOrjhfmzqmhbN/3ZhVAPXhw8X8fX8NzOJnWmAuvBchdE57bpzl/fCjt5A9MDZ+nFARnEL1aDHZ5azQ7nARcePk7nATNPV2MGaB26JMxrt+w0Ba1AMXI2l1NYmE1RI2kG1bRD8GnMXnC1AodeR5YktX2g8FZj57/8ZNSyEfLS/nN/TKUE9HjbDaxc2KxbkF1VJW6jYGmp+Lz8NS8qrxospsHfe+UcKPP8cg+/d8ixw+5ilaOELT7ubxp0jkSGGeDpDDwovwPyLTZTcdsWmmVgiOQlGhW4GHDVMwoVGAhOq5ulYDivZRAK0wY9zfdb6m4G1KeqM23emKiHdDw9x7H7leTAAWMjY0CGIDl6HM8rxjjyFkLPJ0h0jvorxKjoVaEu8hmQquuLZN3yJ9iRlfmEyzGK9ZJtCmnWJN8PtdRUmfm04XEfWynpYeUubMZLtWTKWHLvECw9TD6U6yyRdYv7GrHJjT5GdSLbdwB6efZZFlYtpNw5UNidZMCS9SN7oZPyat8e0JPfHbQuRferR+iB9u2l7ba6pppqzwzeHedJsb+gAloo3k+QbNf5lMFy41BT+9EePOAITdab0NBeZePwc09WuKM31vP2ASbuEH0v7uLWZn044FxNPjKPvdfaCERad5I0MpyJwjYFqPKSGacMItIDhU3n7Qqn8XOL/jhK+ECFGa3sssuTrGPGMGXXEQ9Pnian+POTptoKlN7/S9wMKSczUSTJz/Jc+iNgN3UrLGcT+RJdAzTG9y8Bpesy6mzpbm6bPQgodPZIA2Pk0wAHxwMQAIkjo2AKCejh8AwGA7+O0ikH3wHQBM9CZyCpO0fIBvx5GC/yZsy1hJAQA4V/8oAD29XwBQgzFEtsPnQCqvKJohkPDoeTP0sX/eTzDM20YxAgoGDgEJDToMmLBgw4ELDz4UAoSIECNBigw5il9tGEExnCApmmE5XhAlWVE13TAt23E9PwijOEmzvCirumm7fhineVm3/XA8nS/X2/3xfL0/398fSI+qNYGafhKnzUar0+v2B6PheG19c2NrZ/tg//Do/gOAfFZJQfoEOuVq/k+cmgJl2gEwnViTNJGa1HAAQPsAyFJ4AAwYtEjyYve7INu8/ADAYsVawoRY0Pt8yJd8zafU2w2Af6MDoP9RYwdenpPP+RayRJUi1SrUqlOvRpNmADRaZrkO7/TLeHkZm4AEmpsJGRd/aGcAEMcHhgIAWBQZsNYmg3HpH0xF+QvLz+VO3/l0L1dmMlPZm+N5OH9CJhrGHIuscJdfcWRHuL7x/xmglqmVaiO1Rm2ndldvZsJQY4QTc/77t+N//58YxlHTZ431ttgbtwTn17mIsU/d7Za3GtXQ9g504wOhbDTimSZQTc/t5/nPYv1JLYwuxj8F5bVZt7pv3a6v6xDczd+1tbpW1tI+wnEOhd8nJHVsypGjYytG4dFrpB4+pkvn2+Uvyuh6J4ZvdFwnyhff+h7rFwVSVTqJAgUUKKJApc7sBQXG1c3dm8VHn1FQqJgRjmJQLDcnMwdOKrWG8/tWf0n7p9uKzJWrsmbz9RLWJn51K8qqbtputz8cr641noylfxqdwWSxOf/vH3+TUn9ESv//94jEkk/1aWhqaevo6ukbGBoZWzOxbsOmLdt27Nqz78ChI7ccO3H7Wemiu+6BEIygGE6QFM2wHC+Ikqyomm6xBnfsDqd5Bt3Pg2PD3KPyMI+Onyd5egyYCT3L84zkRV7mVV7nTd7mXRSwETAtn2Oh67fwiNieE/cjnNIMAQL4+Dhk889zj38AmRfi8neu9fzNePf5FPA37Z+NlqdQqxmG2RAgfT6UeyabsyZtz1LhFeEjCZgBfZVevMDL8WzGAEBOs51NrLvMP3SQkh50J+GGArKZ8gMBjasI5BubL3vXdnZyAIDvCb0NOBWjl82sddszGZN+i7skhnwCEwMD7+IhZVH7jMJr6GMq5ppwGppaGMEwAb9+4rz2+EJkZtQZEraApDTWReP2R/S9nFu3CGun6CiitaQloo/J7nWOXZiEXh2jopChgDsG++RbYne9+tkv2Kmxxn01Doq7VSh/EfsNOzdoxmQ+X61fsX1HyjCk42d2ZP9fECD45tN6nShLQt8SZ4TuVoFWYee04lyZxQC8kiCpI/AmDSwPwjz5W0/gB5yZgM9KHE9SDQvUH/p3SDqIvmm6rgbXf0lqx2g3drn4/YTaybMgLNdAzrWE4sDE9zupq6gM6I2blFThVoLRJUR3kGP7+ZA9WvXwYQGoJpnTNhk05gTxcLyHUQWxEMdgkkgkRyDXmGGCYQFet2jvGFo7XrZ7iAQzzcArDqepxGAFBVw3YzaHprhh7bDCvA+g53Mk5HbW4pSJUSd+BTDug+suxPBKg54u+RKaH4F/YBzE97HNMAEwv2OmcDNFDwJCjIQ8IWZCnK/8Kt0Ibo3i/OZs7w+M76vP7T2EAscLqWJfcCMSnv1LHG857uwv5DjN8Qors1X7bhuTz87wik2OQCpMdYNvxL5bquIwK+6TiVKPbBL3OH7h9Rb/3oqJagkQghHv0B7bLBjlHljo2I65fMtl7Dlvx0JotXkSr1/acoVwidDYcmaksjZSjgtwGWqQiM77iXGBPZuW3YzaXRHdGqZ2oqkE6ySn8sLG+yauHAeSeLqOAm6EBRWzmCiuxgFvbzht8SruT26xOwk3eCxb6v54fiHznH/yOhzYyaXxdScuTxcTpAcH467LwvxEK2M7+w/Ly8jYWDa5DMZrXwzelRoIUlfxb4CetniasVNW8J4pcrjWgEN7XGoUdhGZuoLrBBj+Cs8Lka7CPXChvy3H+zmI7Tc8d1KpmMUtqwkeU308ue8N4d3XYO4Aaf+62QPQKfQuE/oenB7O0BeYuzE9gmgDui24Lsb3P3WvT5FuIawKM2qnK0ZVSmrHyEsifbuPH2/0Gde/f69kOlu/AkVIuX9hOX4YMIrjhZbuDV/C3Y6b1gOKLDRcn8PSapQf9q0wVEYpPQE8ZBzpa+ojxc8DKyy2mSAABb/LhXGBNL1m3hHKydyX7hJFA3w/Eo529nz2LaXP9xHNB2SeZC/ajTXEqPXamyOmz7GiASgmwEKByENh5Omcx0Qg4u5HMA+VOgIJiqtns8d1B6F/sAcB5jXvwliy8CqRH4e2j0JLg/jZ0b8+Fwp7EeEZwDAkw+s85zrSAGQHcYi6lYFMxdFR/B63tdMod2Gu7ejTiO47ycPXKq+vQxlhEwTC4obFfvg/l1NnIs+2Bm/PhLVdMFUE3Vr4hgJW5+FxQiMfHnfcje0XXe0apeCGA1nWZHPo3Npf0fJi4jB4mnHKoGJ3InmPT/B8ddJ32xftb9UY4QHCwUcAoDT/N46ML7AvgmCVAhgMQShD0PKP5VgexcAwOCxlOAw6GYEj68pIhCDAoEU/MDq+wRgDmYEGM5rIWLRnwdZsZHEAArTRAAClZyEkdC66icGwO8kQou56DYpodBmGcrwlDtF4RpDZ2YzEcoYZDcsNY3RKH1vGwHHbIZkUfcZYWr6kD9m8svO/jVCv+dDs4tveUJ29kdsblexxWt/fbLTC/dG/XNZj8K/eadcz9tqJb4uUrT3t/yOpic9MnM3sAvRqUup243FfKT0MKOc+VMVHasvt2TUKIeqNECoWdF735WA8cqSRnZSgt74lse54Tu8shUjoPJmmQ81IjFGfIeZrjGFCidYcCQbl3RrJcRjhcYI1AYczHSLabJvWuq4PehMSXDI42U+xoLrGOgPh/ihbOMGFdQezf0tcfIqDAT5jmcBLvQAFFvT5VuZy7pIYaFzy7zqMUlT2QiIiTkhJf+BEUHOaNLchX9LGhTdILfMM8WGAjERslyaOObp19wMe3xvT5+e4L1yG94Vn1vtJ6E3BzqT0x/iQOGNgrggGEnmWJVLrza95BFYFdZr0pqn4xxnAz7ZPOrvxu9ln9GCHuHCq4qq2L77Vp8JuQPrgG8R7iwRQrDcvyLhL2T1n5FNGYDi1WhnLFfWeY4GC7lB0PGKUoMbrzuuEsy8uljTmTmNQAmvoaLLZQOmok1jKn+jxKpDVzuCgR3FYa2Y1qrBICINYO3L1m0h4Nbwz8Pagv0tjAAAA") format("woff2");}</style></defs> ++ <rect x="0" y="0" width="1209.6444540126743" height="730.1247574550649" fill="#ffffff"/><g stroke-linecap="round" transform="translate(449.046845608731 593.6612861565363) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C38.27 1.54, 69.71 0.86, 123.42 0 M12.25 0 C57.08 1.25, 100.41 1.21, 123.42 0 M123.42 0 C130.05 0.87, 137.62 3.02, 135.67 12.25 M123.42 0 C129.41 -0.25, 135.23 3.57, 135.67 12.25 M135.67 12.25 C137.2 17.84, 137.08 24.32, 135.67 36.75 M135.67 12.25 C135.63 18.87, 135.53 27.61, 135.67 36.75 M135.67 36.75 C135.96 45.56, 130.18 48.49, 123.42 49 M135.67 36.75 C134.67 43.8, 132.19 48.93, 123.42 49 M123.42 49 C95.98 49.68, 66.49 49.35, 12.25 49 M123.42 49 C90.86 49.22, 59.02 49.84, 12.25 49 M12.25 49 C2.25 49.89, 1.02 46.81, 0 36.75 M12.25 49 C3.81 48.52, 1.25 44.93, 0 36.75 M0 36.75 C0.91 27.08, 1.15 18.34, 0 12.25 M0 36.75 C0.47 29.41, 0.25 22.37, 0 12.25 M0 12.25 C-0.6 2.28, 3.59 0.5, 12.25 0 M0 12.25 C-0.87 4.69, 5.95 0.6, 12.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(454.23300001903044 610.7636079418028) rotate(0 62.647987365722656 7.397678214733503)"><text x="62.647987365722656" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ListPodSandboxStats</text></g><g stroke-linecap="round" transform="translate(447.35828363198516 671.1247574550649) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C34.97 0.97, 59.23 -1.52, 123.42 0 M12.25 0 C40.77 0.34, 68.84 -0.06, 123.42 0 M123.42 0 C132.18 -0.16, 134.7 5.15, 135.67 12.25 M123.42 0 C133.44 1.32, 136.16 6.03, 135.67 12.25 M135.67 12.25 C135.14 19.88, 133.97 29.85, 135.67 36.75 M135.67 12.25 C135.08 21.23, 135.94 30.75, 135.67 36.75 M135.67 36.75 C136.32 45.43, 130.5 50.16, 123.42 49 M135.67 36.75 C136.02 43.05, 130.87 48.21, 123.42 49 M123.42 49 C83.36 47.82, 40.33 49.98, 12.25 49 M123.42 49 C88.35 49.03, 52.52 49.67, 12.25 49 M12.25 49 C5.04 50.61, 0.69 45.22, 0 36.75 M12.25 49 C4 50.41, -1.68 44.04, 0 36.75 M0 36.75 C-0.96 27.04, 0.16 18.04, 0 12.25 M0 36.75 C-0.03 30.79, -0.54 24.97, 0 12.25 M0 12.25 C1.26 5.34, 4.4 -1.81, 12.25 0 M0 12.25 C1.86 5.23, 3.09 0.52, 12.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(463.93497911894474 688.2270792403314) rotate(0 51.2574462890625 7.397678214733503)"><text x="51.2574462890625" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">PodSandboxStats</text></g><g stroke-linecap="round" transform="translate(737.1843705885069 631.1247574550649) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C37.96 -0.25, 58.44 0.26, 123.42 0 M123.42 0 C130.73 1.63, 137.21 3.69, 135.67 12.25 M135.67 12.25 C134.46 18.36, 134.06 28.96, 135.67 36.75 M135.67 36.75 C136.93 45.64, 130.11 49.46, 123.42 49 M123.42 49 C101.97 49.01, 78.66 48.81, 12.25 49 M12.25 49 C2.69 49.52, 1.03 43.94, 0 36.75 M0 36.75 C-1.43 30.68, -0.02 19.27, 0 12.25 M0 12.25 C1.09 3.03, 5.16 1.26, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(753.7610660754665 648.2270792403314) rotate(0 51.2574462890625 7.397678214733503)"><text x="51.2574462890625" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">PodSandboxStats</text></g><g stroke-linecap="round"><g transform="translate(588.1924254080072 612.3090108639277) rotate(0 72.91304347826085 19.572020984666608)"><path d="M0 0 C24.3 6.52, 121.52 32.62, 145.83 39.14 M0 0 C24.3 6.52, 121.52 32.62, 145.83 39.14" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(588.1924254080072 612.3090108639277) rotate(0 72.91304347826085 19.572020984666608)"><path d="M120.92 41.31 C130.67 40.46, 140.41 39.62, 145.83 39.14 M120.92 41.31 C126.71 40.81, 132.49 40.3, 145.83 39.14" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(588.1924254080072 612.3090108639277) rotate(0 72.91304347826085 19.572020984666608)"><path d="M125.35 24.8 C133.36 30.41, 141.37 36.02, 145.83 39.14 M125.35 24.8 C130.11 28.13, 134.86 31.46, 145.83 39.14" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(588.6924254080072 698.0370401330747) rotate(0 72.41304347826087 -17.873370462502862)"><path d="M0 0 C24.14 -5.96, 120.69 -29.79, 144.83 -35.75 M0 0 C24.14 -5.96, 120.69 -29.79, 144.83 -35.75" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(588.6924254080072 698.0370401330747) rotate(0 72.41304347826087 -17.873370462502862)"><path d="M124.07 -21.82 C130.73 -26.29, 137.4 -30.76, 144.83 -35.75 M124.07 -21.82 C131.38 -26.73, 138.7 -31.64, 144.83 -35.75" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(588.6924254080072 698.0370401330747) rotate(0 72.41304347826087 -17.873370462502862)"><path d="M119.97 -38.42 C127.95 -37.56, 135.93 -36.7, 144.83 -35.75 M119.97 -38.42 C128.73 -37.48, 137.49 -36.54, 144.83 -35.75" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(729.416109037202 442.1119697312797) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C52.26 1.83, 92.48 -2.78, 123.42 0 C129.04 0.42, 133.83 5.84, 135.67 12.25 C132.67 20.6, 134.35 27.62, 135.67 36.75 C137.13 47.24, 132.64 47.3, 123.42 49 C93.26 47.31, 71.23 51.51, 12.25 49 C5.81 50.31, -0.79 47.47, 0 36.75 C-2.88 31.99, -0.43 24.72, 0 12.25 C-3.1 6.69, 2.85 -2.69, 12.25 0" stroke="none" stroke-width="0" fill="#ffc9c9"/><path d="M12.25 0 C51.86 2.69, 95.82 2.16, 123.42 0 M123.42 0 C132.3 -1.29, 135.2 2.36, 135.67 12.25 M135.67 12.25 C137.06 20.22, 137.41 30.75, 135.67 36.75 M135.67 36.75 C135.19 45.62, 131.19 50.69, 123.42 49 M123.42 49 C82.02 50.49, 39.83 51.58, 12.25 49 M12.25 49 C5.05 50.62, 0.05 45.54, 0 36.75 M0 36.75 C-0.87 30.51, -0.24 21.91, 0 12.25 M0 12.25 C-0.62 3.97, 3.68 -1.22, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(752.8472433058998 459.21429151654615) rotate(0 44.40300750732422 7.397678214733503)"><text x="44.40300750732422" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ContainerStats</text></g><g stroke-linecap="round" transform="translate(1058.3871235299557 443.909071180555) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C47.76 2.13, 86.54 3.47, 123.42 0 C131.17 -2.58, 138.83 1.41, 135.67 12.25 C137 22.8, 136.6 27, 135.67 36.75 C133.75 42.58, 131.42 46.28, 123.42 49 C96.7 51.76, 65.76 53.01, 12.25 49 C1.88 45.96, -3.34 41.66, 0 36.75 C-3 31.26, -1.14 17.88, 0 12.25 C-2.95 5.25, 6.94 2.67, 12.25 0" stroke="none" stroke-width="0" fill="#a5d8ff"/><path d="M12.25 0 C57.28 -2.1, 100.41 -2.53, 123.42 0 M123.42 0 C132.63 1.29, 136.7 5.32, 135.67 12.25 M135.67 12.25 C135.38 19.5, 135.47 27.03, 135.67 36.75 M135.67 36.75 C137.15 46.45, 133.13 49.2, 123.42 49 M123.42 49 C87.54 51.57, 49.78 50.58, 12.25 49 M12.25 49 C4.62 47.26, 1.3 43.83, 0 36.75 M0 36.75 C-0.56 27.13, -0.35 16.51, 0 12.25 M0 12.25 C0.99 4.55, 3.78 0.65, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(1094.586774339181 461.0113929658214) rotate(0 31.634490966796875 7.397678214733503)"><text x="31.634490966796875" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">SwapUsage</text></g><g stroke-linecap="round" transform="translate(441.564737851662 407.0106688646242) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C39.47 2.25, 65.63 2.19, 123.42 0 M12.25 0 C41.72 -0.34, 71.47 0.15, 123.42 0 M123.42 0 C133.07 1.53, 137.21 4.28, 135.67 12.25 M123.42 0 C132.31 -0.45, 137.65 5.2, 135.67 12.25 M135.67 12.25 C136.21 19.84, 136.98 29.82, 135.67 36.75 M135.67 12.25 C135.3 21.71, 135.41 30.67, 135.67 36.75 M135.67 36.75 C136.66 45.39, 131.28 49.65, 123.42 49 M135.67 36.75 C136.64 46.55, 129.62 47.37, 123.42 49 M123.42 49 C88.54 48.11, 55.89 48.05, 12.25 49 M123.42 49 C86.49 50.03, 49.05 48.91, 12.25 49 M12.25 49 C3.02 47.7, -0.09 43.4, 0 36.75 M12.25 49 C3.07 50.72, 1.66 46.02, 0 36.75 M0 36.75 C-1.17 26.82, -1.8 18.46, 0 12.25 M0 36.75 C-0.81 29.38, -0.29 19.82, 0 12.25 M0 12.25 C-1.64 4.73, 5.67 1.48, 12.25 0 M0 12.25 C-0.84 3.96, 5.85 -0.74, 12.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(453.6053310436997 424.11299064989066) rotate(0 55.793548583984375 7.39767821473356)"><text x="55.793548583984375" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ListContainerStats</text></g><g stroke-linecap="round" transform="translate(444.0526464631515 481.59178722197635) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C52.12 0.9, 94.56 -1.16, 123.42 0 M12.25 0 C50.23 -0.17, 87.51 -0.84, 123.42 0 M123.42 0 C132.21 -0.39, 137.39 5.05, 135.67 12.25 M123.42 0 C133.44 0.06, 136.39 4.7, 135.67 12.25 M135.67 12.25 C137.59 17.02, 136.99 22.08, 135.67 36.75 M135.67 12.25 C136.11 19.7, 135.47 27, 135.67 36.75 M135.67 36.75 C136.52 46.33, 129.87 47.58, 123.42 49 M135.67 36.75 C135.94 43.74, 132.71 48.73, 123.42 49 M123.42 49 C98.74 47.61, 73.32 48.98, 12.25 49 M123.42 49 C82.35 47.83, 42.3 47.72, 12.25 49 M12.25 49 C3.2 50.5, 1.45 45.87, 0 36.75 M12.25 49 C4.92 48.5, 1.63 43.51, 0 36.75 M0 36.75 C0.83 31.02, 0.04 24.47, 0 12.25 M0 36.75 C-0.73 27.95, 0.88 19.25, 0 12.25 M0 12.25 C-0.73 3.98, 5.62 -0.65, 12.25 0 M0 12.25 C0 5.49, 3.49 0.09, 12.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(467.4837807318494 498.6941090072428) rotate(0 44.40300750732422 7.397678214733503)"><text x="44.40300750732422" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ContainerStats</text></g><g stroke-linecap="round"><g transform="translate(576.9767391334976 432.5751501807533) rotate(0 74.34782608695647 16.484967377403393)"><path d="M0 0 C24.78 5.49, 123.91 27.47, 148.7 32.97 M0 0 C24.78 5.49, 123.91 27.47, 148.7 32.97" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(576.9767391334976 432.5751501807533) rotate(0 74.34782608695647 16.484967377403393)"><path d="M123.91 36.23 C129.98 35.43, 136.05 34.63, 148.7 32.97 M123.91 36.23 C132.94 35.04, 141.98 33.85, 148.7 32.97" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(576.9767391334976 432.5751501807533) rotate(0 74.34782608695647 16.484967377403393)"><path d="M127.61 19.54 C132.78 22.83, 137.94 26.12, 148.7 32.97 M127.61 19.54 C135.3 24.43, 142.98 29.33, 148.7 32.97" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(580.7209300151958 505.1786965359955) rotate(0 72.5708108941958 -16.0807266192665)"><path d="M0 0 C24.19 -5.36, 120.95 -26.8, 145.14 -32.16 M0 0 C24.19 -5.36, 120.95 -26.8, 145.14 -32.16" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(580.7209300151958 505.1786965359955) rotate(0 72.5708108941958 -16.0807266192665)"><path d="M124.06 -18.73 C132.23 -23.94, 140.41 -29.15, 145.14 -32.16 M124.06 -18.73 C130.69 -22.96, 137.32 -27.18, 145.14 -32.16" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(580.7209300151958 505.1786965359955) rotate(0 72.5708108941958 -16.0807266192665)"><path d="M120.36 -35.43 C129.97 -34.16, 139.58 -32.89, 145.14 -32.16 M120.36 -35.43 C128.15 -34.4, 135.95 -33.37, 145.14 -32.16" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(866.6723913074106 465.7425365028266) rotate(0 94.48550724637687 1.2469152729007646)"><path d="M0 0 C31.5 0.42, 157.48 2.08, 188.97 2.49 M0 0 C31.5 0.42, 157.48 2.08, 188.97 2.49" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(866.6723913074106 465.7425365028266) rotate(0 94.48550724637687 1.2469152729007646)"><path d="M165.37 10.73 C174.72 7.47, 184.08 4.2, 188.97 2.49 M165.37 10.73 C172.32 8.31, 179.28 5.88, 188.97 2.49" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(866.6723913074106 465.7425365028266) rotate(0 94.48550724637687 1.2469152729007646)"><path d="M165.59 -6.37 C174.86 -2.85, 184.13 0.66, 188.97 2.49 M165.59 -6.37 C172.48 -3.76, 179.37 -1.14, 188.97 2.49" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(964.0837738280634 583.0659339256533) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C48.8 -2.21, 87.24 -0.06, 123.42 0 M123.42 0 C132.53 -1.15, 134.69 2.8, 135.67 12.25 M135.67 12.25 C134.89 16.84, 135.91 23.74, 135.67 36.75 M135.67 36.75 C136.13 46.83, 132.71 49.42, 123.42 49 M123.42 49 C93.76 48.35, 64.94 50.91, 12.25 49 M12.25 49 C3.4 48.95, -1.46 45.53, 0 36.75 M0 36.75 C-0.96 28.74, 2.04 19.08, 0 12.25 M0 12.25 C-1.77 5.87, 5.27 1.44, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(969.5892794346519 592.7705774961863) rotate(0 62.32863616943358 14.795356429467063)"><text x="62.328636169433594" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">LinuxPodSandboxStat</text><text x="62.328636169433594" y="14.795356429467045" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">s</text></g><g stroke-linecap="round"><g transform="translate(962.0215132445969 597.845547329838) rotate(0 -73.06174139787645 -51.90632488420255)"><path d="M0 0 C-24.35 -17.3, -121.77 -86.51, -146.12 -103.81 M0 0 C-24.35 -17.3, -121.77 -86.51, -146.12 -103.81" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(962.0215132445969 597.845547329838) rotate(0 -73.06174139787645 -51.90632488420255)"><path d="M-122.02 -97.18 C-128.57 -98.98, -135.11 -100.78, -146.12 -103.81 M-122.02 -97.18 C-129.08 -99.12, -136.13 -101.06, -146.12 -103.81" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(962.0215132445969 597.845547329838) rotate(0 -73.06174139787645 -51.90632488420255)"><path d="M-131.92 -83.24 C-135.78 -88.83, -139.64 -94.41, -146.12 -103.81 M-131.92 -83.24 C-136.08 -89.26, -140.24 -95.29, -146.12 -103.81" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(875.3309150101162 655.6653273259251) rotate(0 43.085057460981474 -18.431487882991235)"><path d="M0 0 C14.36 -6.14, 71.81 -30.72, 86.17 -36.86 M0 0 C14.36 -6.14, 71.81 -30.72, 86.17 -36.86" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(875.3309150101162 655.6653273259251) rotate(0 43.085057460981474 -18.431487882991235)"><path d="M67.93 -19.76 C71.73 -23.32, 75.52 -26.88, 86.17 -36.86 M67.93 -19.76 C71.95 -23.53, 75.97 -27.3, 86.17 -36.86" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(875.3309150101162 655.6653273259251) rotate(0 43.085057460981474 -18.431487882991235)"><path d="M61.21 -35.48 C66.4 -35.77, 71.59 -36.06, 86.17 -36.86 M61.21 -35.48 C66.71 -35.79, 72.21 -36.09, 86.17 -36.86" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(440.28283606165286 110.97769863153599) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C37.48 0.59, 59.06 -1, 123.42 0 M12.25 0 C46.22 0.32, 80.52 0.37, 123.42 0 M123.42 0 C132.79 1.69, 137.64 2.99, 135.67 12.25 M123.42 0 C133.57 2.17, 137.15 5.37, 135.67 12.25 M135.67 12.25 C136.28 20.69, 135.68 26.29, 135.67 36.75 M135.67 12.25 C135.93 17, 135.67 23.28, 135.67 36.75 M135.67 36.75 C135.54 46.09, 132.02 50.95, 123.42 49 M135.67 36.75 C135.97 43.99, 130.58 47.75, 123.42 49 M123.42 49 C80.9 50.2, 42.23 51.63, 12.25 49 M123.42 49 C95 48.82, 65.85 49.56, 12.25 49 M12.25 49 C3.95 50.76, -1.13 45.98, 0 36.75 M12.25 49 C4.66 47.42, 1.86 45.85, 0 36.75 M0 36.75 C1.57 28.95, 1.32 23.15, 0 12.25 M0 36.75 C-0.86 28.79, 0.15 19.26, 0 12.25 M0 12.25 C-0.53 4.66, 3.6 -1.84, 12.25 0 M0 12.25 C-1.54 6.17, 3 1.27, 12.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(450.2771569758586 128.08002041680243) rotate(0 57.839820861816406 7.39767821473356)"><text x="57.839820861816406" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">GetContainerEvents</text></g><g stroke-linecap="round" transform="translate(728.152401279044 113.97769863153599) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C45.72 0.76, 82.23 -2.19, 123.42 0 C131.19 -3.52, 137.93 3.84, 135.67 12.25 C136.32 19.27, 138.34 32.72, 135.67 36.75 C136.11 41.88, 130.16 51.45, 123.42 49 C92.54 51.07, 61.34 47.71, 12.25 49 C5.92 49.64, -2 43.97, 0 36.75 C-2.31 31.96, -1.59 22.63, 0 12.25 C-2.54 4.65, 4.97 -0.48, 12.25 0" stroke="none" stroke-width="0" fill="#ffc9c9"/><path d="M12.25 0 C47.53 1.66, 81.62 -0.19, 123.42 0 M123.42 0 C131.32 -1.13, 135.51 5.28, 135.67 12.25 M135.67 12.25 C137.63 23.57, 137.2 32.22, 135.67 36.75 M135.67 36.75 C135.2 45.39, 132.59 49.55, 123.42 49 M123.42 49 C79.5 50.53, 37.49 49.48, 12.25 49 M12.25 49 C2.22 48.69, -0.67 44.79, 0 36.75 M0 36.75 C0.21 27.15, -0.93 18.09, 0 12.25 M0 12.25 C1.37 3.08, 5.67 -1.56, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(733.1907132821169 123.68234220206887) rotate(0 62.79582977294922 14.795356429467006)"><text x="62.79582977294922" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ContainerEventRespon</text><text x="62.79582977294922" y="14.795356429467045" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">se</text></g><g stroke-linecap="round"><g transform="translate(576.9511196136971 133.05499219504964) rotate(0 73.77981685505767 0.6953772588850597)"><path d="M0 0 C24.59 0.23, 122.97 1.16, 147.56 1.39 M0 0 C24.59 0.23, 122.97 1.16, 147.56 1.39" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(576.9511196136971 133.05499219504964) rotate(0 73.77981685505767 0.6953772588850597)"><path d="M123.99 9.72 C133.3 6.43, 142.62 3.14, 147.56 1.39 M123.99 9.72 C128.86 8, 133.73 6.28, 147.56 1.39" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(576.9511196136971 133.05499219504964) rotate(0 73.77981685505767 0.6953772588850597)"><path d="M124.15 -7.38 C133.4 -3.91, 142.65 -0.45, 147.56 1.39 M124.15 -7.38 C128.98 -5.57, 133.82 -3.76, 147.56 1.39" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(190.6875 271.9700851482246) rotate(0 28.459141776022108 14.5)"><path d="M7.25 0 C22.47 0.96, 35.44 1.05, 49.67 0 M7.25 0 C19.57 -0.58, 29.82 -0.66, 49.67 0 M49.67 0 C56.11 1.71, 56.24 2.04, 56.92 7.25 M49.67 0 C56.6 -0.4, 58.55 3.5, 56.92 7.25 M56.92 7.25 C57.84 11.89, 56.37 17.08, 56.92 21.75 M56.92 7.25 C57.19 12.2, 56.26 18.78, 56.92 21.75 M56.92 21.75 C55.53 25.19, 56.5 30.77, 49.67 29 M56.92 21.75 C58.21 28.15, 53.48 27.59, 49.67 29 M49.67 29 C34.46 28.12, 21.15 27.91, 7.25 29 M49.67 29 C38.26 29.2, 25.66 29.51, 7.25 29 M7.25 29 C1.54 27.68, 0.89 26.76, 0 21.75 M7.25 29 C2.68 31.2, -0.19 26.24, 0 21.75 M0 21.75 C-0.23 16.83, 1.21 13.58, 0 7.25 M0 21.75 C-0.23 17.25, 0.04 12.34, 0 7.25 M0 7.25 C1.85 1.35, 3.18 0.6, 7.25 0 M0 7.25 C0.96 2.27, 1.5 1.34, 7.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g stroke-linecap="round" transform="translate(11.5625 271.9700851482246) rotate(0 28.45914177602208 14.5)"><path d="M7.25 0 C16.26 -1.51, 26.66 3.03, 49.67 0 C54.02 0.86, 57.57 -1.18, 56.92 7.25 C56.58 13.55, 56.08 20.6, 56.92 21.75 C58.6 25.45, 54.66 30.06, 49.67 29 C36.06 31.69, 13.94 26.06, 7.25 29 C-0.53 28.97, -1.13 24.58, 0 21.75 C1.95 17.99, 1 13.32, 0 7.25 C1.74 3.98, 2.82 -0.37, 7.25 0" stroke="none" stroke-width="0" fill="#a5d8ff"/><path d="M7.25 0 C17.98 -1.84, 29.71 0.81, 49.67 0 M7.25 0 C18.89 0.19, 30.25 -0.86, 49.67 0 M49.67 0 C53.49 0.99, 57.95 3.46, 56.92 7.25 M49.67 0 C52.83 -2.26, 54.84 0.38, 56.92 7.25 M56.92 7.25 C56.18 9.02, 56 12.67, 56.92 21.75 M56.92 7.25 C56.36 12.38, 57.11 17.53, 56.92 21.75 M56.92 21.75 C54.94 25.29, 56.27 27.43, 49.67 29 M56.92 21.75 C55.57 28.54, 53.16 28.8, 49.67 29 M49.67 29 C36.16 29.79, 26.82 26.72, 7.25 29 M49.67 29 C37.75 27.83, 25.56 29.45, 7.25 29 M7.25 29 C4.15 27.12, -0.22 25.03, 0 21.75 M7.25 29 C2.28 28.26, 1.7 25.88, 0 21.75 M0 21.75 C0.16 17.22, 0.34 14.92, 0 7.25 M0 21.75 C-0.47 16.95, 0.62 12.83, 0 7.25 M0 7.25 C0.87 2.52, 0.94 -0.97, 7.25 0 M0 7.25 C-1.34 2.49, 2.24 1.35, 7.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(99.0216417760222 221.4700851482246) rotate(0 24.000045776367188 10)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="16px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">无变更</text></g><g transform="translate(100.02159599965489 277.7200851482246) rotate(0 16.000030517578125 10)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="16px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">新增</text></g><g stroke-linecap="round" transform="translate(191.625 354.4700851482246) rotate(0 28.459141776022108 14.5)"><path d="M7.25 0 C20.73 0.66, 35.61 1.39, 49.67 0 M49.67 0 C55.84 0.42, 58.86 3.94, 56.92 7.25 M56.92 7.25 C56.67 12.23, 56.52 18.54, 56.92 21.75 M56.92 21.75 C57.51 25.79, 56.33 28.62, 49.67 29 M49.67 29 C31.86 30.65, 17.6 30.27, 7.25 29 M7.25 29 C1.29 28.27, 0.65 28.07, 0 21.75 M0 21.75 C1.32 18.48, -0.62 11.43, 0 7.25 M0 7.25 C0.02 0.68, 0.81 -0.83, 7.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g stroke-linecap="round" transform="translate(10 351.9700851482246) rotate(0 28.45914177602208 14.5)"><path d="M7.25 0 C22.43 0.79, 34.43 1.63, 49.67 0 C53.54 -1.26, 59.87 4.57, 56.92 7.25 C59.25 8.58, 54.91 14.64, 56.92 21.75 C57.53 27.64, 53.58 27.03, 49.67 29 C35.65 28.18, 20.44 31.03, 7.25 29 C-0.55 26.97, 2.02 28.52, 0 21.75 C-1.45 16.59, 2.11 9.43, 0 7.25 C0.36 3.17, 4.4 -0.65, 7.25 0" stroke="none" stroke-width="0" fill="#ffc9c9"/><path d="M7.25 0 C22.57 1.51, 40.7 0.21, 49.67 0 M7.25 0 C24.54 -0.61, 40.7 -0.64, 49.67 0 M49.67 0 C53.86 0.58, 57.51 1.63, 56.92 7.25 M49.67 0 C56.61 -0.44, 58.63 2.04, 56.92 7.25 M56.92 7.25 C56.57 13.42, 55.96 17.7, 56.92 21.75 M56.92 7.25 C57.29 12.13, 57.71 17.52, 56.92 21.75 M56.92 21.75 C56.13 24.79, 54.52 27.26, 49.67 29 M56.92 21.75 C55.07 25.62, 53.19 26.87, 49.67 29 M49.67 29 C41.95 29.45, 33.19 29.64, 7.25 29 M49.67 29 C38.13 27.91, 25.01 28.1, 7.25 29 M7.25 29 C0.44 28.4, 0.39 26.31, 0 21.75 M7.25 29 C0.13 27.85, 1.98 27.31, 0 21.75 M0 21.75 C0.38 15.34, -0.46 12.82, 0 7.25 M0 21.75 C-0.68 18.48, -0.45 14.78, 0 7.25 M0 7.25 C-0.73 0.91, 1.24 0.02, 7.25 0 M0 7.25 C-0.56 0.78, 3.14 0.64, 7.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(100.95909599965489 357.7200851482248) rotate(0 16.000030517578125 10)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="16px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">修改</text></g><g transform="translate(279.55286125844384 278.4700851482246) rotate(0 16.000030517578125 10)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="16px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">接口</text></g><g transform="translate(280.49036125844384 358.4700851482248) rotate(0 32.00006103515625 10)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="16px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">内部数据</text></g><g stroke-linecap="round" transform="translate(441.51073438300847 10.751633986928255) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C41.29 -2.53, 75.89 1.52, 123.42 0 C132.17 -0.98, 135.09 3.89, 135.67 12.25 C133.36 24.32, 137.05 29.24, 135.67 36.75 C137.92 46.6, 130.07 51.58, 123.42 49 C83.19 49.13, 38.97 49.55, 12.25 49 C4.7 45.52, -2.43 43.88, 0 36.75 C-0.11 28.49, 0.37 26.29, 0 12.25 C-1.69 4.38, 3.87 -1.66, 12.25 0" stroke="none" stroke-width="0" fill="#a5d8ff"/><path d="M12.25 0 C40.5 0.38, 68.72 0.57, 123.42 0 M12.25 0 C55.6 0.03, 100.91 -0.01, 123.42 0 M123.42 0 C130.91 1.57, 136.88 5.9, 135.67 12.25 M123.42 0 C132.06 -1.24, 137.56 3.82, 135.67 12.25 M135.67 12.25 C135.51 17.2, 134.08 24.91, 135.67 36.75 M135.67 12.25 C134.61 19.88, 134.58 27.19, 135.67 36.75 M135.67 36.75 C137.48 46.76, 131.69 48.23, 123.42 49 M135.67 36.75 C137.06 43.82, 129.39 48.33, 123.42 49 M123.42 49 C98.21 47.23, 77.19 50.26, 12.25 49 M123.42 49 C91.01 48.47, 56.89 48.69, 12.25 49 M12.25 49 C3.53 48.6, 0.28 46.55, 0 36.75 M12.25 49 C5.16 50.53, -1.41 43.61, 0 36.75 M0 36.75 C-2.07 30.49, -0.06 21.95, 0 12.25 M0 36.75 C-0.15 28.95, -0.46 18.52, 0 12.25 M0 12.25 C-1.08 4.97, 3.28 -1.96, 12.25 0 M0 12.25 C-0.18 2.58, 5.05 2.17, 12.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(470.6193894036595 27.853955772194695) rotate(0 38.725486755371094 7.39767821473356)"><text x="38.725486755371094" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">RuntimeConfig</text></g><g stroke-linecap="round" transform="translate(729.5760938601327 10) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C38.04 -4.08, 64.6 -0.03, 123.42 0 C129.17 -2.84, 134.04 3.08, 135.67 12.25 C137.41 21.5, 133.52 23.94, 135.67 36.75 C136.11 48.08, 129.01 45.67, 123.42 49 C84.43 48.66, 39.71 48.74, 12.25 49 C7.6 47.15, -2.39 42.97, 0 36.75 C-0.1 25.96, 1.69 23.27, 0 12.25 C-0.35 1.25, 3.51 -3.23, 12.25 0" stroke="none" stroke-width="0" fill="#a5d8ff"/><path d="M12.25 0 C46.61 0.02, 77.46 -1.82, 123.42 0 M123.42 0 C132.1 0.28, 136.1 3.41, 135.67 12.25 M135.67 12.25 C136.23 20.44, 137.46 30.56, 135.67 36.75 M135.67 36.75 C135.25 44.63, 132.11 48.87, 123.42 49 M123.42 49 C100.57 46.89, 76.6 46.7, 12.25 49 M12.25 49 C4.57 47.2, 0.35 46.73, 0 36.75 M0 36.75 C1.2 26.19, -1.93 16.95, 0 12.25 M0 12.25 C-1.8 3.46, 3.29 -1.87, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(738.4999420570532 19.70464357053288) rotate(0 58.91029357910156 14.795356429467006)"><text x="58.91029357910156" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">LinuxRuntimeConfigur</text><text x="58.91029357910156" y="14.795356429467045" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ation</text></g><g stroke-linecap="round" transform="translate(1058.0597539908517 11.307189542483684) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C35.82 -3.77, 62.58 -1.93, 123.42 0 C130.87 0.51, 138.6 5.76, 135.67 12.25 C135.84 24.39, 132.82 26.87, 135.67 36.75 C138.17 45.2, 132.5 51.1, 123.42 49 C96.22 46.72, 59.59 46.63, 12.25 49 C5.69 47.56, -3.53 44.64, 0 36.75 C-0.54 28.17, -0.76 22.02, 0 12.25 C-2.85 3.19, 5.28 0.58, 12.25 0" stroke="none" stroke-width="0" fill="#a5d8ff"/><path d="M12.25 0 C46.29 1.29, 79.73 0.34, 123.42 0 M123.42 0 C133.15 1.22, 137.48 4.5, 135.67 12.25 M135.67 12.25 C135.45 17.99, 136.4 24.18, 135.67 36.75 M135.67 36.75 C133.78 43.36, 130.56 49.03, 123.42 49 M123.42 49 C99.14 46.24, 74.25 49.85, 12.25 49 M12.25 49 C5.93 49.11, -0.77 46.13, 0 36.75 M0 36.75 C-1.57 29.94, -0.56 22.52, 0 12.25 M0 12.25 C-1.33 4.53, 5.78 -0.09, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(1089.7055565334754 28.409511327750124) rotate(0 36.18833923339844 7.39767821473356)"><text x="36.18833923339844" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">CgroupDriver</text></g><g stroke-linecap="round"><g transform="translate(581.9852550437765 33.93286180168093) rotate(0 71.7256341871495 1.8430938087597042)"><path d="M0 0 C23.91 0.61, 119.54 3.07, 143.45 3.69 M0 0 C23.91 0.61, 119.54 3.07, 143.45 3.69" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(581.9852550437765 33.93286180168093) rotate(0 71.7256341871495 1.8430938087597042)"><path d="M119.75 11.63 C127.19 9.14, 134.63 6.64, 143.45 3.69 M119.75 11.63 C125.58 9.68, 131.41 7.72, 143.45 3.69" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(581.9852550437765 33.93286180168093) rotate(0 71.7256341871495 1.8430938087597042)"><path d="M120.19 -5.46 C127.49 -2.59, 134.79 0.28, 143.45 3.69 M120.19 -5.46 C125.91 -3.21, 131.63 -0.96, 143.45 3.69" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(865.0479184137803 32.85299659664042) rotate(0 96.0059177885357 1.8349593476192467)"><path d="M0 0 C32 0.61, 160.01 3.06, 192.01 3.67 M0 0 C32 0.61, 160.01 3.06, 192.01 3.67" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(865.0479184137803 32.85299659664042) rotate(0 96.0059177885357 1.8349593476192467)"><path d="M168.36 11.77 C174.66 9.61, 180.96 7.46, 192.01 3.67 M168.36 11.77 C175.19 9.43, 182.01 7.09, 192.01 3.67" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(865.0479184137803 32.85299659664042) rotate(0 96.0059177885357 1.8349593476192467)"><path d="M168.69 -5.33 C174.9 -2.93, 181.11 -0.54, 192.01 3.67 M168.69 -5.33 C175.42 -2.73, 182.15 -0.13, 192.01 3.67" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(438.04995006928294 211.70261437908425) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C43.9 -1.21, 78.24 0.72, 123.42 0 M12.25 0 C46.77 0.14, 79.54 1.86, 123.42 0 M123.42 0 C133.51 -0.74, 135.06 3.55, 135.67 12.25 M123.42 0 C131.49 1.42, 134.35 2.09, 135.67 12.25 M135.67 12.25 C137.42 20.35, 134.37 30.45, 135.67 36.75 M135.67 12.25 C135.51 20, 135.59 28.32, 135.67 36.75 M135.67 36.75 C137.61 46.04, 131.47 47.46, 123.42 49 M135.67 36.75 C133.49 46.11, 133.38 47.18, 123.42 49 M123.42 49 C80.97 49.88, 34.97 47.77, 12.25 49 M123.42 49 C91.85 49.86, 57.62 49.38, 12.25 49 M12.25 49 C4.82 48.28, 1.71 43.69, 0 36.75 M12.25 49 C2.69 48.7, -1.47 43.07, 0 36.75 M0 36.75 C1.08 28.82, -1.09 19.76, 0 12.25 M0 36.75 C-0.17 31.6, 0.84 26, 0 12.25 M0 12.25 C1.32 2.85, 5.07 0.08, 12.25 0 M0 12.25 C-0.83 4.7, 2.84 -0.16, 12.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(451.26743840292227 228.8049361643507) rotate(0 54.61665344238281 7.39767821473356)"><text x="54.61665344238281" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">PodSandboxStatus</text></g><g stroke-linecap="round" transform="translate(725.0499500692829 214.70261437908425) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C47.77 4.34, 86.83 5.22, 123.42 0 C131.32 -2.47, 133.67 5.41, 135.67 12.25 C133.87 18.24, 133.75 22.28, 135.67 36.75 C138.29 45.26, 134.85 50.99, 123.42 49 C80.96 49.64, 42.72 49.5, 12.25 49 C5.98 51.88, 2.2 47.28, 0 36.75 C-1.51 31.86, -2.16 24.81, 0 12.25 C0.71 3.07, 7.04 2.86, 12.25 0" stroke="none" stroke-width="0" fill="#ffc9c9"/><path d="M12.25 0 C44.99 -2.91, 75.89 0.03, 123.42 0 M123.42 0 C130.08 -1.44, 137.6 6, 135.67 12.25 M135.67 12.25 C135.51 19.93, 134.45 23.4, 135.67 36.75 M135.67 36.75 C137.28 43.52, 132.96 50.92, 123.42 49 M123.42 49 C97.27 47.05, 71.3 48.85, 12.25 49 M12.25 49 C3.98 49.26, 0.92 46.86, 0 36.75 M0 36.75 C-1.88 29.05, 1.57 17.7, 0 12.25 M0 12.25 C1.82 3.02, 4.89 1.02, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(731.0226721675707 224.40725794961713) rotate(0 61.861419677734375 14.795356429467006)"><text x="61.861419677734375" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">PodSandboxStatusRe</text><text x="61.861419677734375" y="14.795356429467045" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">sponse</text></g><g stroke-linecap="round"><g transform="translate(574.7182336213272 233.78163227706204) rotate(0 73.34503424636208 0.6933682538535777)"><path d="M0 0 C24.45 0.23, 122.24 1.16, 146.69 1.39 M0 0 C24.45 0.23, 122.24 1.16, 146.69 1.39" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(574.7182336213272 233.78163227706204) rotate(0 73.34503424636208 0.6933682538535777)"><path d="M123.12 9.71 C127.94 8.01, 132.76 6.31, 146.69 1.39 M123.12 9.71 C129.32 7.52, 135.52 5.33, 146.69 1.39" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(574.7182336213272 233.78163227706204) rotate(0 73.34503424636208 0.6933682538535777)"><path d="M123.28 -7.39 C128.07 -5.59, 132.85 -3.8, 146.69 1.39 M123.28 -7.39 C129.44 -5.08, 135.59 -2.77, 146.69 1.39" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(440.4979095910648 299.03384611883644) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C50.17 -0.59, 88.6 0.89, 123.42 0 M12.25 0 C46.77 0.83, 82.81 -0.36, 123.42 0 M123.42 0 C131.94 0.68, 136.22 2.42, 135.67 12.25 M123.42 0 C130.55 -0.75, 135.13 4.41, 135.67 12.25 M135.67 12.25 C137.68 20.92, 137.27 31.19, 135.67 36.75 M135.67 12.25 C135.29 19.8, 135.07 24.76, 135.67 36.75 M135.67 36.75 C136.35 43.35, 132.94 47.95, 123.42 49 M135.67 36.75 C135.17 43.63, 133.04 47.65, 123.42 49 M123.42 49 C94.28 49.81, 61.5 47.59, 12.25 49 M123.42 49 C94.72 47.73, 65.1 49.03, 12.25 49 M12.25 49 C2.49 47.86, -1.49 44.59, 0 36.75 M12.25 49 C5.13 49.61, 0.15 43.68, 0 36.75 M0 36.75 C-1.43 30.31, -1.35 24.99, 0 12.25 M0 36.75 C-0.76 31.1, 0.39 25.52, 0 12.25 M0 12.25 C0.61 4.67, 2.62 0.38, 12.25 0 M0 12.25 C-1.02 4.55, 6.02 2.1, 12.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(460.5698367064424 316.1361679041029) rotate(0 47.76221466064453 7.39767821473356)"><text x="47.76221466064453" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ContainerStatus</text></g><g stroke-linecap="round" transform="translate(1063.97617046063 294.6860200318804) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C38.01 -3.37, 65.19 -1.9, 123.42 0 C130 1.95, 134.84 2.34, 135.67 12.25 C134.54 21.74, 137.66 26.41, 135.67 36.75 C132.31 45.9, 134.07 47.89, 123.42 49 C93.12 45.92, 65.34 49.01, 12.25 49 C4.74 47.11, -1.45 47.57, 0 36.75 C-3.76 26.75, -2.28 23.95, 0 12.25 C2.37 2.99, 3.64 0.89, 12.25 0" stroke="none" stroke-width="0" fill="#ffc9c9"/><path d="M12.25 0 C35.15 1.26, 59.02 -1.03, 123.42 0 M123.42 0 C130.63 0.72, 135.03 5.82, 135.67 12.25 M135.67 12.25 C135.27 18.6, 135.64 27.45, 135.67 36.75 M135.67 36.75 C134.55 46.3, 132.05 48.1, 123.42 49 M123.42 49 C93.36 48.93, 66.43 46.47, 12.25 49 M12.25 49 C2.87 48.24, -0.73 46.28, 0 36.75 M0 36.75 C-1.05 30.11, -1.84 21.45, 0 12.25 M0 12.25 C-1.26 3.76, 4.69 1.73, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(1084.0480975760076 311.78834181714683) rotate(0 47.76221466064453 7.39767821473356)"><text x="47.76221466064453" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ContainerStatus</text></g><g stroke-linecap="round"><g transform="translate(578.46527799029 323.3089478003749) rotate(0 72.04068642027516 0.656871557697059)"><path d="M0 0 C24.01 0.22, 120.07 1.09, 144.08 1.31 M0 0 C24.01 0.22, 120.07 1.09, 144.08 1.31" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(578.46527799029 323.3089478003749) rotate(0 72.04068642027516 0.656871557697059)"><path d="M120.51 9.65 C127.06 7.33, 133.61 5.02, 144.08 1.31 M120.51 9.65 C129.45 6.49, 138.39 3.33, 144.08 1.31" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(578.46527799029 323.3089478003749) rotate(0 72.04068642027516 0.656871557697059)"><path d="M120.67 -7.45 C127.18 -5.01, 133.68 -2.58, 144.08 1.31 M120.67 -7.45 C129.55 -4.13, 138.43 -0.8, 144.08 1.31" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(862.5877988387184 241.0759395996838) rotate(0 98.5082612422305 35.21081741989781)"><path d="M0 0 C32.84 11.74, 164.18 58.68, 197.02 70.42 M0 0 C32.84 11.74, 164.18 58.68, 197.02 70.42" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(862.5877988387184 241.0759395996838) rotate(0 98.5082612422305 35.21081741989781)"><path d="M172.02 70.57 C178.03 70.53, 184.04 70.5, 197.02 70.42 M172.02 70.57 C178.58 70.53, 185.15 70.49, 197.02 70.42" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(862.5877988387184 241.0759395996838) rotate(0 98.5082612422305 35.21081741989781)"><path d="M177.77 54.46 C182.4 58.3, 187.03 62.14, 197.02 70.42 M177.77 54.46 C182.83 58.66, 187.88 62.85, 197.02 70.42" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(727.4544313301952 300.7729765536194) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C46.9 -1.24, 85.99 1.29, 123.42 0 C134.41 1.01, 139.25 3.2, 135.67 12.25 C134.88 21, 135.89 27.33, 135.67 36.75 C136.77 44.49, 130.7 47.51, 123.42 49 C84.18 44.76, 48.31 47.88, 12.25 49 C3.12 50.96, -2.99 43.09, 0 36.75 C-3.67 31.78, -3.89 24.69, 0 12.25 C2.47 6.06, 6.49 -1.51, 12.25 0" stroke="none" stroke-width="0" fill="#ffc9c9"/><path d="M12.25 0 C44.14 0.12, 79.36 1.51, 123.42 0 M123.42 0 C130.87 1.61, 136.83 4.15, 135.67 12.25 M135.67 12.25 C134.3 17.07, 135.03 22.89, 135.67 36.75 M135.67 36.75 C135.86 43.46, 131.72 47.36, 123.42 49 M123.42 49 C86.43 47.79, 44.01 50.5, 12.25 49 M12.25 49 C4.16 50.52, -0.37 46.36, 0 36.75 M0 36.75 C-0.57 27.56, 0.46 21.49, 0 12.25 M0 12.25 C1.17 2.39, 5.03 0.32, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(734.154593308854 310.4776201241523) rotate(0 61.13397979736328 14.795356429467006)"><text x="61.13397979736328" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ContainerStatusResp</text><text x="61.13397979736328" y="14.795356429467045" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">onse</text></g><g stroke-linecap="round"><g transform="translate(867.6498770813782 324.6731306541085) rotate(0 95.03000037266514 1.0137009031917614)"><path d="M0 0 C31.68 0.34, 158.38 1.69, 190.06 2.03 M0 0 C31.68 0.34, 158.38 1.69, 190.06 2.03" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(867.6498770813782 324.6731306541085) rotate(0 95.03000037266514 1.0137009031917614)"><path d="M166.48 10.33 C174.33 7.56, 182.18 4.8, 190.06 2.03 M166.48 10.33 C171.44 8.58, 176.4 6.84, 190.06 2.03" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(867.6498770813782 324.6731306541085) rotate(0 95.03000037266514 1.0137009031917614)"><path d="M166.66 -6.77 C174.45 -3.84, 182.24 -0.92, 190.06 2.03 M166.66 -6.77 C171.58 -4.92, 176.5 -3.07, 190.06 2.03" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(864.9870171207247 140.22977320777773) rotate(0 98.12764294201418 79.82581204149369)"><path d="M0 0 C32.71 26.61, 163.55 133.04, 196.26 159.65 M0 0 C32.71 26.61, 163.55 133.04, 196.26 159.65" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(864.9870171207247 140.22977320777773) rotate(0 98.12764294201418 79.82581204149369)"><path d="M172.64 151.46 C178.74 153.58, 184.84 155.69, 196.26 159.65 M172.64 151.46 C180.78 154.29, 188.93 157.11, 196.26 159.65" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(864.9870171207247 140.22977320777773) rotate(0 98.12764294201418 79.82581204149369)"><path d="M183.43 138.19 C186.74 143.74, 190.05 149.28, 196.26 159.65 M183.43 138.19 C187.85 145.6, 192.28 153, 196.26 159.65" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/></svg> +\ No newline at end of file +-- +2.34.1 + diff --git a/0046-oom-monitor-in-manual-cases.patch b/0046-oom-monitor-in-manual-cases.patch new file mode 100644 index 0000000..37d7d40 --- /dev/null +++ b/0046-oom-monitor-in-manual-cases.patch @@ -0,0 +1,120 @@ +From aa77c85ea6879698663d4ef9e01bb63a0db1e57d Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Wed, 3 Apr 2024 09:34:39 +0000 +Subject: [PATCH 46/69] oom monitor in manual cases + +Signed-off-by: jikai <jikai11@huawei.com> +--- + CI/test_cases/container_cases/inspect.sh | 15 ------ + CI/test_cases/manual_cases/oom_monitor.sh | 59 +++++++++++++++++++++++ + 2 files changed, 59 insertions(+), 15 deletions(-) + create mode 100755 CI/test_cases/manual_cases/oom_monitor.sh + +diff --git a/CI/test_cases/container_cases/inspect.sh b/CI/test_cases/container_cases/inspect.sh +index 5d976281..cde9ea1f 100755 +--- a/CI/test_cases/container_cases/inspect.sh ++++ b/CI/test_cases/container_cases/inspect.sh +@@ -27,7 +27,6 @@ function test_inspect_spec() + { + local ret=0 + local image="busybox" +- local ubuntu_image="ubuntu" + local test="container inspect test => (${FUNCNAME[@]})" + + msg_info "${test} starting..." +@@ -38,12 +37,6 @@ function test_inspect_spec() + isula images | grep busybox + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++)) + +- isula pull ${ubuntu_image} +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${ubuntu_image}" && return ${FAILURE} +- +- isula images | grep ubuntu +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${ubuntu_image}" && ((ret++)) +- + containername=test_inspect + + isula create --name $containername --ipc host --pid host --uts host --restart=on-failure:10 --hook-spec ${test_data_path}/test-hookspec.json --cpu-shares 100 --memory 5MB --memory-reservation 4MB --cpu-period 1000000 --cpu-quota 200000 --cpuset-cpus 1 --cpuset-mems 0 --kernel-memory 50M --pids-limit=10000 --volume /home:/root --env a=1 $image /bin/sh ls +@@ -146,14 +139,6 @@ function test_inspect_spec() + + isula rm -f $containername + +- # use more than 10m memory limit, otherwise it might fail to run +- isula run -it -m 10m --runtime runc --name $containername $ubuntu_image perl -e 'for ($i = 0; $i < 100000000; $i++) { $a .= " " x 1024 }' +- +- isula inspect -f "{{json .State.OOMKilled}} {{.Name}}" $containername 2>&1 | sed -n '1p' | grep "true" +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container with image: ${ubuntu_image}" && ((ret++)) +- +- isula rm -f $containername +- + msg_info "${test} finished with return ${ret}..." + return ${ret} + } +diff --git a/CI/test_cases/manual_cases/oom_monitor.sh b/CI/test_cases/manual_cases/oom_monitor.sh +new file mode 100755 +index 00000000..a1c2503d +--- /dev/null ++++ b/CI/test_cases/manual_cases/oom_monitor.sh +@@ -0,0 +1,59 @@ ++#!/bin/bash ++# ++# attributes: isulad oom monitor ++# concurrent: NA ++# spend time: 6 ++ ++####################################################################### ++##- Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. ++# - iSulad licensed under the Mulan PSL v2. ++# - You can use this software according to the terms and conditions of the Mulan PSL v2. ++# - You may obtain a copy of Mulan PSL v2 at: ++# - http://license.coscl.org.cn/MulanPSL2 ++# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++# - PURPOSE. ++# - See the Mulan PSL v2 for more details. ++##- @Description:CI ++##- @Author: jikai ++##- @Create: 2024-04-03 ++####################################################################### ++ ++declare -r curr_path=$(dirname $(readlink -f "$0")) ++source ../helpers.sh ++test_data_path=$(realpath $curr_path/test_data) ++ ++function test_oom_monitor() ++{ ++ local ret=0 ++ local ubuntu_image="ubuntu" ++ local test="container oom monitor test => (${FUNCNAME[@]})" ++ containername="oommonitor" ++ ++ msg_info "${test} starting..." ++ ++ isula pull ${ubuntu_image} ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${ubuntu_image}" && return ${FAILURE} ++ ++ isula images | grep ubuntu ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${ubuntu_image}" && ((ret++)) ++ ++ # use more than 10m memory limit, otherwise it might fail to run ++ # iSulad monitor cgroup file for oom event, however oom triggers cgroup files delete ++ # if cgroup files were deleted before oom event was handled for iSulad we might failed to detect oom event ++ isula run -it -m 10m --runtime runc --name $containername $ubuntu_image perl -e 'for ($i = 0; $i < 100000000; $i++) { $a .= " " x 1024 }' ++ ++ isula inspect -f "{{json .State.OOMKilled}} {{.Name}}" $containername 2>&1 | sed -n '1p' | grep "true" ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container with image: ${ubuntu_image}" && ((ret++)) ++ ++ isula rm -f $containername ++ ++ msg_info "${test} finished with return ${ret}..." ++ return ${ret} ++} ++ ++declare -i ans=0 ++ ++test_oom_monitor || ((ans++)) ++ ++show_result ${ans} "${curr_path}/${0}" +-- +2.34.1 + diff --git a/0047-add-usage-restrictions-for-CRI-1.29-update.patch b/0047-add-usage-restrictions-for-CRI-1.29-update.patch new file mode 100644 index 0000000..e6990b7 --- /dev/null +++ b/0047-add-usage-restrictions-for-CRI-1.29-update.patch @@ -0,0 +1,27 @@ +From 9f066a405a95299c182ef7356b6518a9457af298 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Mon, 8 Apr 2024 02:52:11 +0000 +Subject: [PATCH 47/69] add usage restrictions for CRI 1.29 update + +Signed-off-by: jikai <jikai11@huawei.com> +--- + docs/design/detailed/CRI/CRI_1.29_update_design.md | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/docs/design/detailed/CRI/CRI_1.29_update_design.md b/docs/design/detailed/CRI/CRI_1.29_update_design.md +index 0a0c860b..0c66db48 100644 +--- a/docs/design/detailed/CRI/CRI_1.29_update_design.md ++++ b/docs/design/detailed/CRI/CRI_1.29_update_design.md +@@ -234,4 +234,8 @@ enum ContainerEventType { + + ### 使用限制 + +-以上特性仅保证容器运行时设置为runc时支持。 ++1. 以上新增特性，iSulad仅提供容器运行时设置为runc时的支持。 ++2. 由于cgroup oom会同时触发容器cgroup路径删除，若iSulad对oom事件处理发生在 ++cgroup路径删除之后，iSulad则无法成功捕捉容器oom事件， ++可能导致ContainerStatus中reason字段设置不正确。 ++3. iSulad不支持交叉使用不同的cgroup驱动管理容器，启动容器后iSulad的cgroup驱动配置不应该发生变化。 +-- +2.34.1 + diff --git a/0048-CDI-interface-definition.patch b/0048-CDI-interface-definition.patch new file mode 100644 index 0000000..e84db63 --- /dev/null +++ b/0048-CDI-interface-definition.patch @@ -0,0 +1,1192 @@ +From 040e39827049cd557987f505d1bd6c6e4f18b4b3 Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Mon, 11 Mar 2024 17:48:11 +0800 +Subject: [PATCH 48/69] CDI:interface definition + +--- + cmake/options.cmake | 11 +++ + src/daemon/modules/CMakeLists.txt | 11 +++ + src/daemon/modules/api/cdi_operate_api.h | 39 ++++++++++ + src/daemon/modules/device/CMakeLists.txt | 23 ++++++ + src/daemon/modules/device/cdi/CMakeLists.txt | 13 ++++ + .../device/cdi/behavior/CMakeLists.txt | 13 ++++ + .../device/cdi/behavior/cdi_container_edits.c | 49 ++++++++++++ + .../device/cdi/behavior/cdi_container_edits.h | 39 ++++++++++ + .../modules/device/cdi/behavior/cdi_device.c | 40 ++++++++++ + .../modules/device/cdi/behavior/cdi_device.h | 46 ++++++++++++ + .../modules/device/cdi/behavior/cdi_spec.c | 60 +++++++++++++++ + .../modules/device/cdi/behavior/cdi_spec.h | 56 ++++++++++++++ + .../device/cdi/behavior/cdi_spec_dirs.c | 29 +++++++ + .../device/cdi/behavior/cdi_spec_dirs.h | 47 ++++++++++++ + .../modules/device/cdi/behavior/cdi_version.c | 40 ++++++++++ + .../modules/device/cdi/behavior/cdi_version.h | 34 +++++++++ + .../device/cdi/behavior/parser/CMakeLists.txt | 3 + + .../device/cdi/behavior/parser/cdi_parser.c | 55 ++++++++++++++ + .../device/cdi/behavior/parser/cdi_parser.h | 38 ++++++++++ + .../modules/device/cdi/cdi_annotations.c | 31 ++++++++ + .../modules/device/cdi/cdi_annotations.h | 32 ++++++++ + src/daemon/modules/device/cdi/cdi_cache.c | 69 +++++++++++++++++ + src/daemon/modules/device/cdi/cdi_cache.h | 75 +++++++++++++++++++ + src/daemon/modules/device/cdi/cdi_registry.c | 25 +++++++ + src/daemon/modules/device/cdi/cdi_registry.h | 36 +++++++++ + src/daemon/modules/device/cdi_operate.c | 35 +++++++++ + 26 files changed, 949 insertions(+) + create mode 100644 src/daemon/modules/api/cdi_operate_api.h + create mode 100644 src/daemon/modules/device/CMakeLists.txt + create mode 100644 src/daemon/modules/device/cdi/CMakeLists.txt + create mode 100644 src/daemon/modules/device/cdi/behavior/CMakeLists.txt + create mode 100644 src/daemon/modules/device/cdi/behavior/cdi_container_edits.c + create mode 100644 src/daemon/modules/device/cdi/behavior/cdi_container_edits.h + create mode 100644 src/daemon/modules/device/cdi/behavior/cdi_device.c + create mode 100644 src/daemon/modules/device/cdi/behavior/cdi_device.h + create mode 100644 src/daemon/modules/device/cdi/behavior/cdi_spec.c + create mode 100644 src/daemon/modules/device/cdi/behavior/cdi_spec.h + create mode 100644 src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c + create mode 100644 src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h + create mode 100644 src/daemon/modules/device/cdi/behavior/cdi_version.c + create mode 100644 src/daemon/modules/device/cdi/behavior/cdi_version.h + create mode 100644 src/daemon/modules/device/cdi/behavior/parser/CMakeLists.txt + create mode 100644 src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c + create mode 100644 src/daemon/modules/device/cdi/behavior/parser/cdi_parser.h + create mode 100644 src/daemon/modules/device/cdi/cdi_annotations.c + create mode 100644 src/daemon/modules/device/cdi/cdi_annotations.h + create mode 100644 src/daemon/modules/device/cdi/cdi_cache.c + create mode 100644 src/daemon/modules/device/cdi/cdi_cache.h + create mode 100644 src/daemon/modules/device/cdi/cdi_registry.c + create mode 100644 src/daemon/modules/device/cdi/cdi_registry.h + create mode 100644 src/daemon/modules/device/cdi_operate.c + +diff --git a/cmake/options.cmake b/cmake/options.cmake +index b7a7c65d..c1eac472 100644 +--- a/cmake/options.cmake ++++ b/cmake/options.cmake +@@ -40,6 +40,17 @@ if (ENABLE_CRI_API_V1 STREQUAL "ON") + message("${Green}-- Enable CRI API V1${ColourReset}") + endif() + ++option(ENABLE_CDI "Enable CDI" OFF) ++if (ENABLE_CDI STREQUAL "ON") ++ if (ENABLE_CRI_API_V1) ++ add_definitions(-DENABLE_CDI) ++ set(ENABLE_CDI 1) ++ message("${Green}-- Enable CDI${ColourReset}") ++ else() ++ message("${Yellow}-- Can not enable CDI, CDI need enable CRI API V1 first ${ColourReset}") ++ endif() ++endif() ++ + option(ENABLE_SANDBOXER "Enable sandbox API" ON) + if (ENABLE_SANDBOXER STREQUAL "ON") + add_definitions(-DENABLE_SANDBOXER) +diff --git a/src/daemon/modules/CMakeLists.txt b/src/daemon/modules/CMakeLists.txt +index a70c094f..ff2ea226 100644 +--- a/src/daemon/modules/CMakeLists.txt ++++ b/src/daemon/modules/CMakeLists.txt +@@ -49,6 +49,16 @@ if (ENABLE_PLUGIN) + ) + endif() + ++if (ENABLE_CDI) ++ add_subdirectory(device) ++ list(APPEND local_modules_srcs ++ ${DEVICE_SRCS} ++ ) ++ list(APPEND local_modules_incs ++ ${DEVICE_INCS} ++ ) ++endif() ++ + set(MODULES_SRCS + ${local_modules_srcs} + PARENT_SCOPE +@@ -70,3 +80,4 @@ if(ENABLE_NETWORK) + ) + endif() + ++ +diff --git a/src/daemon/modules/api/cdi_operate_api.h b/src/daemon/modules/api/cdi_operate_api.h +new file mode 100644 +index 00000000..4f4c339e +--- /dev/null ++++ b/src/daemon/modules/api/cdi_operate_api.h +@@ -0,0 +1,39 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi device manager function definition ++ ******************************************************************************/ ++#ifndef CDI_OPERATE_API_H ++#define CDI_OPERATE_API_H ++ ++#include <isula_libutils/oci_runtime_spec.h> ++#include <isula_libutils/json_common.h> ++ ++#include "utils_array.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++int cdi_operate_registry_init(char **specs_dirs, size_t specs_dirs_len); ++ ++char *cdi_operate_refresh(void); ++ ++string_array *cdi_operate_inject_devices(oci_runtime_spec *spec, string_array *devices, char **error); ++ ++char *cdi_operate_parse_annotations(json_map_string_string *annotations, string_array **keys, string_array **devices); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +diff --git a/src/daemon/modules/device/CMakeLists.txt b/src/daemon/modules/device/CMakeLists.txt +new file mode 100644 +index 00000000..e3468c0b +--- /dev/null ++++ b/src/daemon/modules/device/CMakeLists.txt +@@ -0,0 +1,23 @@ ++# get current directory sources files ++aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} device_top_srcs) ++ ++add_subdirectory(cdi) ++ ++set(local_device_srcs ++ ${device_top_srcs} ++ ${CDI_SRCS} ++ ) ++ ++set(local_device_incs ++ ${CMAKE_CURRENT_SOURCE_DIR} ++ ${CDI_INCS} ++ ) ++ ++set(DEVICE_SRCS ++ ${local_device_srcs} ++ PARENT_SCOPE ++ ) ++set(DEVICE_INCS ++ ${local_device_incs} ++ PARENT_SCOPE ++ ) +\ No newline at end of file +diff --git a/src/daemon/modules/device/cdi/CMakeLists.txt b/src/daemon/modules/device/cdi/CMakeLists.txt +new file mode 100644 +index 00000000..12d9b222 +--- /dev/null ++++ b/src/daemon/modules/device/cdi/CMakeLists.txt +@@ -0,0 +1,13 @@ ++aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} cdi_top_srcs) ++add_subdirectory(behavior) ++ ++set(CDI_SRCS ++ ${cdi_top_srcs} ++ ${BEHAVIOR_SRCS} ++ PARENT_SCOPE ++ ) ++set(CDI_INCS ++ ${CMAKE_CURRENT_SOURCE_DIR} ++ ${BEHAVIOR_INCS} ++ PARENT_SCOPE ++ ) +diff --git a/src/daemon/modules/device/cdi/behavior/CMakeLists.txt b/src/daemon/modules/device/cdi/behavior/CMakeLists.txt +new file mode 100644 +index 00000000..39881a7b +--- /dev/null ++++ b/src/daemon/modules/device/cdi/behavior/CMakeLists.txt +@@ -0,0 +1,13 @@ ++aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} behavior_top_srcs) ++add_subdirectory(parser) ++ ++set(BEHAVIOR_SRCS ++ ${behavior_top_srcs} ++ ${PARSER_SRCS} ++ PARENT_SCOPE ++ ) ++set(BEHAVIOR_INCS ++ ${CMAKE_CURRENT_SOURCE_DIR} ++ ${CMAKE_CURRENT_SOURCE_DIR}/parser/ ++ PARENT_SCOPE ++ ) +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c +new file mode 100644 +index 00000000..ce7b16db +--- /dev/null ++++ b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c +@@ -0,0 +1,49 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi container edits function ++ ******************************************************************************/ ++#include "cdi_container_edits.h" ++ ++// PRESTART_HOOK is the name of the OCI "prestart" hook. ++#define PRESTART_HOOK "prestart" ++// CREATE_RUNTIME_HOOK is the name of the OCI "createRuntime" hook. ++#define CREATE_RUNTIME_HOOK "createRuntime" ++// CREATE_CONTAINER_HOOK is the name of the OCI "createContainer" hook. ++#define CREATE_CONTAINER_HOOK "createContainer" ++// START_CONTAINER_HOOK is the name of the OCI "startContainer" hook. ++#define START_CONTAINER_HOOK "startContainer" ++// POSTSTART_HOOK is the name of the OCI "poststart" hook. ++#define POSTSTART_HOOK "poststart" ++// POSTSTOP_HOOK is the name of the OCI "poststop" hook. ++#define POSTSTOP_HOOK "poststop" ++ ++char *cdi_container_edits_apply(cdi_container_edits *e, oci_runtime_spec *spec) ++{ ++ return NULL; ++} ++ ++char *cdi_container_edits_validate(cdi_container_edits *e) ++{ ++ return NULL; ++} ++ ++cdi_container_edits *cdi_container_edits_append(cdi_container_edits *e, cdi_container_edits *o) ++{ ++ return NULL; ++} ++ ++bool cdi_container_edits_is_empty(cdi_container_edits *e) ++{ ++ return true; ++} ++ +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_container_edits.h b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.h +new file mode 100644 +index 00000000..7b16d2bc +--- /dev/null ++++ b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.h +@@ -0,0 +1,39 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi container edits function definition ++ ******************************************************************************/ ++#ifndef CDI_CONTAINER_EDITS_H ++#define CDI_CONTAINER_EDITS_H ++ ++#include <isula_libutils/cdi_container_edits.h> ++#include <isula_libutils/cdi_device_node.h> ++#include <isula_libutils/oci_runtime_spec.h> ++#include <isula_libutils/cdi_hook.h> ++#include <isula_libutils/cdi_mount.h> ++ ++#include "utils_array.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++char *cdi_container_edits_apply(cdi_container_edits *e, oci_runtime_spec *spec); ++char *cdi_container_edits_validate(cdi_container_edits *e); ++cdi_container_edits *cdi_container_edits_append(cdi_container_edits *e, cdi_container_edits *o); ++bool cdi_container_edits_is_empty(cdi_container_edits *e); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +\ No newline at end of file +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_device.c b/src/daemon/modules/device/cdi/behavior/cdi_device.c +new file mode 100644 +index 00000000..9904e9ee +--- /dev/null ++++ b/src/daemon/modules/device/cdi/behavior/cdi_device.c +@@ -0,0 +1,40 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi device function ++ ******************************************************************************/ ++#include "cdi_device.h" ++ ++void free_cdi_cache_device(struct cdi_cache_device *d) ++{ ++ (void)d; ++} ++ ++struct cdi_cache_device *cdi_device_new_device(struct cdi_cache_spec *spec, cdi_device *d, char **error) ++{ ++ return NULL; ++} ++ ++struct cdi_cache_spec *cdi_device_get_spec(struct cdi_cache_device *d) ++{ ++ return NULL; ++} ++ ++char *cdi_device_get_qualified_name(struct cdi_cache_device *d) ++{ ++ return NULL; ++} ++ ++cdi_container_edits *cdi_device_edits(struct cdi_cache_device *d) ++{ ++ return NULL; ++} +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_device.h b/src/daemon/modules/device/cdi/behavior/cdi_device.h +new file mode 100644 +index 00000000..3f460152 +--- /dev/null ++++ b/src/daemon/modules/device/cdi/behavior/cdi_device.h +@@ -0,0 +1,46 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi device function definition ++ ******************************************************************************/ ++#ifndef CDI_DEVICE_H ++#define CDI_DEVICE_H ++ ++#include <stdbool.h> ++#include <isula_libutils/cdi_device.h> ++#include <isula_libutils/oci_runtime_spec.h> ++ ++#include "cdi_container_edits.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++struct cdi_cache_spec; ++ ++struct cdi_cache_device { ++ cdi_device *raw_device; ++ struct cdi_cache_spec *cache_spec; ++}; ++ ++void free_cdi_cache_device(struct cdi_cache_device *d); ++ ++struct cdi_cache_device *cdi_device_new_device(struct cdi_cache_spec *spec, cdi_device *d, char **error); ++struct cdi_cache_spec *cdi_device_get_spec(struct cdi_cache_device *d); ++char *cdi_device_get_qualified_name(struct cdi_cache_device *d); ++cdi_container_edits *cdi_device_edits(struct cdi_cache_device *d); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +\ No newline at end of file +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec.c b/src/daemon/modules/device/cdi/behavior/cdi_spec.c +new file mode 100644 +index 00000000..38fc9e38 +--- /dev/null ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec.c +@@ -0,0 +1,60 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi spec function ++ ******************************************************************************/ ++#include "cdi_spec.h" ++ ++void free_cdi_cache_spec(struct cdi_cache_spec *s) ++{ ++ (void)s; ++} ++ ++struct cdi_cache_spec *cdi_spec_read_spec(const char *path, int priority, char **error) ++{ ++ return NULL; ++} ++ ++struct cdi_cache_spec *cdi_spec_new_spec(cdi_spec *raw, const char *path, int priority, char **error) ++{ ++ return NULL; ++} ++ ++const char *cdi_spec_get_vendor(struct cdi_cache_spec *s) ++{ ++ return NULL; ++} ++ ++const char *cdi_spec_get_class(struct cdi_cache_spec *s) ++{ ++ return NULL; ++} ++ ++struct cdi_cache_device *cdi_spec_get_cache_device(struct cdi_cache_spec *s, const char *name) ++{ ++ return NULL; ++} ++ ++const char *cdi_spec_get_path(struct cdi_cache_spec *s) ++{ ++ return NULL; ++} ++ ++int cdi_spec_get_priority(struct cdi_cache_spec *s) ++{ ++ return 0; ++} ++ ++cdi_container_edits *cdi_spec_edits(struct cdi_cache_spec *s) ++{ ++ return NULL; ++} +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec.h b/src/daemon/modules/device/cdi/behavior/cdi_spec.h +new file mode 100644 +index 00000000..bd4fc9d1 +--- /dev/null ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec.h +@@ -0,0 +1,56 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi spec function definition ++ ******************************************************************************/ ++#ifndef CDI_SPEC_H ++#define CDI_SPEC_H ++ ++#include <isula_libutils/cdi_spec.h> ++#include <isula_libutils/oci_runtime_spec.h> ++ ++#include "map.h" ++#include "cdi_container_edits.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++struct cdi_cache_device; ++ ++struct cdi_cache_spec { ++ cdi_spec *raw_spec; ++ char *vendor; ++ char *class; ++ char *path; ++ int priority; ++ map_t *devices; // MAP_STR_PTR devices[cdi_device.name] = cdi_cache_device* ++}; ++ ++#define CDI_DEFAULT_SPEC_EXT ".json" ++ ++void free_cdi_cache_spec(struct cdi_cache_spec *s); ++ ++struct cdi_cache_spec *cdi_spec_read_spec(const char *path, int priority, char **error); ++struct cdi_cache_spec *cdi_spec_new_spec(cdi_spec *raw, const char *path, int priority, char **error); ++const char *cdi_spec_get_vendor(struct cdi_cache_spec *s); ++const char *cdi_spec_get_class(struct cdi_cache_spec *s); ++struct cdi_cache_device *cdi_spec_get_cache_device(struct cdi_cache_spec *s, const char *name); ++const char *cdi_spec_get_path(struct cdi_cache_spec *s); ++int cdi_spec_get_priority(struct cdi_cache_spec *s); ++cdi_container_edits *cdi_spec_edits(struct cdi_cache_spec *s); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +\ No newline at end of file +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c +new file mode 100644 +index 00000000..5df4c937 +--- /dev/null ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c +@@ -0,0 +1,29 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi spec dirs function ++ ******************************************************************************/ ++#include "cdi_spec_dirs.h" ++ ++#define DEFAULT_SPEC_DIRS_LEN 2 ++static char *default_spec_dirs_items[DEFAULT_SPEC_DIRS_LEN] = {CDI_DEFAULT_STATIC_DIR, CDI_DEFAULT_DYNAMIC_DIR}; ++ ++string_array g_default_spec_dirs = { ++ .items = default_spec_dirs_items, ++ .len = DEFAULT_SPEC_DIRS_LEN, ++ .cap = DEFAULT_SPEC_DIRS_LEN, ++}; ++ ++char *cdi_scan_spec_dirs(string_array *dirs, struct cdi_scan_fn_maps *scan_fn_maps, cdi_scan_spec_func scan_fn) ++{ ++ return NULL; ++} +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h +new file mode 100644 +index 00000000..bd00e318 +--- /dev/null ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h +@@ -0,0 +1,47 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi spec dirs function definition ++ ******************************************************************************/ ++#ifndef CDI_SPEC_DIRS_H ++#define CDI_SPEC_DIRS_H ++ ++#include "cdi_cache.h" ++#include "utils_array.h" ++#include "map.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++#define CDI_DEFAULT_STATIC_DIR "/etc/cdi" ++#define CDI_DEFAULT_DYNAMIC_DIR "/var/run/cdi" ++ ++extern string_array g_default_spec_dirs; ++ ++struct cdi_scan_fn_maps { ++ map_t *specs; ++ map_t *devices; ++ map_t *conflicts; ++ map_t *spec_errors; ++ string_array *result; ++}; ++typedef char *(*cdi_scan_spec_func)(struct cdi_scan_fn_maps *scan_fn_maps, const char *path, int priority, ++ struct cdi_cache_spec *spec, char **error); ++ ++char *cdi_scan_spec_dirs(string_array *dirs, struct cdi_scan_fn_maps *scan_fn_maps, cdi_scan_spec_func scan_fn); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +\ No newline at end of file +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_version.c b/src/daemon/modules/device/cdi/behavior/cdi_version.c +new file mode 100644 +index 00000000..3e87c111 +--- /dev/null ++++ b/src/daemon/modules/device/cdi/behavior/cdi_version.c +@@ -0,0 +1,40 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi version function ++ ******************************************************************************/ ++#include "cdi_version.h" ++ ++#define CDI_V_CURRENT_VERSION "v"##CDI_CURRENT_VERSION ++ ++#define CDI_V010 "v0.1.0" ++#define CDI_V020 "v0.2.0" ++#define CDI_V030 "v0.3.0" ++#define CDI_V040 "v0.4.0" ++#define CDI_V050 "v0.5.0" ++#define CDI_V060 "v0.6.0" ++#define CDI_V_EARLIEST CDI_V030 ++ ++const char *cdi_minimum_required_version(cdi_spec *spec) ++{ ++ return NULL; ++} ++ ++bool cdi_is_greater_than_version(const char *v, const char *o) ++{ ++ return true; ++} ++ ++bool cdi_is_valid_version(const char *spec_version) ++{ ++ return true; ++} +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_version.h b/src/daemon/modules/device/cdi/behavior/cdi_version.h +new file mode 100644 +index 00000000..99b7e692 +--- /dev/null ++++ b/src/daemon/modules/device/cdi/behavior/cdi_version.h +@@ -0,0 +1,34 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi version function definition ++ ******************************************************************************/ ++#ifndef CDI_VERSION_H ++#define CDI_VERSION_H ++ ++#include <isula_libutils/cdi_spec.h> ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++#define CDI_CURRENT_VERSION "0.6.0" ++ ++const char *cdi_minimum_required_version(cdi_spec *spec); ++bool cdi_is_greater_than_version(const char *v, const char *o); ++bool cdi_is_valid_version(const char *spec_version); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +\ No newline at end of file +diff --git a/src/daemon/modules/device/cdi/behavior/parser/CMakeLists.txt b/src/daemon/modules/device/cdi/behavior/parser/CMakeLists.txt +new file mode 100644 +index 00000000..9bf5f3f2 +--- /dev/null ++++ b/src/daemon/modules/device/cdi/behavior/parser/CMakeLists.txt +@@ -0,0 +1,3 @@ ++# get current directory sources files ++aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_cdi_parser_srcs) ++set(PARSER_SRCS ${local_cdi_parser_srcs} PARENT_SCOPE) +diff --git a/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c +new file mode 100644 +index 00000000..45048f9a +--- /dev/null ++++ b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c +@@ -0,0 +1,55 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi parser linux function ++ ******************************************************************************/ ++#include "cdi_parser.h" ++ ++char *cdi_parser_qualified_name(const char *vendor, const char *class, const char *name) ++{ ++ return NULL; ++} ++ ++bool cdi_parser_is_qualified_name(const char *device) ++{ ++ return true; ++} ++ ++char *cdi_parser_parse_qualified_name(const char *device, char **vendor, char **class, char **name) ++{ ++ return NULL; ++} ++ ++int cdi_parser_parse_device(const char *device, char **vendor, char **class, char **name) ++{ ++ return 0; ++} ++ ++int cdi_parser_parse_qualifier(const char *kind, char **vendor, char **class) ++{ ++ return 0; ++} ++ ++char *cdi_parser_validate_vendor_name(const char *vendor) ++{ ++ return NULL; ++} ++ ++char *cdi_parser_validate_class_name(const char *class) ++{ ++ return NULL; ++} ++ ++char *cdi_parser_validate_device_name(const char *name) ++{ ++ return NULL; ++} +diff --git a/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.h b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.h +new file mode 100644 +index 00000000..d9c057ea +--- /dev/null ++++ b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.h +@@ -0,0 +1,38 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi parser function definition ++ ******************************************************************************/ ++#ifndef CDI_PARSER_H ++#define CDI_PARSER_H ++ ++#include <stdbool.h> ++#include <stddef.h> ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++char *cdi_parser_qualified_name(const char *vendor, const char *class, const char *name); ++bool cdi_parser_is_qualified_name(const char *device); ++char *cdi_parser_parse_qualified_name(const char *device, char **vendor, char **class, char **name); ++int cdi_parser_parse_device(const char *device, char **vendor, char **class, char **name); ++int cdi_parser_parse_qualifier(const char *kind, char **vendor, char **class); ++char *cdi_parser_validate_vendor_name(const char *vendor); ++char *cdi_parser_validate_class_name(const char *class); ++char *cdi_parser_validate_device_name(const char *name); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +\ No newline at end of file +diff --git a/src/daemon/modules/device/cdi/cdi_annotations.c b/src/daemon/modules/device/cdi/cdi_annotations.c +new file mode 100644 +index 00000000..3cb9be84 +--- /dev/null ++++ b/src/daemon/modules/device/cdi/cdi_annotations.c +@@ -0,0 +1,31 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi annotations function ++ ******************************************************************************/ ++#include "cdi_annotations.h" ++ ++#include <ctype.h> ++#include <isula_libutils/log.h> ++#include <isula_libutils/auto_cleanup.h> ++#include <isula_libutils/utils_string.h> ++ ++#include "error.h" ++#include "utils.h" ++#include "cdi_parser.h" ++ ++#define CDI_ANNOTATIONS_PREFIX "cdi.k8s.io/" ++ ++char *cdi_parse_annotations(json_map_string_string *annotations, string_array **keys, string_array **devices) ++{ ++ return NULL; ++} +diff --git a/src/daemon/modules/device/cdi/cdi_annotations.h b/src/daemon/modules/device/cdi/cdi_annotations.h +new file mode 100644 +index 00000000..52355099 +--- /dev/null ++++ b/src/daemon/modules/device/cdi/cdi_annotations.h +@@ -0,0 +1,32 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi annotations function definition ++ ******************************************************************************/ ++#ifndef CDI_ANNOTATIONS_H ++#define CDI_ANNOTATIONS_H ++ ++#include <isula_libutils/json_common.h> ++ ++#include "utils_array.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++char *cdi_parse_annotations(json_map_string_string *annotations, string_array **keys, string_array **devices); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +\ No newline at end of file +diff --git a/src/daemon/modules/device/cdi/cdi_cache.c b/src/daemon/modules/device/cdi/cdi_cache.c +new file mode 100644 +index 00000000..9c54acbf +--- /dev/null ++++ b/src/daemon/modules/device/cdi/cdi_cache.c +@@ -0,0 +1,69 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi cache function ++ ******************************************************************************/ ++#include "cdi_cache.h" ++ ++void free_cdi_cache(struct cdi_cache *c) ++{ ++ (void)c; ++} ++ ++struct cdi_cache *cdi_new_cache(string_array *spec_dirs, char **error) ++{ ++ return NULL; ++} ++ ++static string_array *cdi_inject_devices(struct cdi_cache *c, oci_runtime_spec *oci_spec, string_array *devices, char **error) ++{ ++ return NULL; ++} ++ ++static char *cdi_configure(struct cdi_cache *c, string_array *spec_dirs) ++{ ++ return NULL; ++} ++ ++static char *cdi_refresh(struct cdi_cache *c) ++{ ++ return NULL; ++} ++ ++static map_t *cdi_get_errors(struct cdi_cache *c) ++{ ++ return NULL; ++} ++ ++static string_array *cdi_get_spec_directories(struct cdi_cache *c) ++{ ++ return NULL; ++} ++ ++static map_t *cdi_get_spec_dir_errors(struct cdi_cache *c) ++{ ++ return NULL; ++} ++ ++static struct cdi_cache_ops g_cdi_cache_ops = { ++ .inject_devices = cdi_inject_devices, ++ .configure = cdi_configure, ++ .refresh = cdi_refresh, ++ .get_errors = cdi_get_errors, ++ .get_spec_directories = cdi_get_spec_directories, ++ .get_spec_dir_errors = cdi_get_spec_dir_errors ++}; ++ ++struct cdi_cache_ops *cdi_get_cache_ops(void) ++{ ++ return &g_cdi_cache_ops; ++} +\ No newline at end of file +diff --git a/src/daemon/modules/device/cdi/cdi_cache.h b/src/daemon/modules/device/cdi/cdi_cache.h +new file mode 100644 +index 00000000..92fb64af +--- /dev/null ++++ b/src/daemon/modules/device/cdi/cdi_cache.h +@@ -0,0 +1,75 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi cache function definition ++ ******************************************************************************/ ++#ifndef CDI_CACHE_H ++#define CDI_CACHE_H ++ ++#include <stdbool.h> ++#include <stddef.h> ++#include <isula_libutils/oci_runtime_spec.h> ++ ++#include "utils_array.h" ++#include "map.h" ++#include "cdi_device.h" ++#include "cdi_spec.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++struct cdi_cache; ++ ++struct cdi_cache_ops { ++ // injecting CDI devices into an OCI Spec. ++ // Resolver ++ string_array *(*inject_devices)(struct cdi_cache *c, oci_runtime_spec *spec, string_array *devices, char **error); ++ ++ // refreshing the cache of CDI Specs and devices. ++ // Refresher ++ char *(*configure)(struct cdi_cache *c, string_array *spec_dirs); ++ char *(*refresh)(struct cdi_cache *c); ++ map_t *(*get_errors)(struct cdi_cache *c); ++ string_array *(*get_spec_directories)(struct cdi_cache *c); ++ map_t *(*get_spec_dir_errors)(struct cdi_cache *c); ++}; ++ ++struct cdi_watch { ++ int watcher_fd; // inotify fd ++ map_t *tracked; // MAP_STR_BOOL tracked[spec_dirs[i]] = bool ++ map_t *wd_dirs; // MAP_INT_STR wd_dirs[wd] = spec_dirs[i] ++}; ++ ++// Cache stores CDI Specs loaded from Spec directories. ++struct cdi_cache { ++ pthread_mutex_t mutex; ++ string_array *spec_dirs; // cdi-spec-dirs will scan for CDI Spec files ++ map_t *specs; // MAP_STR_PTR specs[vendor] = cdi_cache_spec** ++ map_t *devices; // MAP_STR_PTR devices[cdi_device.name] = cdi_cache_device* ++ map_t *errors; // MAP_STR_PTR errors[cdi_cache_spec.path] = string_array *errors ++ map_t *dir_errors; // MAP_STR_STR dir_errors[spec_dirs[i]] = error ++ ++ bool auto_refresh; ++ struct cdi_watch *watch; ++}; ++ ++void free_cdi_cache(struct cdi_cache *c); ++ ++struct cdi_cache *cdi_new_cache(string_array *spec_dirs, char **error); ++struct cdi_cache_ops *cdi_get_cache_ops(void); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +\ No newline at end of file +diff --git a/src/daemon/modules/device/cdi/cdi_registry.c b/src/daemon/modules/device/cdi/cdi_registry.c +new file mode 100644 +index 00000000..68767a5f +--- /dev/null ++++ b/src/daemon/modules/device/cdi/cdi_registry.c +@@ -0,0 +1,25 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi registry function ++ ******************************************************************************/ ++#include "cdi_registry.h" ++ ++int cdi_registry_init(string_array *spec_dirs) ++{ ++ return 0; ++} ++ ++struct cdi_registry *cdi_get_registry(void) ++{ ++ return NULL; ++} +diff --git a/src/daemon/modules/device/cdi/cdi_registry.h b/src/daemon/modules/device/cdi/cdi_registry.h +new file mode 100644 +index 00000000..c27d37e3 +--- /dev/null ++++ b/src/daemon/modules/device/cdi/cdi_registry.h +@@ -0,0 +1,36 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi registry function definition ++ ******************************************************************************/ ++#ifndef CDI_REGISTRY_H ++#define CDI_REGISTRY_H ++ ++#include "cdi_cache.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++struct cdi_registry { ++ struct cdi_cache *cdi_cache; ++ struct cdi_cache_ops *ops; ++}; ++ ++int cdi_registry_init(string_array *spec_dirs); ++struct cdi_registry *cdi_get_registry(void); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +\ No newline at end of file +diff --git a/src/daemon/modules/device/cdi_operate.c b/src/daemon/modules/device/cdi_operate.c +new file mode 100644 +index 00000000..c7aa77d8 +--- /dev/null ++++ b/src/daemon/modules/device/cdi_operate.c +@@ -0,0 +1,35 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi device manager function ++ ******************************************************************************/ ++#include "cdi_operate_api.h" ++ ++int cdi_operate_registry_init(char **specs_dirs, size_t specs_dirs_len) ++{ ++ return 0; ++} ++ ++char *cdi_operate_refresh(void) ++{ ++ return NULL; ++} ++ ++string_array *cdi_operate_inject_devices(oci_runtime_spec *spec, string_array *devices, char **error) ++{ ++ return NULL; ++} ++ ++char *cdi_operate_parse_annotations(json_map_string_string *annotations, string_array **keys, string_array **devices) ++{ ++ return NULL; ++} +\ No newline at end of file +-- +2.34.1 + diff --git a/0049-distinguish-between-runtime-and-runtime_cmd-in-isula.patch b/0049-distinguish-between-runtime-and-runtime_cmd-in-isula.patch new file mode 100644 index 0000000..4366d7a --- /dev/null +++ b/0049-distinguish-between-runtime-and-runtime_cmd-in-isula.patch @@ -0,0 +1,145 @@ +From 491baece02522128720b3bd992a76dc5148aa7b2 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Mon, 8 Apr 2024 11:37:13 +0800 +Subject: [PATCH 49/69] distinguish between runtime and runtime_cmd in + isulad-shim + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/cmd/isulad-shim/process.c | 20 +++++++++---------- + src/cmd/isulad-shim/process.h | 4 ++-- + .../modules/runtime/isula/isula_rt_ops.c | 2 ++ + 3 files changed, 14 insertions(+), 12 deletions(-) + +diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c +index 8a4ca175..6b5f8f7f 100644 +--- a/src/cmd/isulad-shim/process.c ++++ b/src/cmd/isulad-shim/process.c +@@ -1131,7 +1131,7 @@ static int init_root_path(process_t *p) + return SHIM_ERR; + } + +- if (buffer->nappend(buffer, PATH_MAX, "%s/%s", state_path, p->runtime) < 0) { ++ if (buffer->nappend(buffer, PATH_MAX, "%s/%s", state_path, p->state->runtime) < 0) { + ERROR("Failed to append state_path\n"); + isula_buffer_free(buffer); + return SHIM_ERR; +@@ -1146,7 +1146,7 @@ static int init_root_path(process_t *p) + return SHIM_OK; + } + +-process_t *new_process(char *id, char *bundle, char *runtime) ++process_t *new_process(char *id, char *bundle, char *runtime_cmd) + { + shim_client_process_state *p_state; + process_t *p = NULL; +@@ -1174,7 +1174,7 @@ process_t *new_process(char *id, char *bundle, char *runtime) + + p->id = id; + p->bundle = bundle; +- p->runtime = runtime; ++ p->runtime_cmd = runtime_cmd; + p->state = p_state; + p->console_sock_path = NULL; + p->exit_fd = -1; +@@ -1247,7 +1247,7 @@ static void set_common_params(process_t *p, const char *params[], int *index, co + { + int j; + +- params[(*index)++] = p->runtime; ++ params[(*index)++] = p->runtime_cmd; + for (j = 0; j < p->state->runtime_args_len; j++) { + params[(*index)++] = p->state->runtime_args[j]; + } +@@ -1261,7 +1261,7 @@ static void set_common_params(process_t *p, const char *params[], int *index, co + + // In addition to kata, other commonly used oci runtimes (runc, crun, youki, gvisor) + // need to set the --root option +- if (strcasecmp(p->runtime, "kata-runtime") != 0) { ++ if (strcasecmp(p->state->runtime, "kata-runtime") != 0) { + params[(*index)++] = "--root"; + params[(*index)++] = p->root_path; + } +@@ -1347,7 +1347,7 @@ static void process_kill_all(process_t *p) + params[i++] = p->id; + params[i++] = "SIGKILL"; + +- (void)cmd_combined_output(p->runtime, params, output, &output_len); ++ (void)cmd_combined_output(p->runtime_cmd, params, output, &output_len); + + return; + } +@@ -1375,7 +1375,7 @@ static void process_delete(process_t *p) + params[i++] = "--force"; + params[i++] = p->id; + +- (void)cmd_combined_output(p->runtime, params, output, &output_len); ++ (void)cmd_combined_output(p->runtime_cmd, params, output, &output_len); + + return; + } +@@ -1444,8 +1444,8 @@ static void exec_runtime_process(process_t *p, int exec_fd) + + const char *params[MAX_RUNTIME_ARGS] = { 0 }; + get_runtime_cmd(p, log_path, pid_path, process_desc, params); +- execvp(p->runtime, (char * const *)params); +- (void)dprintf(exec_fd, "run process: %s error: %s", p->runtime, strerror(errno)); ++ execvp(p->runtime_cmd, (char * const *)params); ++ (void)dprintf(exec_fd, "run process: %s error: %s", p->runtime_cmd, strerror(errno)); + _exit(EXIT_FAILURE); + } + +@@ -1586,7 +1586,7 @@ static int waitpid_with_timeout(int ctr_pid, int *status, const uint64_t timeou + static int wait_container_process_with_timeout(process_t *p, const uint64_t timeout, int *status) + { + // currently, kata runtime does not support setting timeout during exec +- if (strcasecmp(p->runtime, "kata-runtime") != 0 && timeout > 0) { ++ if (strcasecmp(p->state->runtime, "kata-runtime") != 0 && timeout > 0) { + return waitpid_with_timeout(p->ctr_pid, status, timeout); + } + +diff --git a/src/cmd/isulad-shim/process.h b/src/cmd/isulad-shim/process.h +index 32ba7366..05fd87b0 100644 +--- a/src/cmd/isulad-shim/process.h ++++ b/src/cmd/isulad-shim/process.h +@@ -44,7 +44,7 @@ typedef struct { + typedef struct process { + char *id; + char *bundle; +- char *runtime; ++ char *runtime_cmd; + char *console_sock_path; // pty socket path + char *workdir; + char *root_path; +@@ -70,7 +70,7 @@ typedef struct { + int status; + } process_exit_t; + +-process_t* new_process(char *id, char *bundle, char *runtime); ++process_t* new_process(char *id, char *bundle, char *runtime_cmd); + + int prepare_attach_socket(process_t *p); + +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index b9aba3e3..bc3c36c8 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -1154,6 +1154,7 @@ int rt_isula_create(const char *id, const char *runtime, const rt_create_params_ + p.isulad_stdin = (char *)params->stdin; + p.isulad_stdout = (char *)params->stdout; + p.isulad_stderr = (char *)params->stderr; ++ p.runtime = (char *)runtime; + p.runtime_args = (char **)runtime_args; + p.runtime_args_len = runtime_args_len; + p.attach_socket = attach_socket; +@@ -1409,6 +1410,7 @@ static int preparation_exec(const char *id, const char *runtime, const char *wor + p.isulad_stdout = (char *)params->console_fifos[1]; + p.isulad_stderr = (char *)params->console_fifos[2]; + p.resize_fifo = resize_fifo_dir; ++ p.runtime = (char *)runtime; + p.runtime_args = (char **)runtime_args; + p.runtime_args_len = runtime_args_len; + copy_process(&p, process); +-- +2.34.1 + diff --git a/0050-Use-user-defined-shm-for-CRI-request.patch b/0050-Use-user-defined-shm-for-CRI-request.patch new file mode 100644 index 0000000..2d61c9d --- /dev/null +++ b/0050-Use-user-defined-shm-for-CRI-request.patch @@ -0,0 +1,171 @@ +From 162123bdec0f45f7b2001b2b0b83705cc6b9b1b1 Mon Sep 17 00:00:00 2001 +From: xuxuepeng <xuxuepeng1@huawei.com> +Date: Mon, 8 Apr 2024 20:53:57 +0800 +Subject: [PATCH 50/69] Use user defined shm for CRI request + +Signed-off-by: xuxuepeng <xuxuepeng1@huawei.com> +--- + src/daemon/modules/spec/specs_mount.c | 128 +++++++++++++++++--------- + 1 file changed, 85 insertions(+), 43 deletions(-) + +diff --git a/src/daemon/modules/spec/specs_mount.c b/src/daemon/modules/spec/specs_mount.c +index 20bf5378..6903ae40 100644 +--- a/src/daemon/modules/spec/specs_mount.c ++++ b/src/daemon/modules/spec/specs_mount.c +@@ -2799,33 +2799,31 @@ out_free: + return ret; + } + +-#define SHM_MOUNT_POINT "/dev/shm" +-static int set_shm_path(host_config *host_spec, container_config_v2_common_config *v2_spec) ++static inline int set_sharable_ipc_mode(host_config *host_spec, container_config_v2_common_config *v2_spec) + { +- int ret = 0; +- container_t *cont = NULL; +- char *tmp_cid = NULL; +- char *right_path = NULL; +- +- // ignore shm of system container +- if (host_spec->system_container) { ++ free(v2_spec->shm_path); ++#ifdef ENABLE_CRI_API_V1 ++ // In the case of sandbox API is used, the shm path has already been created in CRI, ++ // so we need to use the sandbox's shm path ++ if (is_sandbox_container(v2_spec->sandbox_info)) { ++ v2_spec->shm_path = util_strdup_s(v2_spec->sandbox_info->shm_path); + return 0; + } +- // setup shareable dirs +- if (is_shareable_ipc(host_spec->ipc_mode)) { +- // has mount for /dev/shm +- if (has_mount_shm(host_spec, v2_spec)) { +- return 0; +- } ++#endif ++ v2_spec->shm_path = get_prepare_share_shm_path(host_spec->runtime, v2_spec->id); ++ if (v2_spec->shm_path == NULL) { ++ ERROR("Failed to get prepare share shm path"); ++ return -1; ++ } + +- v2_spec->shm_path = get_prepare_share_shm_path(host_spec->runtime, v2_spec->id); +- if (v2_spec->shm_path == NULL) { +- ERROR("Failed to get prepare share shm path"); +- return -1; +- } ++ return 0; ++} + +- return 0; +- } ++static inline int set_connected_container_shm_path(host_config *host_spec, container_config_v2_common_config *v2_spec) ++{ ++ container_t *cont = NULL; ++ char *tmp_cid = NULL; ++ char *right_path = NULL; + + #ifdef ENABLE_CRI_API_V1 + // Sandbox API is used and the connected container is actually a sandbox +@@ -2833,34 +2831,78 @@ static int set_shm_path(host_config *host_spec, container_config_v2_common_confi + if (namespace_is_sandbox(host_spec->ipc_mode, v2_spec->sandbox_info)) { + free(v2_spec->shm_path); + v2_spec->shm_path = util_strdup_s(v2_spec->sandbox_info->shm_path); +- goto out; ++ return 0; + } + #endif + +- if (namespace_is_container(host_spec->ipc_mode)) { +- tmp_cid = namespace_get_connected_container(host_spec->ipc_mode); +- cont = containers_store_get(tmp_cid); +- if (cont == NULL) { +- ERROR("Invalid share path: %s", host_spec->ipc_mode); +- ret = -1; +- goto out; +- } +- right_path = util_strdup_s(cont->common_config->shm_path); +- container_unref(cont); +- } else if (namespace_is_host(host_spec->ipc_mode)) { +- if (!util_file_exists(SHM_MOUNT_POINT)) { +- ERROR("/dev/shm is not mounted, but must be for --ipc=host"); +- ret = -1; +- goto out; +- } +- right_path = util_strdup_s(SHM_MOUNT_POINT); ++ tmp_cid = namespace_get_connected_container(host_spec->ipc_mode); ++ cont = containers_store_get(tmp_cid); ++ if (cont == NULL) { ++ ERROR("Invalid share path: %s", host_spec->ipc_mode); ++ return -1; + } ++ right_path = util_strdup_s(cont->common_config->shm_path); ++ container_unref(cont); + + free(v2_spec->shm_path); + v2_spec->shm_path = right_path; +-out: +- free(tmp_cid); +- return ret; ++ ++ return 0; ++} ++ ++#define SHM_MOUNT_POINT "/dev/shm" ++static inline int set_host_ipc_shm_path(container_config_v2_common_config *v2_spec) ++{ ++ if (!util_file_exists(SHM_MOUNT_POINT)) { ++ ERROR("/dev/shm is not mounted, but must be for --ipc=host"); ++ return -1; ++ } ++ free(v2_spec->shm_path); ++ v2_spec->shm_path = util_strdup_s(SHM_MOUNT_POINT); ++ return 0; ++} ++ ++/** ++ * There are 4 cases for setting shm path: ++ * 1. The user defined /dev/shm in mounts, which takes the first priority ++ * 2. If sharable is set in ipc mode (or by default ipc_mode is null), the container provides shm path, ++ * in the case of sandbox API is used, the sandbox module has already provided shm path ++ * 3. Use the connected container's shm path if ipc_mode is set to container:<cid>, ++ * if connected containerd is a sandbox, use the sandbox's shm path ++ * 4. Use /dev/shm if ipc_mode is set to host ++ */ ++static int set_shm_path(host_config *host_spec, container_config_v2_common_config *v2_spec) ++{ ++ // ignore shm of system container ++ if (host_spec->system_container) { ++ return 0; ++ } ++ ++ // case 1: Defined in mounts already ++ if (has_mount_shm(host_spec, v2_spec)) { ++ return 0; ++ } ++ ++ // case 2: Container has its own IPC namespace ++ if (is_shareable_ipc(host_spec->ipc_mode)) { ++ return set_sharable_ipc_mode(host_spec, v2_spec); ++ } ++ ++ // case 3: Connected container ++ if (namespace_is_container(host_spec->ipc_mode)) { ++ return set_connected_container_shm_path(host_spec, v2_spec); ++ } ++ ++ // case 4: Host IPC namespace ++ if (namespace_is_host(host_spec->ipc_mode)) { ++ return set_host_ipc_shm_path(v2_spec); ++ } ++ ++ // Otherwise, the case is unknown, nothing is set ++ free(v2_spec->shm_path); ++ v2_spec->shm_path = NULL; ++ ++ return 0; + } + + int destination_compare(const void *p1, const void *p2) +-- +2.34.1 + diff --git a/0051-Fix-memory-leak-in-set_connected_container_shm_path.patch b/0051-Fix-memory-leak-in-set_connected_container_shm_path.patch new file mode 100644 index 0000000..c31e092 --- /dev/null +++ b/0051-Fix-memory-leak-in-set_connected_container_shm_path.patch @@ -0,0 +1,26 @@ +From 1052a7b67a35b3504045729e1408d4ace8bf50ca Mon Sep 17 00:00:00 2001 +From: xuxuepeng <xuxuepeng1@huawei.com> +Date: Tue, 9 Apr 2024 06:35:03 +0800 +Subject: [PATCH 51/69] Fix memory leak in set_connected_container_shm_path + +Signed-off-by: xuxuepeng <xuxuepeng1@huawei.com> +--- + src/daemon/modules/spec/specs_mount.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/daemon/modules/spec/specs_mount.c b/src/daemon/modules/spec/specs_mount.c +index 6903ae40..50ee9a85 100644 +--- a/src/daemon/modules/spec/specs_mount.c ++++ b/src/daemon/modules/spec/specs_mount.c +@@ -2822,7 +2822,7 @@ static inline int set_sharable_ipc_mode(host_config *host_spec, container_config + static inline int set_connected_container_shm_path(host_config *host_spec, container_config_v2_common_config *v2_spec) + { + container_t *cont = NULL; +- char *tmp_cid = NULL; ++ __isula_auto_free char *tmp_cid = NULL; + char *right_path = NULL; + + #ifdef ENABLE_CRI_API_V1 +-- +2.34.1 + diff --git a/0052-init-enable_pod_events-as-false.patch b/0052-init-enable_pod_events-as-false.patch new file mode 100644 index 0000000..08cfa2f --- /dev/null +++ b/0052-init-enable_pod_events-as-false.patch @@ -0,0 +1,62 @@ +From cfbb9f5ea40b3b654d7b6f9ad861877e97ed24be Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Thu, 11 Apr 2024 02:04:47 +0000 +Subject: [PATCH 52/69] init enable_pod_events as false + +Signed-off-by: jikai <jikai11@huawei.com> +--- + src/daemon/entry/connect/grpc/cri/cri_service.cc | 3 +-- + src/daemon/entry/connect/grpc/cri/cri_service.h | 1 - + .../connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc | 2 +- + 3 files changed, 2 insertions(+), 4 deletions(-) + +diff --git a/src/daemon/entry/connect/grpc/cri/cri_service.cc b/src/daemon/entry/connect/grpc/cri/cri_service.cc +index d10a60b5..80bcfef0 100644 +--- a/src/daemon/entry/connect/grpc/cri/cri_service.cc ++++ b/src/daemon/entry/connect/grpc/cri/cri_service.cc +@@ -89,9 +89,8 @@ int CRIService::Init(const isulad_daemon_configs *config) + + #ifdef ENABLE_CRI_API_V1 + m_enableCRIV1 = config->enable_cri_v1; +- m_enablePodEvents = config->enable_pod_events; + if (m_enableCRIV1) { +- m_runtimeV1RuntimeService.Init(m_podSandboxImage, m_pluginManager, m_enablePodEvents, err); ++ m_runtimeV1RuntimeService.Init(m_podSandboxImage, m_pluginManager, config->enable_pod_events, err); + if (err.NotEmpty()) { + ERROR("Init CRI v1 runtime service failed: %s", err.GetCMessage()); + return -1; +diff --git a/src/daemon/entry/connect/grpc/cri/cri_service.h b/src/daemon/entry/connect/grpc/cri/cri_service.h +index 041c7c63..77b2eb72 100644 +--- a/src/daemon/entry/connect/grpc/cri/cri_service.h ++++ b/src/daemon/entry/connect/grpc/cri/cri_service.h +@@ -56,7 +56,6 @@ private: + std::string m_podSandboxImage; + std::shared_ptr<Network::PluginManager> m_pluginManager; + bool m_enableCRIV1; +- bool m_enablePodEvents; + }; + + } +diff --git a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc +index bc5ab591..e2591ce0 100644 +--- a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc ++++ b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc +@@ -62,6 +62,7 @@ void RuntimeV1RuntimeServiceImpl::Init(std::string &podSandboxImage, + return; + } + ++ m_enablePodEvents = false; + if (enablePodEvents) { + if (mailbox_register_topic_handler(MAILBOX_TOPIC_CRI_CONTAINER, cri_container_topic_handler, + this, cri_container_topic_release, true) != 0) { +@@ -72,7 +73,6 @@ void RuntimeV1RuntimeServiceImpl::Init(std::string &podSandboxImage, + m_enablePodEvents = enablePodEvents; + } + +- + m_rService = std::unique_ptr<CRIV1::CRIRuntimeService>(new CRIRuntimeServiceImpl(podSandboxImage, cb, networkPlugin, m_enablePodEvents)); + } + +-- +2.34.1 + diff --git a/0053-remove-container-root-path-in-rt_lcr_rm-if-lcr-runti.patch b/0053-remove-container-root-path-in-rt_lcr_rm-if-lcr-runti.patch new file mode 100644 index 0000000..1d9b3a3 --- /dev/null +++ b/0053-remove-container-root-path-in-rt_lcr_rm-if-lcr-runti.patch @@ -0,0 +1,33 @@ +From 0cb96d50302f9f3ad1c17e0bb650ac37db4d5206 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Wed, 10 Apr 2024 08:41:46 +0000 +Subject: [PATCH 53/69] remove container root path in rt_lcr_rm if lcr runtime + missing + +Signed-off-by: jikai <jikai11@huawei.com> +--- + src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c | 8 +++++++- + 1 file changed, 7 insertions(+), 1 deletion(-) + +diff --git a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c +index 6b862958..978da079 100644 +--- a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c ++++ b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c +@@ -238,7 +238,13 @@ int rt_lcr_rm(const char *name, const char *runtime, const rt_rm_params_t *param + + engine_ops = engines_get_handler(runtime); + if (engine_ops == NULL || engine_ops->engine_delete_op == NULL) { +- ERROR("Failed to get engine delete operations"); ++ // if engine_ops is NULL, container root path may have been corrupted, try to remove by daemon ++ // If user runs container with lcr but remove lcr runtime after, there might be resources remaining ++ ERROR("Failed to get engine delete operations, container %s root path may have been corrupted, try to remove by daemon", name); ++ if (remove_container_rootpath(name, params->rootpath) == 0) { ++ ret = 0; ++ goto out; ++ } + ret = -1; + goto out; + } +-- +2.34.1 + diff --git a/0054-ensure-sandbox-can-be-removed-if-sandbox-container-r.patch b/0054-ensure-sandbox-can-be-removed-if-sandbox-container-r.patch new file mode 100644 index 0000000..4fefbe1 --- /dev/null +++ b/0054-ensure-sandbox-can-be-removed-if-sandbox-container-r.patch @@ -0,0 +1,58 @@ +From 21afb41e02886df0b5251889cc443f28b7da274f Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Thu, 11 Apr 2024 01:21:34 +0000 +Subject: [PATCH 54/69] ensure sandbox can be removed if sandbox container + removed + +Signed-off-by: jikai <jikai11@huawei.com> +--- + .../sandbox/controller/shim/shim_controller.cc | 16 ++++++++++++---- + src/daemon/sandbox/sandbox.cc | 3 ++- + 2 files changed, 14 insertions(+), 5 deletions(-) + +diff --git a/src/daemon/sandbox/controller/shim/shim_controller.cc b/src/daemon/sandbox/controller/shim/shim_controller.cc +index 593fade9..4da637c7 100644 +--- a/src/daemon/sandbox/controller/shim/shim_controller.cc ++++ b/src/daemon/sandbox/controller/shim/shim_controller.cc +@@ -517,12 +517,20 @@ bool ShimController::Shutdown(const std::string &sandboxId, Errors &error) + container_delete_response *response {nullptr}; + int ret = m_cb->container.remove(request, &response); + auto responseWrapper = makeUniquePtrCStructWrapper<container_delete_response>(response, free_container_delete_response); ++ if (ret == 0) { ++ return true; ++ } + +- if (ret != 0) { +- std::string msg = (response != nullptr && response->errmsg != nullptr) ? response->errmsg : "internal"; +- ERROR("Failed to remove sandbox %s: %s", sandboxId.c_str(), msg.c_str()); +- error.SetError(msg); ++ std::string errMsg = "internal"; ++ if (response != nullptr && response->errmsg != nullptr) { ++ if (strstr(response->errmsg, "No such container") != nullptr) { ++ ERROR("Container for sandbox %s not found", sandboxId.c_str()); ++ return true; ++ } ++ errMsg = response->errmsg; + } ++ ERROR("Failed to remove sandbox %s: %s", sandboxId.c_str(), errMsg.c_str()); ++ error.SetError(errMsg); + return error.Empty(); + } + +diff --git a/src/daemon/sandbox/sandbox.cc b/src/daemon/sandbox/sandbox.cc +index c70116c1..bae5b8db 100644 +--- a/src/daemon/sandbox/sandbox.cc ++++ b/src/daemon/sandbox/sandbox.cc +@@ -757,7 +757,8 @@ auto Sandbox::Remove(Errors &error) -> bool + + WriteGuard<RWMutex> lock(m_mutex); + +- if (!DoStop(DEFAULT_STOP_TIMEOUT, error)) { ++ // Only stop the sandbox when it is running ++ if (IsReady() && !DoStop(DEFAULT_STOP_TIMEOUT, error)) { + ERROR("Failed to stop Sandbox before removing, id='%s'", m_id.c_str()); + return false; + } +-- +2.34.1 + diff --git a/0055-bugfix-for-shim-timeout-exit-error-log-changes.patch b/0055-bugfix-for-shim-timeout-exit-error-log-changes.patch new file mode 100644 index 0000000..7009560 --- /dev/null +++ b/0055-bugfix-for-shim-timeout-exit-error-log-changes.patch @@ -0,0 +1,175 @@ +From 35ffb77f568124e6e7c8fd7b3d021878b92c13f7 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Tue, 9 Apr 2024 20:04:33 +0800 +Subject: [PATCH 55/69] bugfix for shim timeout exit error log changes + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + .../modules/runtime/isula/isula_rt_ops.c | 55 ++++++++++++------- + 1 file changed, 34 insertions(+), 21 deletions(-) + +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index bc3c36c8..1875cf5b 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -861,6 +861,8 @@ static int shim_create(shim_create_args *args) + pid_t pid = 0; + int shim_stderr_pipe[2] = { -1, -1 }; + int shim_stdout_pipe[2] = { -1, -1 }; ++ // used to accept exec error msg ++ int exec_err_pipe[2] = {-1, -1}; + int num = 0; + int ret = 0; + char exec_buff[BUFSIZ + 1] = { 0 }; +@@ -904,6 +906,11 @@ static int shim_create(shim_create_args *args) + return -1; + } + ++ if (pipe2(exec_err_pipe, O_CLOEXEC) != 0) { ++ ERROR("Failed to create pipe for exec err"); ++ return -1; ++ } ++ + pid = fork(); + if (pid < 0) { + SYSERROR("Failed fork for shim parent"); +@@ -911,30 +918,32 @@ static int shim_create(shim_create_args *args) + close(shim_stderr_pipe[1]); + close(shim_stdout_pipe[0]); + close(shim_stdout_pipe[1]); ++ close(exec_err_pipe[0]); ++ close(exec_err_pipe[1]); + return -1; + } + + if (pid == (pid_t)0) { + if (chdir(args->workdir) < 0) { +- (void)dprintf(shim_stderr_pipe[1], "%s: failed chdir to %s", args->id, args->workdir); ++ (void)dprintf(exec_err_pipe[1], "%s: failed chdir to %s", args->id, args->workdir); + exit(EXIT_FAILURE); + } + + //prevent the child process from having the same standard streams as the parent process + if (isula_null_stdfds() != 0) { +- (void)dprintf(shim_stderr_pipe[1], "failed to set std console to /dev/null"); ++ (void)dprintf(exec_err_pipe[1], "failed to set std console to /dev/null"); + exit(EXIT_FAILURE); + } + + if (args->fg) { + // child process, dup2 shim_stdout_pipe[1] to STDOUT, get container process exit_code in STDOUT + if (dup2(shim_stdout_pipe[1], STDOUT_FILENO) < 0) { +- (void)dprintf(shim_stderr_pipe[1], "Dup stdout fd error: %s", strerror(errno)); ++ (void)dprintf(exec_err_pipe[1], "Dup stdout fd error: %s", strerror(errno)); + exit(EXIT_FAILURE); + } + // child process, dup2 shim_stderr_pipe[1] to STDERR, get isulad-shim errmsg in STDERR + if (dup2(shim_stderr_pipe[1], STDERR_FILENO) < 0) { +- (void)dprintf(shim_stderr_pipe[1], "Dup stderr fd error: %s", strerror(errno)); ++ (void)dprintf(exec_err_pipe[1], "Dup stderr fd error: %s", strerror(errno)); + exit(EXIT_FAILURE); + } + goto realexec; +@@ -942,18 +951,18 @@ static int shim_create(shim_create_args *args) + + // clear NOTIFY_SOCKET from the env to adapt runc create + if (unsetenv("NOTIFY_SOCKET") != 0) { +- (void)dprintf(shim_stderr_pipe[1], "%s: unset env NOTIFY_SOCKET failed %s", args->id, strerror(errno)); ++ (void)dprintf(exec_err_pipe[1], "%s: unset env NOTIFY_SOCKET failed %s", args->id, strerror(errno)); + exit(EXIT_FAILURE); + } + + pid = fork(); + if (pid < 0) { +- (void)dprintf(shim_stderr_pipe[1], "%s: fork shim-process failed %s", args->id, strerror(errno)); ++ (void)dprintf(exec_err_pipe[1], "%s: fork shim-process failed %s", args->id, strerror(errno)); + _exit(EXIT_FAILURE); + } + if (pid != 0) { + if (file_write_int(fpid, pid) != 0) { +- (void)dprintf(shim_stderr_pipe[1], "%s: write %s with %d failed", args->id, fpid, pid); ++ (void)dprintf(exec_err_pipe[1], "%s: write %s with %d failed", args->id, fpid, pid); + } + _exit(EXIT_SUCCESS); + } +@@ -962,35 +971,38 @@ realexec: + /* real shim process. */ + close(shim_stderr_pipe[0]); + close(shim_stdout_pipe[0]); ++ close(exec_err_pipe[0]); + + if (setsid() < 0) { +- (void)dprintf(shim_stderr_pipe[1], "%s: failed setsid for process %d", args->id, getpid()); ++ (void)dprintf(exec_err_pipe[1], "%s: failed setsid for process %d", args->id, getpid()); + exit(EXIT_FAILURE); + } + + if (util_check_inherited(true, shim_stderr_pipe[1]) != 0) { +- (void)dprintf(shim_stderr_pipe[1], "close inherited fds failed"); ++ (void)dprintf(exec_err_pipe[1], "close inherited fds failed"); + exit(EXIT_FAILURE); + } + + if (setenv(SHIIM_LOG_PATH_ENV, engine_log_path, 1) != 0) { +- (void)dprintf(shim_stderr_pipe[1], "%s: failed to set SHIIM_LOG_PATH_ENV for process %d", args->id, getpid()); ++ (void)dprintf(exec_err_pipe[1], "%s: failed to set SHIIM_LOG_PATH_ENV for process %d", args->id, getpid()); + exit(EXIT_FAILURE); + } + + if (setenv(SHIIM_LOG_LEVEL_ENV, log_level, 1) != 0) { +- (void)dprintf(shim_stderr_pipe[1], "%s: failed to set SHIIM_LOG_LEVEL_ENV env for process %d", args->id, getpid()); ++ (void)dprintf(exec_err_pipe[1], "%s: failed to set SHIIM_LOG_LEVEL_ENV env for process %d", args->id, getpid()); + exit(EXIT_FAILURE); + } + + execvp(SHIM_BINARY, (char * const *)params); +- (void)dprintf(shim_stderr_pipe[1], "run process: %s failed: %s", SHIM_BINARY, strerror(errno)); ++ (void)dprintf(exec_err_pipe[1], "run process: %s failed: %s", SHIM_BINARY, strerror(errno)); + exit(EXIT_FAILURE); + } + + close(shim_stderr_pipe[1]); + close(shim_stdout_pipe[1]); +- num = util_read_nointr(shim_stderr_pipe[0], exec_buff, sizeof(exec_buff) - 1); ++ close(exec_err_pipe[1]); ++ num = util_read_nointr(exec_err_pipe[0], exec_buff, sizeof(exec_buff) - 1); ++ close(exec_err_pipe[0]); + + status = util_wait_for_pid_status(pid); + if (status < 0) { +@@ -1035,8 +1047,10 @@ realexec: + out: + close(shim_stdout_pipe[0]); + if (ret != 0) { +- show_runtime_errlog(args->workdir); + show_shim_errlog(shim_stderr_pipe[0]); ++ // Since users are more concerned about runtime error information, ++ // the runtime log will overwrite the shim log if it exists. ++ show_runtime_errlog(args->workdir); + if (args->timeout != NULL) { + kill(pid, SIGKILL); /* can kill other process? */ + } +@@ -1491,14 +1505,13 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p + args.exit_code = exit_code; + args.timeout = timeout; + ret = shim_create(&args); +- if (args.shim_exit_code == SHIM_EXIT_TIMEOUT) { +- ret = -1; +- isulad_set_error_message("Exec container error;exec timeout"); +- ERROR("isulad-shim %d exit for execing timeout", pid); +- goto errlog_out; +- } + if (ret != 0) { +- ERROR("%s: failed create shim process for exec %s", id, exec_id); ++ if (args.shim_exit_code == SHIM_EXIT_TIMEOUT) { ++ isulad_set_error_message("Exec container error;exec timeout"); ++ ERROR("isulad-shim %d exit for execing timeout", pid); ++ } else { ++ ERROR("%s: failed create shim process for exec %s", id, exec_id); ++ } + goto errlog_out; + } + +-- +2.34.1 + diff --git a/0056-bugfix-for-the-pre-created-pipe-was-not-closed-when-.patch b/0056-bugfix-for-the-pre-created-pipe-was-not-closed-when-.patch new file mode 100644 index 0000000..607f721 --- /dev/null +++ b/0056-bugfix-for-the-pre-created-pipe-was-not-closed-when-.patch @@ -0,0 +1,36 @@ +From e8ba4368f4be369f99d7da6fc04dcbe173985cd0 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Fri, 12 Apr 2024 14:42:05 +0800 +Subject: [PATCH 56/69] bugfix for the pre-created pipe was not closed when the + pipe creation failed + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/modules/runtime/isula/isula_rt_ops.c | 6 ++++++ + 1 file changed, 6 insertions(+) + +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 1875cf5b..47a14b1d 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -903,11 +903,17 @@ static int shim_create(shim_create_args *args) + + if (pipe2(shim_stdout_pipe, O_CLOEXEC) != 0) { + ERROR("Failed to create pipe for shim stdout"); ++ close(shim_stderr_pipe[0]); ++ close(shim_stderr_pipe[1]); + return -1; + } + + if (pipe2(exec_err_pipe, O_CLOEXEC) != 0) { + ERROR("Failed to create pipe for exec err"); ++ close(shim_stderr_pipe[0]); ++ close(shim_stderr_pipe[1]); ++ close(shim_stdout_pipe[0]); ++ close(shim_stdout_pipe[1]); + return -1; + } + +-- +2.34.1 + diff --git a/0057-add-debug-msg-info-in-image_load.sh.patch b/0057-add-debug-msg-info-in-image_load.sh.patch new file mode 100644 index 0000000..81f61c8 --- /dev/null +++ b/0057-add-debug-msg-info-in-image_load.sh.patch @@ -0,0 +1,26 @@ +From c3f7cf2a54188e5fc890a8d23b95254ac69cfa52 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Fri, 12 Apr 2024 15:23:07 +0800 +Subject: [PATCH 57/69] add debug msg info in image_load.sh + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + CI/test_cases/image_cases/image_load.sh | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/CI/test_cases/image_cases/image_load.sh b/CI/test_cases/image_cases/image_load.sh +index a2cada5f..d50b3203 100755 +--- a/CI/test_cases/image_cases/image_load.sh ++++ b/CI/test_cases/image_cases/image_load.sh +@@ -103,6 +103,8 @@ function test_concurrent_load() + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - fail to do isulad load $i" && ((ret++)) + done + ++ tail -n 50 /var/lib/isulad/isulad.log ++ + ubuntu_id=`isula inspect -f '{{.image.id}}' ubuntu` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - fail to inspect image: ubuntu" && ((ret++)) + +-- +2.34.1 + diff --git a/0058-empty-pointer-check-in-lcr_rt_ops.patch b/0058-empty-pointer-check-in-lcr_rt_ops.patch new file mode 100644 index 0000000..1cbe4b2 --- /dev/null +++ b/0058-empty-pointer-check-in-lcr_rt_ops.patch @@ -0,0 +1,214 @@ +From ea1bc00c894a3717ea375f5ff40c3eb05447ae17 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Sat, 13 Apr 2024 14:07:33 +0000 +Subject: [PATCH 58/69] empty pointer check in lcr_rt_ops + +Signed-off-by: jikai <jikai11@huawei.com> +--- + .../modules/runtime/engines/lcr/lcr_rt_ops.c | 84 ++++++++++++++++++- + 1 file changed, 81 insertions(+), 3 deletions(-) + +diff --git a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c +index 978da079..a89d0375 100644 +--- a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c ++++ b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c +@@ -53,6 +53,11 @@ int rt_lcr_create(const char *name, const char *runtime, const rt_create_params_ + char *runtime_root = NULL; + struct engine_operation *engine_ops = NULL; + ++ if (name == NULL || runtime == NULL || params == NULL) { ++ ERROR("Nullptr arguments not allowed"); ++ return -1; ++ } ++ + if (conf_get_systemd_cgroup()) { + ERROR("Systemd cgroup not supported for lcr runtime"); + isulad_set_error_message("Systemd cgroup not supported for lcr runtime"); +@@ -129,6 +134,11 @@ int rt_lcr_start(const char *name, const char *runtime, const rt_start_params_t + struct engine_operation *engine_ops = NULL; + engine_start_request_t request = { 0 }; + ++ if (name == NULL || runtime == NULL || params == NULL || pid_info == NULL) { ++ ERROR("Nullptr arguments not allowed"); ++ return -1; ++ } ++ + engine_ops = engines_get_handler(runtime); + if (engine_ops == NULL || engine_ops->engine_start_op == NULL) { + ERROR("Failed to get engine start operations"); +@@ -183,6 +193,11 @@ int rt_lcr_clean_resource(const char *name, const char *runtime, const rt_clean_ + int ret = 0; + struct engine_operation *engine_ops = NULL; + ++ if (name == NULL || runtime == NULL || params == NULL) { ++ ERROR("Nullptr arguments not allowed"); ++ return -1; ++ } ++ + engine_ops = engines_get_handler(runtime); + if (engine_ops == NULL || engine_ops->engine_clean_op == NULL) { + ERROR("Failed to get engine clean operations"); +@@ -236,6 +251,15 @@ int rt_lcr_rm(const char *name, const char *runtime, const rt_rm_params_t *param + int ret = 0; + struct engine_operation *engine_ops = NULL; + ++ if (name == NULL || runtime == NULL || params == NULL) { ++ ERROR("Nullptr arguments not allowed"); ++ return -1; ++ } ++ if (params->rootpath == NULL) { ++ ERROR("Missing root path"); ++ return -1; ++ } ++ + engine_ops = engines_get_handler(runtime); + if (engine_ops == NULL || engine_ops->engine_delete_op == NULL) { + // if engine_ops is NULL, container root path may have been corrupted, try to remove by daemon +@@ -284,6 +308,11 @@ int rt_lcr_status(const char *name, const char *runtime, const rt_status_params_ + int nret = 0; + struct engine_operation *engine_ops = NULL; + ++ if (name == NULL || runtime == NULL || params == NULL || status == NULL) { ++ ERROR("Nullptr arguments not allowed"); ++ return -1; ++ } ++ + engine_ops = engines_get_handler(runtime); + if (engine_ops == NULL || engine_ops->engine_get_container_status_op == NULL) { + ERROR("Failed to get engine status operations"); +@@ -322,6 +351,11 @@ int rt_lcr_resources_stats(const char *name, const char *runtime, const rt_stats + int nret = 0; + struct engine_operation *engine_ops = NULL; + ++ if (name == NULL || runtime == NULL || params == NULL || rs_stats == NULL) { ++ ERROR("Nullptr arguments not allowed"); ++ return -1; ++ } ++ + engine_ops = engines_get_handler(runtime); + if (engine_ops == NULL || engine_ops->engine_get_container_resources_stats_op == NULL) { + ERROR("Failed to get engine stats operations"); +@@ -451,6 +485,11 @@ int rt_lcr_exec(const char *id, const char *runtime, const rt_exec_params_t *par + char *user = NULL; + char *add_gids = NULL; + ++ if (id == NULL || runtime == NULL || params == NULL || exit_code == NULL) { ++ ERROR("Nullptr arguments not allowed"); ++ return -1; ++ } ++ + engine_ops = engines_get_handler(runtime); + if (engine_ops == NULL || engine_ops->engine_exec_op == NULL) { + DEBUG("Failed to get engine exec operations"); +@@ -519,6 +558,11 @@ int rt_lcr_pause(const char *name, const char *runtime, const rt_pause_params_t + int ret = 0; + struct engine_operation *engine_ops = NULL; + ++ if (name == NULL || runtime == NULL || params == NULL) { ++ ERROR("Nullptr arguments not allowed"); ++ return -1; ++ } ++ + engine_ops = engines_get_handler(runtime); + if (engine_ops == NULL || engine_ops->engine_pause_op == NULL) { + DEBUG("Failed to get engine pause operations"); +@@ -549,6 +593,11 @@ int rt_lcr_resume(const char *name, const char *runtime, const rt_resume_params_ + int ret = 0; + struct engine_operation *engine_ops = NULL; + ++ if (name == NULL || runtime == NULL || params == NULL) { ++ ERROR("Nullptr arguments not allowed"); ++ return -1; ++ } ++ + engine_ops = engines_get_handler(runtime); + if (engine_ops == NULL || engine_ops->engine_resume_op == NULL) { + DEBUG("Failed to get engine resume operations"); +@@ -579,6 +628,11 @@ int rt_lcr_attach(const char *name, const char *runtime, const rt_attach_params_ + int ret = 0; + struct engine_operation *engine_ops = NULL; + ++ if (name == NULL || runtime == NULL || params == NULL) { ++ ERROR("Null argument"); ++ return -1; ++ } ++ + engine_ops = engines_get_handler(runtime); + if (engine_ops == NULL || engine_ops->engine_console_op == NULL) { + DEBUG("Failed to get engine attach operations"); +@@ -641,6 +695,11 @@ int rt_lcr_update(const char *id, const char *runtime, const rt_update_params_t + struct engine_operation *engine_ops = NULL; + struct engine_cgroup_resources cr = { 0 }; + ++ if (id == NULL || runtime == NULL || params == NULL) { ++ ERROR("Nullptr arguments not allowed"); ++ return -1; ++ } ++ + engine_ops = engines_get_handler(runtime); + if (engine_ops == NULL || engine_ops->engine_update_op == NULL) { + DEBUG("Failed to get engine update operations"); +@@ -673,10 +732,9 @@ int rt_lcr_listpids(const char *name, const char *runtime, const rt_listpids_par + int ret = 0; + struct engine_operation *engine_ops = NULL; + +- if (out == NULL) { ++ if (name == NULL || runtime == NULL || params == NULL || out == NULL) { + ERROR("Invalid arguments"); +- ret = -1; +- goto out; ++ return -1; + } + + engine_ops = engines_get_handler(runtime); +@@ -709,6 +767,11 @@ int rt_lcr_resize(const char *id, const char *runtime, const rt_resize_params_t + int ret = 0; + struct engine_operation *engine_ops = NULL; + ++ if (id == NULL || runtime == NULL || params == NULL) { ++ ERROR("Invalid arguments"); ++ return -1; ++ } ++ + engine_ops = engines_get_handler(runtime); + if (engine_ops == NULL || engine_ops->engine_resize_op == NULL) { + DEBUG("Failed to get engine resume operations"); +@@ -740,6 +803,11 @@ int rt_lcr_exec_resize(const char *id, const char *runtime, const rt_exec_resize + int ret = 0; + struct engine_operation *engine_ops = NULL; + ++ if (id == NULL || runtime == NULL || params == NULL) { ++ ERROR("Nullptr arguments not allowed"); ++ return -1; ++ } ++ + engine_ops = engines_get_handler(runtime); + if (engine_ops == NULL || engine_ops->engine_resize_op == NULL) { + DEBUG("Failed to get engine resume operations"); +@@ -767,6 +835,11 @@ out: + + int rt_lcr_kill(const char *id, const char *runtime, const rt_kill_params_t *params) + { ++ if (id == NULL || runtime == NULL || params == NULL || params->pid < 0) { ++ ERROR("Invalid arguments not allowed"); ++ return -1; ++ } ++ + if (util_process_alive(params->pid, params->start_time) == false) { + if (params->signal == params->stop_signal || params->signal == SIGKILL) { + WARN("Process %d is not alive", params->pid); +@@ -798,6 +871,11 @@ int rt_lcr_rebuild_config(const char *name, const char *runtime, const rt_rebuil + oci_runtime_spec *oci_spec = NULL; + __isula_auto_free parser_error err = NULL; + ++ if (name == NULL || runtime == NULL || params == NULL) { ++ ERROR("Invalid arguments not allowed"); ++ return -1; ++ } ++ + engine_ops = engines_get_handler(runtime); + if (engine_ops == NULL || engine_ops->engine_create_op == NULL) { + ERROR("Failed to get engine rebuild config operations"); +-- +2.34.1 + diff --git a/0059-modify-some-grpc-status-codes-of-cri-in-case-of-erro.patch b/0059-modify-some-grpc-status-codes-of-cri-in-case-of-erro.patch new file mode 100644 index 0000000..1153418 --- /dev/null +++ b/0059-modify-some-grpc-status-codes-of-cri-in-case-of-erro.patch @@ -0,0 +1,315 @@ +From 628f4ceb329e16991ed33d3a460bcf8f5542ba99 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Mon, 8 Apr 2024 09:30:04 +0000 +Subject: [PATCH 59/69] modify some grpc status codes of cri in case of error + +Signed-off-by: jikai <jikai11@huawei.com> +--- + .../cri/v1/cri_v1_runtime_runtime_service.cc | 41 +++++++++++++------ + .../cri/v1/cri_v1_runtime_runtime_service.h | 3 ++ + .../v1alpha/cri_runtime_runtime_service.cc | 41 +++++++++++++------ + .../cri/v1alpha/cri_runtime_runtime_service.h | 2 + + 4 files changed, 63 insertions(+), 24 deletions(-) + +diff --git a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc +index e2591ce0..fb5aad3c 100644 +--- a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc ++++ b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc +@@ -50,6 +50,23 @@ static void cri_container_topic_release(void *arg) + delete resp; + } + ++grpc::Status RuntimeV1RuntimeServiceImpl::ToGRPCStatus(Errors &error) ++{ ++ if (error.Empty()) { ++ return grpc::Status::OK; ++ } ++ if (error.GetMessage().find("Failed to find") != std::string::npos) { ++ return grpc::Status(grpc::StatusCode::NOT_FOUND, error.GetMessage()); ++ } ++ ++ // Attach exceeded timeout for lxc and Exec container error;exec timeout for runc ++ if (error.GetMessage().find("Attach exceeded timeout") != std::string::npos ++ || error.GetMessage().find("Exec container error;exec timeout") != std::string::npos) { ++ return grpc::Status(grpc::StatusCode::DEADLINE_EXCEEDED, error.GetMessage()); ++ } ++ return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++} ++ + void RuntimeV1RuntimeServiceImpl::Init(std::string &podSandboxImage, + std::shared_ptr<Network::PluginManager> networkPlugin, + bool enablePodEvents, Errors &err) +@@ -167,7 +184,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::CreateContainer(grpc::ServerContext *c + m_rService->CreateContainer(request->pod_sandbox_id(), request->config(), request->sandbox_config(), error); + if (!error.Empty() || responseID.empty()) { + ERROR("Object: CRI, Type: Failed to create container"); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + reply->set_container_id(responseID); + +@@ -192,7 +209,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::StartContainer(grpc::ServerContext *co + m_rService->StartContainer(request->container_id(), error); + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to start container %s", request->container_id().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + EVENT("Event: {Object: CRI, Type: Started Container: %s}", request->container_id().c_str()); +@@ -216,7 +233,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::StopContainer(grpc::ServerContext *con + m_rService->StopContainer(request->container_id(), (int64_t)request->timeout(), error); + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to stop container %s", request->container_id().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + EVENT("Event: {Object: CRI, Type: Stopped Container: %s}", request->container_id().c_str()); +@@ -240,7 +257,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::RemoveContainer(grpc::ServerContext *c + m_rService->RemoveContainer(request->container_id(), error); + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to remove container %s", request->container_id().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + EVENT("Event: {Object: CRI, Type: Removed Container: %s}", request->container_id().c_str()); +@@ -359,7 +376,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::ContainerStatus(grpc::ServerContext *c + m_rService->ContainerStatus(request->container_id(), error); + if (!error.Empty() || !contStatus) { + ERROR("Object: CRI, Type: Failed to get container status %s", request->container_id().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + *(reply->mutable_status()) = *contStatus; + +@@ -384,7 +401,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::ExecSync(grpc::ServerContext *context, + m_rService->ExecSync(request->container_id(), request->cmd(), request->timeout(), reply, error); + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to sync exec container: %s", request->container_id().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + WARN("Event: {Object: CRI, Type: sync execed Container: %s}", request->container_id().c_str()); +@@ -437,7 +454,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::StopPodSandbox(grpc::ServerContext *co + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to stop pod:%s due to %s", request->pod_sandbox_id().c_str(), + error.GetMessage().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + EVENT("Event: {Object: CRI, Type: Stopped Pod: %s}", request->pod_sandbox_id().c_str()); +@@ -462,7 +479,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::RemovePodSandbox(grpc::ServerContext * + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to remove pod:%s due to %s", request->pod_sandbox_id().c_str(), + error.GetMessage().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + EVENT("Event: {Object: CRI, Type: Removed Pod: %s}", request->pod_sandbox_id().c_str()); +@@ -487,7 +504,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::PodSandboxStatus(grpc::ServerContext * + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to status pod:%s due to %s", request->pod_sandbox_id().c_str(), + error.GetMessage().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + INFO("Event: {Object: CRI, Type: Statused Pod: %s}", request->pod_sandbox_id().c_str()); +@@ -608,7 +625,7 @@ RuntimeV1RuntimeServiceImpl::UpdateContainerResources(grpc::ServerContext *conte + if (error.NotEmpty()) { + ERROR("Object: CRI, Type: Failed to update container:%s due to %s", request->container_id().c_str(), + error.GetMessage().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + WARN("Event: {Object: CRI, Type: Updated container resources: %s}", request->container_id().c_str()); +@@ -633,7 +650,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::Exec(grpc::ServerContext *context, + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to exec container:%s due to %s", request->container_id().c_str(), + error.GetMessage().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + EVENT("Event: {Object: CRI, Type: execed Container: %s}", request->container_id().c_str()); +@@ -658,7 +675,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::Attach(grpc::ServerContext *context, + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to attach container:%s due to %s", request->container_id().c_str(), + error.GetMessage().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + EVENT("Event: {Object: CRI, Type: attched Container: %s}", request->container_id().c_str()); +diff --git a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h +index 842d1811..1cf375a4 100644 +--- a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h ++++ b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h +@@ -114,6 +114,9 @@ public: + grpc::ServerWriter<runtime::v1::ContainerEventResponse> *writer) override; + + private: ++ ++ grpc::Status ToGRPCStatus(Errors &error); ++ + std::unique_ptr<CRIV1::CRIRuntimeService> m_rService; + bool m_enablePodEvents; + }; +diff --git a/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.cc b/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.cc +index 5e85702c..1c83f4ca 100644 +--- a/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.cc ++++ b/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.cc +@@ -23,6 +23,23 @@ + + using namespace CRI; + ++grpc::Status RuntimeRuntimeServiceImpl::ToGRPCStatus(Errors &error) ++{ ++ if (error.Empty()) { ++ return grpc::Status::OK; ++ } ++ if (error.GetMessage().find("Failed to find") != std::string::npos) { ++ return grpc::Status(grpc::StatusCode::NOT_FOUND, error.GetMessage()); ++ } ++ ++ // Attach exceeded timeout for lxc and Exec container error;exec timeout for runc ++ if (error.GetMessage().find("Attach exceeded timeout") != std::string::npos ++ || error.GetMessage().find("Exec container error;exec timeout") != std::string::npos) { ++ return grpc::Status(grpc::StatusCode::DEADLINE_EXCEEDED, error.GetMessage()); ++ } ++ return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++} ++ + void RuntimeRuntimeServiceImpl::Init(std::string &podSandboxImage, + std::shared_ptr<Network::PluginManager> networkPlugin, Errors &err) + { +@@ -80,7 +97,7 @@ grpc::Status RuntimeRuntimeServiceImpl::CreateContainer(grpc::ServerContext *con + m_rService->CreateContainer(request->pod_sandbox_id(), request->config(), request->sandbox_config(), error); + if (!error.Empty() || responseID.empty()) { + ERROR("Object: CRI, Type: Failed to create container"); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + reply->set_container_id(responseID); + +@@ -105,7 +122,7 @@ grpc::Status RuntimeRuntimeServiceImpl::StartContainer(grpc::ServerContext *cont + m_rService->StartContainer(request->container_id(), error); + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to start container %s", request->container_id().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + EVENT("Event: {Object: CRI, Type: Started Container: %s}", request->container_id().c_str()); +@@ -129,7 +146,7 @@ grpc::Status RuntimeRuntimeServiceImpl::StopContainer(grpc::ServerContext *conte + m_rService->StopContainer(request->container_id(), (int64_t)request->timeout(), error); + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to stop container %s", request->container_id().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + EVENT("Event: {Object: CRI, Type: Stopped Container: %s}", request->container_id().c_str()); +@@ -153,7 +170,7 @@ grpc::Status RuntimeRuntimeServiceImpl::RemoveContainer(grpc::ServerContext *con + m_rService->RemoveContainer(request->container_id(), error); + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to remove container %s", request->container_id().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + EVENT("Event: {Object: CRI, Type: Removed Container: %s}", request->container_id().c_str()); +@@ -272,7 +289,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ContainerStatus(grpc::ServerContext *con + m_rService->ContainerStatus(request->container_id(), error); + if (!error.Empty() || !contStatus) { + ERROR("Object: CRI, Type: Failed to get container status %s", request->container_id().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + *(reply->mutable_status()) = *contStatus; + +@@ -297,7 +314,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ExecSync(grpc::ServerContext *context, + m_rService->ExecSync(request->container_id(), request->cmd(), request->timeout(), reply, error); + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to sync exec container: %s", request->container_id().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + WARN("Event: {Object: CRI, Type: sync execed Container: %s}", request->container_id().c_str()); +@@ -351,7 +368,7 @@ grpc::Status RuntimeRuntimeServiceImpl::StopPodSandbox(grpc::ServerContext *cont + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to stop pod:%s due to %s", request->pod_sandbox_id().c_str(), + error.GetMessage().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + EVENT("Event: {Object: CRI, Type: Stopped Pod: %s}", request->pod_sandbox_id().c_str()); +@@ -376,7 +393,7 @@ grpc::Status RuntimeRuntimeServiceImpl::RemovePodSandbox(grpc::ServerContext *co + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to remove pod:%s due to %s", request->pod_sandbox_id().c_str(), + error.GetMessage().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + EVENT("Event: {Object: CRI, Type: Removed Pod: %s}", request->pod_sandbox_id().c_str()); +@@ -402,7 +419,7 @@ grpc::Status RuntimeRuntimeServiceImpl::PodSandboxStatus(grpc::ServerContext *co + if (!error.Empty() || !podStatus) { + ERROR("Object: CRI, Type: Failed to status pod:%s due to %s", request->pod_sandbox_id().c_str(), + error.GetMessage().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + *(reply->mutable_status()) = *podStatus; + +@@ -523,7 +540,7 @@ RuntimeRuntimeServiceImpl::UpdateContainerResources(grpc::ServerContext *context + if (error.NotEmpty()) { + ERROR("Object: CRI, Type: Failed to update container:%s due to %s", request->container_id().c_str(), + error.GetMessage().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + WARN("Event: {Object: CRI, Type: Updated container resources: %s}", request->container_id().c_str()); +@@ -548,7 +565,7 @@ grpc::Status RuntimeRuntimeServiceImpl::Exec(grpc::ServerContext *context, + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to exec container:%s due to %s", request->container_id().c_str(), + error.GetMessage().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + EVENT("Event: {Object: CRI, Type: execed Container: %s}", request->container_id().c_str()); +@@ -573,7 +590,7 @@ grpc::Status RuntimeRuntimeServiceImpl::Attach(grpc::ServerContext *context, + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to attach container:%s due to %s", request->container_id().c_str(), + error.GetMessage().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + EVENT("Event: {Object: CRI, Type: attched Container: %s}", request->container_id().c_str()); +diff --git a/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.h b/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.h +index e0f75897..210e67cc 100644 +--- a/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.h ++++ b/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.h +@@ -103,6 +103,8 @@ public: + runtime::v1alpha2::StatusResponse *reply) override; + + private: ++ grpc::Status ToGRPCStatus(Errors &err); ++ + std::unique_ptr<CRI::CRIRuntimeService> m_rService; + }; + +-- +2.34.1 + diff --git a/0060-cdi-return-int-instead-of-error-string.patch b/0060-cdi-return-int-instead-of-error-string.patch new file mode 100644 index 0000000..e2ebc9b --- /dev/null +++ b/0060-cdi-return-int-instead-of-error-string.patch @@ -0,0 +1,363 @@ +From e7b94411b174c8445d9bdc84ec6c94b5d4343470 Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Mon, 15 Apr 2024 15:38:57 +0800 +Subject: [PATCH 60/69] cdi:return int instead of error string + +--- + src/daemon/modules/api/cdi_operate_api.h | 7 ++++--- + .../device/cdi/behavior/cdi_container_edits.c | 12 ++++++------ + .../device/cdi/behavior/cdi_container_edits.h | 6 +++--- + .../modules/device/cdi/behavior/cdi_device.c | 2 +- + .../modules/device/cdi/behavior/cdi_device.h | 2 +- + .../modules/device/cdi/behavior/cdi_spec.c | 2 +- + .../modules/device/cdi/behavior/cdi_spec.h | 2 +- + .../modules/device/cdi/behavior/cdi_spec_dirs.c | 4 ++-- + .../modules/device/cdi/behavior/cdi_spec_dirs.h | 6 +++--- + .../device/cdi/behavior/parser/cdi_parser.c | 16 ++++++++-------- + .../device/cdi/behavior/parser/cdi_parser.h | 8 ++++---- + src/daemon/modules/device/cdi/cdi_annotations.c | 5 +++-- + src/daemon/modules/device/cdi/cdi_annotations.h | 3 ++- + src/daemon/modules/device/cdi/cdi_cache.c | 14 +++++++------- + src/daemon/modules/device/cdi/cdi_cache.h | 8 ++++---- + src/daemon/modules/device/cdi_operate.c | 13 +++++++------ + 16 files changed, 57 insertions(+), 53 deletions(-) + +diff --git a/src/daemon/modules/api/cdi_operate_api.h b/src/daemon/modules/api/cdi_operate_api.h +index 4f4c339e..49820ed7 100644 +--- a/src/daemon/modules/api/cdi_operate_api.h ++++ b/src/daemon/modules/api/cdi_operate_api.h +@@ -26,11 +26,12 @@ extern "C" { + + int cdi_operate_registry_init(char **specs_dirs, size_t specs_dirs_len); + +-char *cdi_operate_refresh(void); ++int cdi_operate_refresh(void); + +-string_array *cdi_operate_inject_devices(oci_runtime_spec *spec, string_array *devices, char **error); ++int cdi_operate_inject_devices(oci_runtime_spec *spec, string_array *devices); + +-char *cdi_operate_parse_annotations(json_map_string_string *annotations, string_array **keys, string_array **devices); ++int cdi_operate_parse_annotations(json_map_string_string *annotations, string_array **keys, ++ string_array **devices, char **error); + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c +index ce7b16db..590118b1 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c +@@ -27,19 +27,19 @@ + // POSTSTOP_HOOK is the name of the OCI "poststop" hook. + #define POSTSTOP_HOOK "poststop" + +-char *cdi_container_edits_apply(cdi_container_edits *e, oci_runtime_spec *spec) ++int cdi_container_edits_apply(cdi_container_edits *e, oci_runtime_spec *spec) + { +- return NULL; ++ return 0; + } + +-char *cdi_container_edits_validate(cdi_container_edits *e) ++int cdi_container_edits_validate(cdi_container_edits *e, char **error) + { +- return NULL; ++ return 0; + } + +-cdi_container_edits *cdi_container_edits_append(cdi_container_edits *e, cdi_container_edits *o) ++int cdi_container_edits_append(cdi_container_edits *e, cdi_container_edits *o) + { +- return NULL; ++ return 0; + } + + bool cdi_container_edits_is_empty(cdi_container_edits *e) +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_container_edits.h b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.h +index 7b16d2bc..ea921e37 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_container_edits.h ++++ b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.h +@@ -27,9 +27,9 @@ + extern "C" { + #endif + +-char *cdi_container_edits_apply(cdi_container_edits *e, oci_runtime_spec *spec); +-char *cdi_container_edits_validate(cdi_container_edits *e); +-cdi_container_edits *cdi_container_edits_append(cdi_container_edits *e, cdi_container_edits *o); ++int cdi_container_edits_apply(cdi_container_edits *e, oci_runtime_spec *spec); ++int cdi_container_edits_validate(cdi_container_edits *e, char **error); ++int cdi_container_edits_append(cdi_container_edits *e, cdi_container_edits *o); + bool cdi_container_edits_is_empty(cdi_container_edits *e); + + #ifdef __cplusplus +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_device.c b/src/daemon/modules/device/cdi/behavior/cdi_device.c +index 9904e9ee..0fef8f42 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_device.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_device.c +@@ -34,7 +34,7 @@ char *cdi_device_get_qualified_name(struct cdi_cache_device *d) + return NULL; + } + +-cdi_container_edits *cdi_device_edits(struct cdi_cache_device *d) ++cdi_container_edits *cdi_device_get_edits(struct cdi_cache_device *d) + { + return NULL; + } +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_device.h b/src/daemon/modules/device/cdi/behavior/cdi_device.h +index 3f460152..5d63a576 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_device.h ++++ b/src/daemon/modules/device/cdi/behavior/cdi_device.h +@@ -37,7 +37,7 @@ void free_cdi_cache_device(struct cdi_cache_device *d); + struct cdi_cache_device *cdi_device_new_device(struct cdi_cache_spec *spec, cdi_device *d, char **error); + struct cdi_cache_spec *cdi_device_get_spec(struct cdi_cache_device *d); + char *cdi_device_get_qualified_name(struct cdi_cache_device *d); +-cdi_container_edits *cdi_device_edits(struct cdi_cache_device *d); ++cdi_container_edits *cdi_device_get_edits(struct cdi_cache_device *d); + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec.c b/src/daemon/modules/device/cdi/behavior/cdi_spec.c +index 38fc9e38..f79b5a44 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_spec.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec.c +@@ -54,7 +54,7 @@ int cdi_spec_get_priority(struct cdi_cache_spec *s) + return 0; + } + +-cdi_container_edits *cdi_spec_edits(struct cdi_cache_spec *s) ++cdi_container_edits *cdi_spec_get_edits(struct cdi_cache_spec *s) + { + return NULL; + } +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec.h b/src/daemon/modules/device/cdi/behavior/cdi_spec.h +index bd4fc9d1..87248041 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_spec.h ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec.h +@@ -47,7 +47,7 @@ const char *cdi_spec_get_class(struct cdi_cache_spec *s); + struct cdi_cache_device *cdi_spec_get_cache_device(struct cdi_cache_spec *s, const char *name); + const char *cdi_spec_get_path(struct cdi_cache_spec *s); + int cdi_spec_get_priority(struct cdi_cache_spec *s); +-cdi_container_edits *cdi_spec_edits(struct cdi_cache_spec *s); ++cdi_container_edits *cdi_spec_get_edits(struct cdi_cache_spec *s); + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c +index 5df4c937..e340abc0 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c +@@ -23,7 +23,7 @@ string_array g_default_spec_dirs = { + .cap = DEFAULT_SPEC_DIRS_LEN, + }; + +-char *cdi_scan_spec_dirs(string_array *dirs, struct cdi_scan_fn_maps *scan_fn_maps, cdi_scan_spec_func scan_fn) ++int cdi_scan_spec_dirs(string_array *dirs, struct cdi_scan_fn_maps *scan_fn_maps, cdi_scan_spec_func scan_fn) + { +- return NULL; ++ return 0; + } +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h +index bd00e318..73d8c0f5 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h +@@ -35,10 +35,10 @@ struct cdi_scan_fn_maps { + map_t *spec_errors; + string_array *result; + }; +-typedef char *(*cdi_scan_spec_func)(struct cdi_scan_fn_maps *scan_fn_maps, const char *path, int priority, +- struct cdi_cache_spec *spec, char **error); ++typedef void(*cdi_scan_spec_func)(struct cdi_scan_fn_maps *scan_fn_maps, const char *path, int priority, ++ struct cdi_cache_spec *spec, char *error); + +-char *cdi_scan_spec_dirs(string_array *dirs, struct cdi_scan_fn_maps *scan_fn_maps, cdi_scan_spec_func scan_fn); ++int cdi_scan_spec_dirs(string_array *dirs, struct cdi_scan_fn_maps *scan_fn_maps, cdi_scan_spec_func scan_fn); + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c +index 45048f9a..14293c72 100644 +--- a/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c ++++ b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c +@@ -24,9 +24,9 @@ bool cdi_parser_is_qualified_name(const char *device) + return true; + } + +-char *cdi_parser_parse_qualified_name(const char *device, char **vendor, char **class, char **name) ++int cdi_parser_parse_qualified_name(const char *device, char **vendor, char **class, char **name) + { +- return NULL; ++ return 0; + } + + int cdi_parser_parse_device(const char *device, char **vendor, char **class, char **name) +@@ -39,17 +39,17 @@ int cdi_parser_parse_qualifier(const char *kind, char **vendor, char **class) + return 0; + } + +-char *cdi_parser_validate_vendor_name(const char *vendor) ++int cdi_parser_validate_vendor_name(const char *vendor, char **error) + { +- return NULL; ++ return 0; + } + +-char *cdi_parser_validate_class_name(const char *class) ++int cdi_parser_validate_class_name(const char *class, char **error) + { +- return NULL; ++ return 0; + } + +-char *cdi_parser_validate_device_name(const char *name) ++int cdi_parser_validate_device_name(const char *name, char **error) + { +- return NULL; ++ return 0; + } +diff --git a/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.h b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.h +index d9c057ea..467641a1 100644 +--- a/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.h ++++ b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.h +@@ -24,12 +24,12 @@ extern "C" { + + char *cdi_parser_qualified_name(const char *vendor, const char *class, const char *name); + bool cdi_parser_is_qualified_name(const char *device); +-char *cdi_parser_parse_qualified_name(const char *device, char **vendor, char **class, char **name); ++int cdi_parser_parse_qualified_name(const char *device, char **vendor, char **class, char **name); + int cdi_parser_parse_device(const char *device, char **vendor, char **class, char **name); + int cdi_parser_parse_qualifier(const char *kind, char **vendor, char **class); +-char *cdi_parser_validate_vendor_name(const char *vendor); +-char *cdi_parser_validate_class_name(const char *class); +-char *cdi_parser_validate_device_name(const char *name); ++int cdi_parser_validate_vendor_name(const char *vendor, char **error); ++int cdi_parser_validate_class_name(const char *class, char **error); ++int cdi_parser_validate_device_name(const char *name, char **error); + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/device/cdi/cdi_annotations.c b/src/daemon/modules/device/cdi/cdi_annotations.c +index 3cb9be84..cfe6e099 100644 +--- a/src/daemon/modules/device/cdi/cdi_annotations.c ++++ b/src/daemon/modules/device/cdi/cdi_annotations.c +@@ -25,7 +25,8 @@ + + #define CDI_ANNOTATIONS_PREFIX "cdi.k8s.io/" + +-char *cdi_parse_annotations(json_map_string_string *annotations, string_array **keys, string_array **devices) ++int cdi_parse_annotations(json_map_string_string *annotations, string_array **keys, ++ string_array **devices, char **error) + { +- return NULL; ++ return 0; + } +diff --git a/src/daemon/modules/device/cdi/cdi_annotations.h b/src/daemon/modules/device/cdi/cdi_annotations.h +index 52355099..49930963 100644 +--- a/src/daemon/modules/device/cdi/cdi_annotations.h ++++ b/src/daemon/modules/device/cdi/cdi_annotations.h +@@ -23,7 +23,8 @@ + extern "C" { + #endif + +-char *cdi_parse_annotations(json_map_string_string *annotations, string_array **keys, string_array **devices); ++int cdi_parse_annotations(json_map_string_string *annotations, string_array **keys, ++ string_array **devices, char **error); + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/device/cdi/cdi_cache.c b/src/daemon/modules/device/cdi/cdi_cache.c +index 9c54acbf..cfc23a1c 100644 +--- a/src/daemon/modules/device/cdi/cdi_cache.c ++++ b/src/daemon/modules/device/cdi/cdi_cache.c +@@ -19,24 +19,24 @@ void free_cdi_cache(struct cdi_cache *c) + (void)c; + } + +-struct cdi_cache *cdi_new_cache(string_array *spec_dirs, char **error) ++struct cdi_cache *cdi_new_cache(string_array *spec_dirs) + { + return NULL; + } + +-static string_array *cdi_inject_devices(struct cdi_cache *c, oci_runtime_spec *oci_spec, string_array *devices, char **error) ++static int cdi_inject_devices(struct cdi_cache *c, oci_runtime_spec *oci_spec, string_array *devices) + { +- return NULL; ++ return 0; + } + +-static char *cdi_configure(struct cdi_cache *c, string_array *spec_dirs) ++static int cdi_configure(struct cdi_cache *c, string_array *spec_dirs) + { +- return NULL; ++ return 0; + } + +-static char *cdi_refresh(struct cdi_cache *c) ++static int cdi_refresh(struct cdi_cache *c) + { +- return NULL; ++ return 0; + } + + static map_t *cdi_get_errors(struct cdi_cache *c) +diff --git a/src/daemon/modules/device/cdi/cdi_cache.h b/src/daemon/modules/device/cdi/cdi_cache.h +index 92fb64af..34c27471 100644 +--- a/src/daemon/modules/device/cdi/cdi_cache.h ++++ b/src/daemon/modules/device/cdi/cdi_cache.h +@@ -33,12 +33,12 @@ struct cdi_cache; + struct cdi_cache_ops { + // injecting CDI devices into an OCI Spec. + // Resolver +- string_array *(*inject_devices)(struct cdi_cache *c, oci_runtime_spec *spec, string_array *devices, char **error); ++ int (*inject_devices)(struct cdi_cache *c, oci_runtime_spec *spec, string_array *devices); + + // refreshing the cache of CDI Specs and devices. + // Refresher +- char *(*configure)(struct cdi_cache *c, string_array *spec_dirs); +- char *(*refresh)(struct cdi_cache *c); ++ int (*configure)(struct cdi_cache *c, string_array *spec_dirs); ++ int (*refresh)(struct cdi_cache *c); + map_t *(*get_errors)(struct cdi_cache *c); + string_array *(*get_spec_directories)(struct cdi_cache *c); + map_t *(*get_spec_dir_errors)(struct cdi_cache *c); +@@ -65,7 +65,7 @@ struct cdi_cache { + + void free_cdi_cache(struct cdi_cache *c); + +-struct cdi_cache *cdi_new_cache(string_array *spec_dirs, char **error); ++struct cdi_cache *cdi_new_cache(string_array *spec_dirs); + struct cdi_cache_ops *cdi_get_cache_ops(void); + + #ifdef __cplusplus +diff --git a/src/daemon/modules/device/cdi_operate.c b/src/daemon/modules/device/cdi_operate.c +index c7aa77d8..c5187ab1 100644 +--- a/src/daemon/modules/device/cdi_operate.c ++++ b/src/daemon/modules/device/cdi_operate.c +@@ -19,17 +19,18 @@ int cdi_operate_registry_init(char **specs_dirs, size_t specs_dirs_len) + return 0; + } + +-char *cdi_operate_refresh(void) ++int cdi_operate_refresh(void) + { +- return NULL; ++ return 0; + } + +-string_array *cdi_operate_inject_devices(oci_runtime_spec *spec, string_array *devices, char **error) ++int cdi_operate_inject_devices(oci_runtime_spec *spec, string_array *devices) + { +- return NULL; ++ return 0; + } + +-char *cdi_operate_parse_annotations(json_map_string_string *annotations, string_array **keys, string_array **devices) ++int cdi_operate_parse_annotations(json_map_string_string *annotations, string_array **keys, ++ string_array **devices, char **error) + { +- return NULL; ++ return 0; + } +\ No newline at end of file +-- +2.34.1 + diff --git a/0061-cdi-support-modules-operate-registry-annotations.patch b/0061-cdi-support-modules-operate-registry-annotations.patch new file mode 100644 index 0000000..aa26ca2 --- /dev/null +++ b/0061-cdi-support-modules-operate-registry-annotations.patch @@ -0,0 +1,516 @@ +From 4527dc8b6f7ab438742a8f7403b24420f646236d Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Mon, 8 Apr 2024 11:57:16 +0800 +Subject: [PATCH 61/69] cdi:support modules operate/registry/annotations + +--- + .../modules/device/cdi/cdi_annotations.c | 72 +++++++ + src/daemon/modules/device/cdi/cdi_registry.c | 14 +- + src/daemon/modules/device/cdi_operate.c | 53 +++++- + src/utils/cutils/utils.c | 11 ++ + src/utils/cutils/utils.h | 2 + + src/utils/cutils/utils_array.c | 175 +++++++++++++++++- + src/utils/cutils/utils_array.h | 31 ++++ + 7 files changed, 350 insertions(+), 8 deletions(-) + +diff --git a/src/daemon/modules/device/cdi/cdi_annotations.c b/src/daemon/modules/device/cdi/cdi_annotations.c +index cfe6e099..020816d7 100644 +--- a/src/daemon/modules/device/cdi/cdi_annotations.c ++++ b/src/daemon/modules/device/cdi/cdi_annotations.c +@@ -21,12 +21,84 @@ + + #include "error.h" + #include "utils.h" ++#include "utils_array.h" + #include "cdi_parser.h" + + #define CDI_ANNOTATIONS_PREFIX "cdi.k8s.io/" + ++static int parse_devices(string_array *devices, const char *value, char **error) ++{ ++ __isula_auto_array_t char **parts = NULL; ++ char **pos; ++ ++ parts = util_string_split(value, ','); ++ if (parts == NULL) { ++ ERROR("Invalid CDI device value %s", value); ++ format_errorf(error, "Invalid CDI device value %s", value); ++ return -1; ++ } ++ for (pos = parts; pos != NULL && *pos != NULL; pos++) { ++ if (!cdi_parser_is_qualified_name(*pos)) { ++ ERROR("Invalid CDI device name %s", *pos); ++ format_errorf(error, "Invalid CDI device name %s", *pos); ++ return -1; ++ } ++ if (util_append_string_array(devices, *pos) != 0) { ++ ERROR("Out of memory"); ++ *error = util_strdup_s("Out of memory"); ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ + int cdi_parse_annotations(json_map_string_string *annotations, string_array **keys, + string_array **devices, char **error) + { ++ char *key = NULL; ++ char *value = NULL; ++ size_t i; ++ __isula_auto_string_array_t string_array *keys_array = NULL; ++ __isula_auto_string_array_t string_array *devices_array = NULL; ++ ++ if (annotations == NULL || keys == NULL || devices == NULL || error == NULL) { ++ ERROR("Invalid argument"); ++ return -1; ++ } ++ ++ keys_array = util_common_calloc_s(sizeof(*keys_array)); ++ if (keys_array == NULL) { ++ ERROR("Out of memory"); ++ *error = util_strdup_s("Out of memory"); ++ return -1; ++ } ++ devices_array = util_common_calloc_s(sizeof(*devices_array)); ++ if (devices_array == NULL) { ++ ERROR("Out of memory"); ++ *error = util_strdup_s("Out of memory"); ++ return -1; ++ } ++ ++ for (i = 0; i < annotations->len; i++) { ++ key = annotations->keys[i]; ++ value = annotations->values[i]; ++ if (!util_has_prefix(key, CDI_ANNOTATIONS_PREFIX)) { ++ continue; ++ } ++ if (parse_devices(devices_array, value, error) != 0) { ++ return -1; ++ } ++ if (util_append_string_array(keys_array, key) != 0) { ++ ERROR("Out of memory"); ++ *error = util_strdup_s("Out of memory"); ++ return -1; ++ } ++ } ++ ++ *keys = keys_array; ++ keys_array = NULL; ++ *devices = devices_array; ++ devices_array = NULL; + return 0; + } +diff --git a/src/daemon/modules/device/cdi/cdi_registry.c b/src/daemon/modules/device/cdi/cdi_registry.c +index 68767a5f..be381132 100644 +--- a/src/daemon/modules/device/cdi/cdi_registry.c ++++ b/src/daemon/modules/device/cdi/cdi_registry.c +@@ -14,12 +14,24 @@ + ******************************************************************************/ + #include "cdi_registry.h" + ++#include <util_atomic.h> ++#include <isula_libutils/auto_cleanup.h> ++ ++static struct cdi_registry g_cdi_reg = { 0 }; ++ + int cdi_registry_init(string_array *spec_dirs) + { ++ // isulad will use default dirs when spec_dirs == NULL ++ g_cdi_reg.cdi_cache = cdi_new_cache(spec_dirs); ++ if (g_cdi_reg.cdi_cache == NULL) { ++ ERROR("Failed to init registry"); ++ return -1; ++ } ++ g_cdi_reg.ops = cdi_get_cache_ops(); + return 0; + } + + struct cdi_registry *cdi_get_registry(void) + { +- return NULL; ++ return &g_cdi_reg; + } +diff --git a/src/daemon/modules/device/cdi_operate.c b/src/daemon/modules/device/cdi_operate.c +index c5187ab1..f99bb7e4 100644 +--- a/src/daemon/modules/device/cdi_operate.c ++++ b/src/daemon/modules/device/cdi_operate.c +@@ -14,23 +14,66 @@ + ******************************************************************************/ + #include "cdi_operate_api.h" + ++#include <isula_libutils/log.h> ++ ++#include "utils.h" ++#include "error.h" ++#include "cdi_registry.h" ++#include "cdi_annotations.h" ++#include "cdi_spec_dirs.h" ++ + int cdi_operate_registry_init(char **specs_dirs, size_t specs_dirs_len) + { +- return 0; ++ string_array spec_dirs_array = { ++ .items = specs_dirs, ++ .len = specs_dirs_len, ++ .cap = specs_dirs_len, ++ }; ++ ++ return cdi_registry_init(&spec_dirs_array); + } + + int cdi_operate_refresh(void) + { +- return 0; ++ struct cdi_registry *registry = cdi_get_registry(); ++ if (registry == NULL || registry->ops == NULL || registry->ops->refresh == NULL) { ++ ERROR("Failed to get registry"); ++ return -1; ++ } ++ ++ return registry->ops->refresh(registry->cdi_cache); + } + + int cdi_operate_inject_devices(oci_runtime_spec *spec, string_array *devices) + { +- return 0; ++ struct cdi_registry *registry = NULL; ++ ++ if (spec == NULL || devices == NULL) { ++ ERROR("Invalid params"); ++ return -1; ++ } ++ ++ registry = cdi_get_registry(); ++ if (registry == NULL || registry->ops == NULL || registry->ops->inject_devices == NULL) { ++ ERROR("Failed to get registry"); ++ return -1; ++ } ++ ++ return registry->ops->inject_devices(registry->cdi_cache, spec, devices); + } + + int cdi_operate_parse_annotations(json_map_string_string *annotations, string_array **keys, + string_array **devices, char **error) + { +- return 0; +-} +\ No newline at end of file ++ if (error == NULL) { ++ ERROR("Invalid argument"); ++ return -1; ++ } ++ if (annotations == NULL || keys == NULL || devices == NULL) { ++ ERROR("Invalid params"); ++ *error = util_strdup_s("Invalid params"); ++ return -1; ++ } ++ ++ return cdi_parse_annotations(annotations, keys, devices, error); ++} +diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c +index 8da2cc60..9a33f935 100644 +--- a/src/utils/cutils/utils.c ++++ b/src/utils/cutils/utils.c +@@ -59,6 +59,17 @@ int malloc_trim(size_t pad) + } + #endif + ++void util_swap_ptr(void **p1, void **p2) ++{ ++ void *tmp; ++ if (p1 == NULL || p2 == NULL) { ++ return; ++ } ++ tmp = *p1; ++ *p1 = *p2; ++ *p2 = tmp; ++} ++ + int util_mem_realloc(void **newptr, size_t newsize, void *oldptr, size_t oldsize) + { + void *tmp = NULL; +diff --git a/src/utils/cutils/utils.h b/src/utils/cutils/utils.h +index 3acf0698..3671272a 100644 +--- a/src/utils/cutils/utils.h ++++ b/src/utils/cutils/utils.h +@@ -320,6 +320,8 @@ struct signame { + } \ + } while (0) + ++void util_swap_ptr(void **p1, void **p2); ++ + int util_mem_realloc(void **newptr, size_t newsize, void *oldptr, size_t oldsize); + + int util_check_inherited(bool closeall, int fd_to_ignore); +diff --git a/src/utils/cutils/utils_array.c b/src/utils/cutils/utils_array.c +index 25f19b8b..72294005 100644 +--- a/src/utils/cutils/utils_array.c ++++ b/src/utils/cutils/utils_array.c +@@ -86,6 +86,27 @@ void util_free_sensitive_array(char **array) + free(array); + } + ++char **util_copy_array_by_len(char **array, size_t len) ++{ ++ char **new_array = NULL; ++ size_t i; ++ ++ if (array == NULL || len == 0) { ++ return NULL; ++ } ++ ++ new_array = util_smart_calloc_s(sizeof(char *), len); ++ if (new_array == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ for (i = 0; i < len; i++) { ++ new_array[i] = util_strdup_s(array[i]); ++ } ++ return new_array; ++} ++ + int util_array_append(char ***array, const char *element) + { + size_t len; +@@ -166,7 +187,7 @@ bool util_array_contain(const char **array, const char *element) + return false; + } + +-static size_t get_string_array_scale_size(size_t old_size) ++static size_t get_array_scale_size(size_t old_size) + { + #define DOUBLE_THRESHOLD 1024 + const size_t max_threshold = MAX_MEMORY_SIZE / sizeof(char *); +@@ -188,7 +209,7 @@ static size_t get_string_array_scale_size(size_t old_size) + + static bool do_expand_array(string_array *array) + { +- size_t new_size = get_string_array_scale_size(array->cap); ++ size_t new_size = get_array_scale_size(array->cap); + char **new_items = NULL; + + // array capability sure less than MAX_MEMORY_SIZE +@@ -237,6 +258,29 @@ out: + return 0; + } + ++string_array *util_copy_string_array(string_array *sarr) ++{ ++ string_array *ptr = NULL; ++ size_t i; ++ ++ if (sarr == NULL) { ++ ERROR("Invalid string array"); ++ return NULL; ++ } ++ ++ ptr = util_string_array_new(sarr->cap); ++ if (ptr == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ for (i = 0; i < sarr->len; i++) { ++ ptr->items[i] = util_strdup_s(sarr->items[i]); ++ ptr->len += 1; ++ } ++ ++ return ptr; ++} ++ + bool util_string_array_contain(const string_array *sarr, const char *elem) + { + size_t i; +@@ -339,3 +383,130 @@ int util_common_array_append_pointer(void ***array, void *element) + + return 0; + } ++ ++void *util_clone_ptr(void *item) ++{ ++ return item; ++} ++ ++common_array *util_common_array_new(size_t len, free_common_array_item_cb free_item_cb, ++ clone_common_array_item_cb clone_item_cb) ++{ ++ common_array *ptr = NULL; ++ ++ if (len == 0 || free_item_cb == NULL || clone_item_cb == NULL) { ++ return NULL; ++ } ++ ++ ptr = (common_array *)util_common_calloc_s(sizeof(common_array)); ++ if (ptr == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ ptr->items = (void **)util_smart_calloc_s(sizeof(void *), len); ++ if (ptr->items == NULL) { ++ ERROR("Out of memory"); ++ free(ptr); ++ return NULL; ++ } ++ ++ ptr->len = 0; ++ ptr->cap = len; ++ ptr->free_item_cb = free_item_cb; ++ ptr->clone_item_cb = clone_item_cb; ++ ++ return ptr; ++} ++ ++void util_free_common_array(common_array *ptr) ++{ ++ size_t i; ++ ++ if (ptr == NULL || ptr->free_item_cb == NULL) { ++ return; ++ } ++ ++ for (i = 0; i < ptr->len; i++) { ++ ptr->free_item_cb(ptr->items[i]); ++ ptr->items[i] = NULL; ++ } ++ free(ptr->items); ++ ptr->items = NULL; ++ ptr->len = 0; ++ ptr->cap = 0; ++ ptr->free_item_cb = NULL; ++ ptr->clone_item_cb = NULL; ++ ++ free(ptr); ++} ++ ++static bool do_expand_common_array(common_array *array) ++{ ++ size_t new_size = get_array_scale_size(array->cap); ++ void **new_items = NULL; ++ ++ // array capability sure less than MAX_MEMORY_SIZE ++ // so we need to check Overflow: ++ if (new_size == array->cap) { ++ ERROR("Too large common array, overflow memory"); ++ return false; ++ } ++ ++ // new_size * sizeof(*new_items) and list->len * sizeof(*list->items) ++ if (util_mem_realloc((void **)&new_items, new_size * sizeof(void *), (void *)array->items, ++ array->len * sizeof(void *)) != 0) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ array->items = new_items; ++ array->cap = new_size; ++ ++ return true; ++} ++ ++int util_append_common_array(common_array *arr, void *val) ++{ ++ if (arr == NULL || arr->clone_item_cb == NULL) { ++ ERROR("Invalid common array"); ++ return -1; ++ } ++ ++ if (val == NULL) { ++ DEBUG("Empty new item, just ignore it"); ++ return 0; ++ } ++ ++ if (arr->len < arr->cap) { ++ goto out; ++ } ++ ++ // expand common array ++ if (!do_expand_common_array(arr)) { ++ return -1; ++ } ++ ++out: ++ arr->items[arr->len] = arr->clone_item_cb(val); ++ arr->len += 1; ++ return 0; ++} ++ ++int util_merge_common_array(common_array *dest_arr, common_array *src_arr) ++{ ++ size_t i; ++ ++ if (dest_arr == NULL || dest_arr->clone_item_cb == NULL || ++ src_arr == NULL || src_arr->clone_item_cb == NULL) { ++ ERROR("Invalid common array"); ++ return -1; ++ } ++ ++ for (i = 0; i < src_arr->len; i++) { ++ if (util_append_common_array(dest_arr, src_arr->items[i]) != 0) { ++ ERROR("Failed to append element"); ++ return -1; ++ } ++ } ++ return 0; ++} +diff --git a/src/utils/cutils/utils_array.h b/src/utils/cutils/utils_array.h +index 1c084595..0c4fd217 100644 +--- a/src/utils/cutils/utils_array.h ++++ b/src/utils/cutils/utils_array.h +@@ -30,6 +30,8 @@ void util_free_array_by_len(char **array, size_t len); + + void util_free_array(char **array); + ++char **util_copy_array_by_len(char **array, size_t len); ++ + int util_grow_array(char ***orig_array, size_t *orig_capacity, size_t size, + size_t increment); + +@@ -52,6 +54,8 @@ void util_free_string_array(string_array *ptr); + + int util_append_string_array(string_array *sarr, const char *val); + ++string_array *util_copy_string_array(string_array *sarr); ++ + bool util_string_array_contain(const string_array *sarr, const char *elem); + + void util_free_sensitive_array(char **array); +@@ -63,6 +67,33 @@ define_auto_cleanup_callback(util_free_array, char *); + // define auto free macro for char * + #define __isula_auto_array_t auto_cleanup_tag(util_free_array) + ++define_auto_cleanup_callback(util_free_string_array, string_array); ++#define __isula_auto_string_array_t auto_cleanup_tag(util_free_string_array) ++ ++typedef void (*free_common_array_item_cb)(void *item); ++typedef void *(*clone_common_array_item_cb)(void *item); ++typedef struct common_array_t { ++ void **items; ++ size_t len; ++ size_t cap; ++ free_common_array_item_cb free_item_cb; ++ clone_common_array_item_cb clone_item_cb; ++} common_array; ++ ++void *util_clone_ptr(void *item); ++ ++common_array *util_common_array_new(size_t len, free_common_array_item_cb free_item_cb, ++ clone_common_array_item_cb clone_item_cb); ++ ++void util_free_common_array(common_array *ptr); ++ ++int util_append_common_array(common_array *arr, void *val); ++ ++int util_merge_common_array(common_array *dest_arr, common_array *src_arr); ++ ++define_auto_cleanup_callback(util_free_common_array, common_array); ++#define __isula_auto_common_array_t auto_cleanup_tag(util_free_common_array) ++ + #ifdef __cplusplus + } + #endif +-- +2.34.1 + diff --git a/0062-do-not-umount-shmpath-for-sandbox-container.patch b/0062-do-not-umount-shmpath-for-sandbox-container.patch new file mode 100644 index 0000000..8b719a3 --- /dev/null +++ b/0062-do-not-umount-shmpath-for-sandbox-container.patch @@ -0,0 +1,30 @@ +From 8e35525073b52b5d161984015de641bd21570380 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Tue, 16 Apr 2024 10:32:50 +0000 +Subject: [PATCH 62/69] do not umount shmpath for sandbox container + +Signed-off-by: jikai <jikai11@huawei.com> +--- + src/daemon/modules/service/service_container.c | 6 ++++++ + 1 file changed, 6 insertions(+) + +diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c +index eb7ce4f4..a2322309 100644 +--- a/src/daemon/modules/service/service_container.c ++++ b/src/daemon/modules/service/service_container.c +@@ -1718,6 +1718,12 @@ void umount_share_shm(container_t *cont) + return; + } + if (cont->hostconfig->ipc_mode == NULL || namespace_is_shareable(cont->hostconfig->ipc_mode)) { ++#ifdef ENABLE_CRI_API_V1 ++ // For sandbox in cri v1, the shm path is created and umounted in CRI ++ if (is_sandbox_container(cont->common_config->sandbox_info)) { ++ return; ++ } ++#endif + if (cont->common_config == NULL || cont->common_config->shm_path == NULL) { + return; + } +-- +2.34.1 + diff --git a/0063-remove-default-systemd-cgroup-and-enable-cri-v1-valu.patch b/0063-remove-default-systemd-cgroup-and-enable-cri-v1-valu.patch new file mode 100644 index 0000000..b6a0e6b --- /dev/null +++ b/0063-remove-default-systemd-cgroup-and-enable-cri-v1-valu.patch @@ -0,0 +1,27 @@ +From c092597565e5f24b29ecd83b4b371a19a9c2db0d Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Wed, 17 Apr 2024 01:52:45 +0000 +Subject: [PATCH 63/69] remove default systemd-cgroup and enable-cri-v1 value + in daemon.json + +Signed-off-by: jikai <jikai11@huawei.com> +--- + src/contrib/config/daemon.json | 4 +--- + 1 file changed, 1 insertion(+), 3 deletions(-) + +diff --git a/src/contrib/config/daemon.json b/src/contrib/config/daemon.json +index 69362c26..711dda94 100644 +--- a/src/contrib/config/daemon.json ++++ b/src/contrib/config/daemon.json +@@ -35,7 +35,5 @@ + "insecure-skip-verify-enforce": false, + "cri-runtimes": { + "kata": "io.containerd.kata.v2" +- }, +- "enable-cri-v1": false, +- "systemd-cgroup": false ++ } + } +-- +2.34.1 + diff --git a/0064-cdi-support-module-cache.patch b/0064-cdi-support-module-cache.patch new file mode 100644 index 0000000..4ed9fb5 --- /dev/null +++ b/0064-cdi-support-module-cache.patch @@ -0,0 +1,862 @@ +From f87f0fddaec4b3aea72f3b9c91f2dc91b89683ce Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Tue, 16 Apr 2024 11:44:22 +0800 +Subject: [PATCH 64/69] cdi:support module cache + +--- + .../device/cdi/behavior/cdi_spec_dirs.h | 3 +- + src/daemon/modules/device/cdi/cdi_cache.c | 760 +++++++++++++++++- + src/daemon/modules/device/cdi/cdi_cache.h | 9 +- + 3 files changed, 747 insertions(+), 25 deletions(-) + +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h +index 73d8c0f5..eedcabad 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h +@@ -32,8 +32,7 @@ struct cdi_scan_fn_maps { + map_t *specs; + map_t *devices; + map_t *conflicts; +- map_t *spec_errors; +- string_array *result; ++ bool *refresh_error_flag; + }; + typedef void(*cdi_scan_spec_func)(struct cdi_scan_fn_maps *scan_fn_maps, const char *path, int priority, + struct cdi_cache_spec *spec, char *error); +diff --git a/src/daemon/modules/device/cdi/cdi_cache.c b/src/daemon/modules/device/cdi/cdi_cache.c +index cfc23a1c..37767855 100644 +--- a/src/daemon/modules/device/cdi/cdi_cache.c ++++ b/src/daemon/modules/device/cdi/cdi_cache.c +@@ -14,56 +14,784 @@ + ******************************************************************************/ + #include "cdi_cache.h" + ++#include <stdlib.h> ++#include <pthread.h> ++#include <sys/inotify.h> ++#include <sys/prctl.h> ++#include <isula_libutils/log.h> ++#include <isula_libutils/auto_cleanup.h> ++#include <isula_libutils/utils_array.h> ++ ++#include "utils.h" ++#include "utils_file.h" ++#include "path.h" ++#include "error.h" ++#include "cdi_device.h" ++#include "cdi_spec.h" ++#include "cdi_spec_dirs.h" ++#include "cdi_container_edits.h" ++ ++// cache ++static int cdi_set_spec_dirs(struct cdi_cache *c, string_array *spec_dirs); ++static int configure(struct cdi_cache *c, string_array *spec_dirs); ++static int refresh(struct cdi_cache *c); ++static bool refresh_if_required(struct cdi_cache *c, bool force, int *ret); ++ ++// watch ++static void free_cdi_watch(struct cdi_watch *watch); ++static void watch_setup(struct cdi_watch *watch, string_array *dirs); ++static void watch_start(struct cdi_cache *c); ++static void watch_stop(struct cdi_watch *w); ++static void *watch_thread_func(void *arg); ++static bool watch_update(struct cdi_watch *w, const char *removed, int wd); ++ ++static int cdi_set_spec_dirs(struct cdi_cache *c, string_array *spec_dirs) ++{ ++ __isula_auto_string_array_t string_array *new_spec_dirs = NULL; ++ char clean_path[PATH_MAX] = { 0 }; ++ size_t i; ++ ++ if (c == NULL || spec_dirs == NULL) { ++ ERROR("Invalid argument"); ++ return -1; ++ } ++ if (spec_dirs->len == 0) { ++ return 0; ++ } ++ ++ new_spec_dirs = util_string_array_new(spec_dirs->len); ++ if (new_spec_dirs == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ for (i = 0; i < spec_dirs->len; i++) { ++ if (util_clean_path(spec_dirs->items[i], clean_path, sizeof(clean_path)) == NULL) { ++ ERROR("Failed to get clean path %s", spec_dirs->items[i]); ++ return -1; ++ } ++ if (util_append_string_array(new_spec_dirs, clean_path) != 0) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ } ++ util_free_string_array(c->spec_dirs); ++ c->spec_dirs = new_spec_dirs; ++ new_spec_dirs = NULL; ++ ++ return 0; ++} ++ + void free_cdi_cache(struct cdi_cache *c) + { +- (void)c; ++ if (c == NULL) { ++ return; ++ } ++ ++ util_free_string_array(c->spec_dirs); ++ c->spec_dirs = NULL; ++ map_free(c->specs); ++ c->specs = NULL; ++ map_free(c->devices); ++ c->devices = NULL; ++ free_cdi_watch(c->watch); ++ c->watch = NULL; ++ ++ free(c); + } + + struct cdi_cache *cdi_new_cache(string_array *spec_dirs) + { ++ struct cdi_cache *c = NULL; ++ int ret = 0; ++ ++ c = util_common_calloc_s(sizeof(*c)); ++ if (c == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ c->refresh_error_flag = false; ++ c->auto_refresh = true; ++ c->watch = util_common_calloc_s(sizeof(struct cdi_watch)); ++ if (c->watch == NULL) { ++ ERROR("Out of memory"); ++ goto free_out; ++ } ++ c->watch->watcher_fd = -1; ++ ++ if (cdi_set_spec_dirs(c, &g_default_spec_dirs) != 0) { ++ ERROR("Failed to set spec dirs by default"); ++ goto free_out; ++ } ++ ++ (void)pthread_mutex_lock(&c->mutex); ++ ret = configure(c, spec_dirs); ++ (void)pthread_mutex_unlock(&c->mutex); ++ if (ret != 0) { ++ ERROR("Failed to configure"); ++ goto free_out; ++ } ++ ++ return c; ++ ++free_out: ++ free_cdi_cache(c); + return NULL; + } + +-static int cdi_inject_devices(struct cdi_cache *c, oci_runtime_spec *oci_spec, string_array *devices) ++static int cdi_configure(struct cdi_cache *c, string_array *spec_dirs) + { +- return 0; ++ int ret = 0; ++ ++ if (c == NULL) { ++ ERROR("Invalid arguments"); ++ return -1; ++ } ++ ++ (void)pthread_mutex_lock(&c->mutex); ++ ret = configure(c, spec_dirs); ++ (void)pthread_mutex_unlock(&c->mutex); ++ ++ return ret; + } + +-static int cdi_configure(struct cdi_cache *c, string_array *spec_dirs) ++static int configure(struct cdi_cache *c, string_array *spec_dirs) + { ++ int ret = 0; ++ ++ if (spec_dirs != NULL) { ++ ret = cdi_set_spec_dirs(c, spec_dirs); ++ if (ret != 0) { ++ ERROR("Failed to apply cache spec dirs"); ++ return -1; ++ } ++ } ++ ++ watch_stop(c->watch); ++ if (c->auto_refresh) { ++ watch_setup(c->watch, c->spec_dirs); ++ watch_start(c); ++ } ++ (void)refresh(c); + return 0; + } + + static int cdi_refresh(struct cdi_cache *c) + { +- return 0; ++ bool refreshed; ++ int ret = 0; ++ ++ if (c == NULL) { ++ ERROR("Invalid arguments"); ++ return -1; ++ } ++ ++ (void)pthread_mutex_lock(&c->mutex); ++ refreshed = refresh_if_required(c, !c->auto_refresh, &ret); ++ if (refreshed) { ++ goto unlock_out; ++ } ++ ++ ret = c->refresh_error_flag ? -1 : 0; ++unlock_out: ++ (void)pthread_mutex_unlock(&c->mutex); ++ return ret; + } + +-static map_t *cdi_get_errors(struct cdi_cache *c) ++static void map_cdi_cache_specs_kvfree(void *key, void *value) + { +- return NULL; ++ free(key); ++ util_free_common_array((common_array *)value); + } + +-static string_array *cdi_get_spec_directories(struct cdi_cache *c) ++static void map_cdi_cache_device_kvfree(void *key, void *value) + { +- return NULL; ++ free(key); ++ free_cdi_cache_device((struct cdi_cache_device *)value); + } + +-static map_t *cdi_get_spec_dir_errors(struct cdi_cache *c) ++static void set_refresh_error_flag(bool *refresh_error_flag, const char *error, const char *path) + { +- return NULL; ++ *refresh_error_flag = true; ++ ERROR("Cdi refresh error: %s, spec %s", error, path); ++} ++ ++static bool resolve_conflict(struct cdi_scan_fn_maps *scan_fn_maps, const char *name, ++ struct cdi_cache_device *dev, struct cdi_cache_device *old) ++{ ++ map_t *conflicts = scan_fn_maps->conflicts; ++ bool *refresh_error_flag = scan_fn_maps->refresh_error_flag; ++ struct cdi_cache_spec *dev_spec = NULL; ++ struct cdi_cache_spec *old_spec = NULL; ++ int dev_prio; ++ int old_prio; ++ bool val = true; ++ const char *dev_path = NULL; ++ const char *old_path = NULL; ++ ++ dev_spec = cdi_device_get_spec(dev); ++ old_spec = cdi_device_get_spec(old); ++ dev_prio = cdi_spec_get_priority(dev_spec); ++ old_prio = cdi_spec_get_priority(old_spec); ++ if (dev_prio > old_prio) { ++ return false; ++ } else if (dev_prio == old_prio) { ++ dev_path = cdi_spec_get_path(dev_spec); ++ old_path = cdi_spec_get_path(old_spec); ++ *refresh_error_flag = true; ++ ERROR("Conflicting device %s (specs %s, %s)", name, dev_path, old_path); ++ if (!map_replace(conflicts, (void *)name, (void *)&val)) { ++ ERROR("Failed to insert bool to conflicts by name %s", name); ++ return true; ++ } ++ } else { ++ // do nothing ++ } ++ ++ return true; ++} ++ ++static void refresh_scan_spec_func(struct cdi_scan_fn_maps *scan_fn_maps, const char *path, ++ int priority, struct cdi_cache_spec *spec, char *error) ++{ ++ map_t *specs = scan_fn_maps->specs; ++ map_t *devices = scan_fn_maps->devices; ++ bool *refresh_error_flag = scan_fn_maps->refresh_error_flag; ++ char clean_path[PATH_MAX] = { 0 }; ++ __isula_auto_free char *tmp_error = NULL; ++ const char *vendor = NULL; ++ __isula_auto_common_array_t common_array *spec_array = NULL; ++ map_itor *itor = NULL; ++ __isula_auto_free char *qualified = NULL; ++ struct cdi_cache_device *dev = NULL; ++ struct cdi_cache_device *other = NULL; ++ ++ if (util_clean_path(path, clean_path, sizeof(clean_path)) == NULL) { ++ ERROR("Failed to get clean path %s", path); ++ format_errorf(&tmp_error, "Failed to get clean path %s", path); ++ return; ++ } ++ if (error != NULL) { ++ ERROR("Failed to load CDI Spec %s", error); ++ format_errorf(&tmp_error, "Failed to load CDI Spec %s", error); ++ goto error_out; ++ } ++ ++ vendor = cdi_spec_get_vendor(spec); ++ spec_array = map_search(specs, (void *)vendor); ++ if (spec_array == NULL) { ++ spec_array = util_common_array_new(1, (free_common_array_item_cb)free_cdi_cache_spec, util_clone_ptr); ++ if (spec_array == NULL) { ++ ERROR("Out of memory"); ++ tmp_error = util_strdup_s("Out of memory"); ++ goto error_out; ++ } ++ if (!map_insert(specs, (void *)vendor, spec_array)) { ++ ERROR("Failed to insert spec array to specs"); ++ tmp_error = util_strdup_s("Failed to insert spec array to specs"); ++ goto error_out; ++ } ++ } ++ if (util_append_common_array(spec_array, spec) != 0) { ++ ERROR("Failed to append spec"); ++ tmp_error = util_strdup_s("Failed to append spec"); ++ goto error_out; ++ } ++ spec_array = NULL; ++ ++ itor = map_itor_new(spec->devices); ++ if (itor == NULL) { ++ ERROR("Out of memory, create new map itor failed"); ++ tmp_error = util_strdup_s("Out of memory, create new map itor failed"); ++ goto error_out; ++ } ++ for (; map_itor_valid(itor); map_itor_next(itor)) { ++ dev = map_itor_value(itor); ++ qualified = cdi_device_get_qualified_name(dev); ++ other = map_search(devices, (void *)qualified); ++ if (other != NULL) { ++ if (resolve_conflict(scan_fn_maps, qualified, dev, other)) { ++ continue; ++ } ++ } ++ if (!map_replace(devices, (void *)qualified, dev)) { ++ ERROR("Failed to insert device to devices by name %s", qualified); ++ format_errorf(&tmp_error, "Failed to insert device to devices by name %s", qualified); ++ goto error_out; ++ } ++ free(qualified); ++ qualified = NULL; ++ } ++ goto out; ++ ++error_out: ++ set_refresh_error_flag(refresh_error_flag, tmp_error, path); ++out: ++ map_itor_free(itor); ++ return; ++} ++ ++static int refresh(struct cdi_cache *c) ++{ ++ int ret = 0; ++ __isula_auto_free char *error = NULL; ++ map_t *specs = NULL; ++ map_t *devices = NULL; ++ map_t *conflicts = NULL; ++ struct cdi_scan_fn_maps scan_fn_maps = { 0 }; ++ map_itor *itor = NULL; ++ char *conflict = NULL; ++ ++ c->refresh_error_flag = false; ++ specs = map_new(MAP_STR_PTR, MAP_DEFAULT_CMP_FUNC, map_cdi_cache_specs_kvfree); ++ if (specs == NULL) { ++ ERROR("Out of memory"); ++ ret = -1; ++ goto free_out; ++ } ++ devices = map_new(MAP_STR_PTR, MAP_DEFAULT_CMP_FUNC, map_cdi_cache_device_kvfree); ++ if (devices == NULL) { ++ ERROR("Out of memory"); ++ ret = -1; ++ goto free_out; ++ } ++ conflicts = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ if (conflicts == NULL) { ++ ERROR("Out of memory"); ++ ret = -1; ++ goto free_out; ++ } ++ ++ scan_fn_maps.specs = specs; ++ scan_fn_maps.devices = devices; ++ scan_fn_maps.conflicts = conflicts; ++ scan_fn_maps.refresh_error_flag = &c->refresh_error_flag; ++ // ignore error when scan spec dirs ++ (void)cdi_scan_spec_dirs(c->spec_dirs, &scan_fn_maps, refresh_scan_spec_func); ++ ++ itor = map_itor_new(conflicts); ++ if (itor == NULL) { ++ ERROR("Out of memory, create new map itor failed"); ++ ret = -1; ++ goto free_out; ++ } ++ for (; map_itor_valid(itor); map_itor_next(itor)) { ++ conflict = map_itor_key(itor); ++ if ((map_search(devices, conflict) != NULL) && ++ !map_remove(devices, conflict)) { ++ ERROR("Failed to remove conflict device from devices"); ++ ret = -1; ++ goto free_out; ++ } ++ } ++ ++ util_swap_ptr((void **)&c->specs, (void **)&specs); ++ util_swap_ptr((void **)&c->devices, (void **)&devices); ++ ++ ret = c->refresh_error_flag ? -1 : 0; ++ ++free_out: ++ map_itor_free(itor); ++ map_free(specs); ++ map_free(devices); ++ map_free(conflicts); ++ return ret; ++} ++ ++static bool refresh_if_required(struct cdi_cache *c, bool force, int *ret) ++{ ++ if (force || (c->auto_refresh && watch_update(c->watch, NULL, -1))) { ++ *ret = refresh(c); ++ return true; ++ } ++ return false; ++} ++ ++static void map_spec_ptr_kvfree(void *key, void *value) ++{ ++ // do not need free spec* ++ (void)key; ++ free(value); ++} ++ ++static int cdi_inject_devices(struct cdi_cache *c, oci_runtime_spec *oci_spec, string_array *devices) ++{ ++ int ret = 0; ++ __isula_auto_string_array_t string_array *unresolved = NULL; ++ cdi_container_edits *edits = NULL; ++ map_t *specs = NULL; ++ size_t i; ++ const char *device = NULL; ++ struct cdi_cache_device *d = NULL; ++ int tmp_val = 0; ++ __isula_auto_free char *unresolved_str = NULL; ++ ++ if (c == NULL || devices == NULL) { ++ ERROR("Can't inject devices"); ++ return -1; ++ } ++ if (oci_spec == NULL) { ++ ERROR("Can't inject devices, nil OCI Spec"); ++ return -1; ++ } ++ ++ unresolved = util_common_calloc_s(sizeof(*unresolved)); ++ if (unresolved == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ specs = map_new(MAP_PTR_INT, MAP_DEFAULT_CMP_FUNC, map_spec_ptr_kvfree); ++ if (specs == NULL) { ++ ERROR("Out of memory"); ++ ret = -1; ++ goto out; ++ } ++ edits = util_common_calloc_s(sizeof(*edits)); ++ if (edits == NULL) { ++ ERROR("Out of memory"); ++ ret = -1; ++ goto out; ++ } ++ ++ (void)pthread_mutex_lock(&c->mutex); ++ ++ (void)refresh_if_required(c, false, &ret); ++ ++ for(i = 0; i < devices->len; i++) { ++ device = devices->items[i]; ++ d = map_search(c->devices, (void *)device); ++ if (d == NULL) { ++ if (util_append_string_array(unresolved, device) != 0) { ++ ERROR("Out of memory"); ++ ret = -1; ++ goto out; ++ } ++ continue; ++ } ++ if (map_search(specs, (void *)cdi_device_get_spec(d)) == NULL) { ++ if (!map_insert(specs, (void *)cdi_device_get_spec(d), (void *)&tmp_val)) { ++ ERROR("Failed to insert spec ptr to specs when find device %s", device); ++ ret = -1; ++ goto out; ++ } ++ if (cdi_container_edits_append(edits, cdi_spec_get_edits(cdi_device_get_spec(d))) != 0) { ++ ERROR("Failed to append edits when find device %s", device); ++ ret = -1; ++ goto out; ++ } ++ } ++ if (cdi_container_edits_append(edits, cdi_device_get_edits(d)) != 0) { ++ ERROR("Failed to append edits when find device %s", device); ++ ret = -1; ++ goto out; ++ } ++ } ++ ++ if (unresolved->len != 0) { ++ unresolved_str = util_string_join(", ", (const char **)unresolved->items, unresolved->len); ++ ERROR("Unresolvable CDI devices %s", unresolved_str); ++ ret = -1; ++ goto out; ++ } ++ ++ ret = cdi_container_edits_apply(edits, oci_spec); ++ if (ret != 0) { ++ ERROR("Failed to apply edits when inject devices"); ++ ret = -1; ++ } ++ ++out: ++ (void)pthread_mutex_unlock(&c->mutex); ++ map_free(specs); ++ free_cdi_container_edits(edits); ++ return ret; + } + + static struct cdi_cache_ops g_cdi_cache_ops = { + .inject_devices = cdi_inject_devices, + .configure = cdi_configure, +- .refresh = cdi_refresh, +- .get_errors = cdi_get_errors, +- .get_spec_directories = cdi_get_spec_directories, +- .get_spec_dir_errors = cdi_get_spec_dir_errors ++ .refresh = cdi_refresh + }; + + struct cdi_cache_ops *cdi_get_cache_ops(void) + { + return &g_cdi_cache_ops; +-} +\ No newline at end of file ++} ++ ++static void free_cdi_watch(struct cdi_watch *w) ++{ ++ if (w == NULL) { ++ return; ++ } ++ ++ watch_stop(w); ++ free(w); ++} ++ ++static int init_tracked(struct cdi_watch *w, string_array *dirs) ++{ ++ size_t i; ++ bool tmp_value = false; ++ ++ w->tracked = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ if (w->tracked == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ for(i = 0; i < dirs->len; i++) { ++ if (!map_replace(w->tracked, (void *)dirs->items[i], (void *)&tmp_value)) { ++ ERROR("Failed to insert tracked by dir %s", dirs->items[i]); ++ goto error_out; ++ } ++ } ++ w->wd_dirs = map_new(MAP_INT_STR, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ if (w->wd_dirs == NULL) { ++ ERROR("Out of memory"); ++ goto error_out; ++ } ++ ++ return 0; ++ ++error_out: ++ map_free(w->tracked); ++ w->tracked = NULL; ++ return -1; ++} ++ ++static void watch_setup(struct cdi_watch *w, string_array *dirs) ++{ ++ __isula_auto_free char *error = NULL; ++ ++ if (w == NULL || dirs == NULL || dirs->len == 0) { ++ ERROR("Invalid param"); ++ return; ++ } ++ ++ if (init_tracked(w, dirs) != 0) { ++ ERROR("Failed to initialize tracked"); ++ return; ++ } ++ ++ w->watcher_fd = inotify_init(); ++ if (w->watcher_fd < 0) { ++ ERROR("Failed to initialize inotify fd"); ++ map_free(w->tracked); ++ w->tracked = NULL; ++ map_free(w->wd_dirs); ++ w->wd_dirs = NULL; ++ return; ++ } ++ ++ (void)watch_update(w, NULL, -1); ++} ++ ++static void watch_start(struct cdi_cache *c) ++{ ++ pthread_t thread = 0; ++ int ret = 0; ++ ++ ret = pthread_create(&thread, NULL, watch_thread_func, c); ++ if (ret != 0) { ++ ERROR("Cdi watch thread create failed"); ++ return; ++ } ++} ++ ++static void watch_stop(struct cdi_watch *w) ++{ ++ if (w == NULL) { ++ return; ++ } ++ ++ if (w->watcher_fd >= 0) { ++ close(w->watcher_fd); ++ w->watcher_fd = -1; ++ } ++ map_free(w->tracked); ++ w->tracked = NULL; ++ map_free(w->wd_dirs); ++ w->wd_dirs = NULL; ++} ++ ++// wait_events wait until inotify ++static int wait_events(int watcher_fd) ++{ ++ fd_set rfds; ++ FD_ZERO(&rfds); ++ FD_SET(watcher_fd, &rfds); ++ return select(FD_SETSIZE, &rfds, NULL, NULL, NULL); ++} ++ ++#define CDI_WATCH_EVENTS (IN_MOVED_TO | IN_MOVED_FROM | IN_DELETE | IN_MODIFY | IN_MOVE_SELF | IN_DELETE_SELF) ++ ++static int process_cdi_events(int watcher_fd, struct cdi_cache *c) ++{ ++ ssize_t events_length = 0; ++ ssize_t events_index = 0; ++ struct inotify_event *cdi_event = NULL; ++ char buffer[MAXLINE] __attribute__((aligned(__alignof__(struct inotify_event)))) = { 0 }; ++ int update_cnt = 0; ++ __isula_auto_free char *event_dir = NULL; ++ ++ events_length = util_read_nointr(watcher_fd, buffer, sizeof(buffer)); ++ if (events_length <= 0) { ++ ERROR("Failed to wait events"); ++ return -1; ++ } ++ ++ (void)pthread_mutex_lock(&c->mutex); ++ ++ while (events_index < events_length) { ++ cdi_event = (struct inotify_event *)(&buffer[events_index]); ++ ssize_t event_size = (ssize_t)(cdi_event->len) + (ssize_t)offsetof(struct inotify_event, name); ++ if (event_size == 0 || event_size > (events_length - events_index)) { ++ break; ++ } ++ events_index += event_size; ++ ++ /* ++ * file: ++ * Rename: mask == IN_MOVED_TO | IN_MOVED_FROM ++ * Remove: mask == IN_MOVED_FROM || mask == IN_DELETE ++ * Write: mask == IN_MODIFY ++ * dir: ++ * Remove: mask == IN_MOVE_SELF || mask == IN_DELETE_SELF ++ */ ++ if ((cdi_event->mask & CDI_WATCH_EVENTS) == 0) { ++ continue; ++ } ++ DEBUG("Cdi spec file %s is changed", cdi_event->name); ++ if (cdi_event->mask == IN_MODIFY) { ++ if (!util_has_suffix(cdi_event->name, ".json")) { ++ WARN("Invalid spec %s ext", cdi_event->name); ++ continue; ++ } ++ } ++ event_dir = util_strdup_s(map_search(c->watch->wd_dirs, &(cdi_event->wd))); ++ if (!(cdi_event->mask == IN_DELETE_SELF || cdi_event->mask == IN_MOVE_SELF)) { ++ free(event_dir); ++ event_dir = NULL; ++ } ++ watch_update(c->watch, event_dir, cdi_event->wd); ++ update_cnt++; ++ } ++ if (update_cnt > 0) { ++ (void)refresh(c); ++ } ++ ++ (void)pthread_mutex_unlock(&c->mutex); ++ return 0; ++} ++ ++// Watch Spec directory changes, triggering a refresh if necessary. ++static void *watch_thread_func(void *arg) ++{ ++ struct cdi_cache *c = (struct cdi_cache *)arg; ++ int errcode = 0; ++ int watcher_fd = -1; ++ ++ errcode = pthread_detach(pthread_self()); ++ if (errcode != 0) { ++ errno = errcode; ++ SYSERROR("Detach thread failed"); ++ return NULL; ++ } ++ ++ prctl(PR_SET_NAME, "cdi-watcher"); ++ ++ watcher_fd = c->watch->watcher_fd; ++ if (watcher_fd < 0) { ++ ERROR("Invalid inotify fd"); ++ return NULL; ++ } ++ ++ for (;;) { ++ if (wait_events(watcher_fd) < 0) { ++ ERROR("Failed to wait events"); ++ break; ++ } ++ if (process_cdi_events(watcher_fd, c) != 0) { ++ break; ++ } ++ } ++ return NULL; ++} ++ ++static void update_remove_watch_dir(struct cdi_watch *w, const char *dir, int wd) ++{ ++ bool tmp_value = false; ++ if (wd >= 0) { ++ (void)inotify_rm_watch(w->watcher_fd, wd); ++ if ((map_search(w->wd_dirs, &wd) != NULL) && ++ !map_remove(w->wd_dirs, &wd)) { ++ ERROR("Failed to remove watch fd of %s", dir); ++ } ++ } ++ if (!map_replace(w->tracked, (void *)dir, (void *)&tmp_value)) { ++ ERROR("Failed to insert tracked by dir %s", dir); ++ } ++} ++ ++static void update_add_watch_dir(struct cdi_watch *w, const char *dir, bool *update) ++{ ++ int wd = -1; ++ bool tmp_value = true; ++ __isula_auto_free char *error = NULL; ++ ++ wd = inotify_add_watch(w->watcher_fd, dir, CDI_WATCH_EVENTS); ++ if (wd < 0) { ++ if (errno == ENOENT) { ++ SYSINFO("Watch device dir %s", dir); ++ } else { ++ SYSERROR("Failed to watch device dir %s", dir); ++ } ++ return; ++ } else { ++ DEBUG("Watching %s for device disovery", dir); ++ tmp_value = true; ++ if (!map_replace(w->tracked, (void *)dir, (void *)&tmp_value)) { ++ ERROR("Failed to insert tracked by dir %s", dir); ++ goto error_out; ++ } ++ if (!map_replace(w->wd_dirs, (void *)&wd, (void *)dir)) { ++ ERROR("Failed to insert dir %s by wd", dir); ++ goto error_out; ++ } ++ *update = true; ++ } ++ return; ++ ++error_out: ++ update_remove_watch_dir(w, dir, wd); ++} ++ ++static bool watch_update(struct cdi_watch *w, const char *removed, int wd) ++{ ++ const char *dir = NULL; ++ bool *ok = NULL; ++ bool update = false; ++ map_itor *itor = NULL; ++ __isula_auto_free char *error = NULL; ++ ++ itor = map_itor_new(w->tracked); ++ if (itor == NULL) { ++ ERROR("Out of memory, create new map itor failed"); ++ return false; ++ } ++ for (; map_itor_valid(itor); map_itor_next(itor)) { ++ dir = map_itor_key(itor); ++ ok = map_itor_value(itor); ++ if (ok == NULL || *ok) { ++ continue; ++ } ++ update_add_watch_dir(w, dir, &update); ++ } ++ ++ if (removed != NULL) { ++ update_remove_watch_dir(w, removed, wd); ++ WARN("Directory removed: %s", removed); ++ update = true; ++ } ++ ++ map_itor_free(itor); ++ return update; ++} +diff --git a/src/daemon/modules/device/cdi/cdi_cache.h b/src/daemon/modules/device/cdi/cdi_cache.h +index 34c27471..da315de2 100644 +--- a/src/daemon/modules/device/cdi/cdi_cache.h ++++ b/src/daemon/modules/device/cdi/cdi_cache.h +@@ -39,9 +39,6 @@ struct cdi_cache_ops { + // Refresher + int (*configure)(struct cdi_cache *c, string_array *spec_dirs); + int (*refresh)(struct cdi_cache *c); +- map_t *(*get_errors)(struct cdi_cache *c); +- string_array *(*get_spec_directories)(struct cdi_cache *c); +- map_t *(*get_spec_dir_errors)(struct cdi_cache *c); + }; + + struct cdi_watch { +@@ -54,11 +51,9 @@ struct cdi_watch { + struct cdi_cache { + pthread_mutex_t mutex; + string_array *spec_dirs; // cdi-spec-dirs will scan for CDI Spec files +- map_t *specs; // MAP_STR_PTR specs[vendor] = cdi_cache_spec** ++ map_t *specs; // MAP_STR_PTR specs[vendor] = common_array of cdi_cache_spec* + map_t *devices; // MAP_STR_PTR devices[cdi_device.name] = cdi_cache_device* +- map_t *errors; // MAP_STR_PTR errors[cdi_cache_spec.path] = string_array *errors +- map_t *dir_errors; // MAP_STR_STR dir_errors[spec_dirs[i]] = error +- ++ bool refresh_error_flag; + bool auto_refresh; + struct cdi_watch *watch; + }; +-- +2.34.1 + diff --git a/0065-change-default-subscribe-timeout-to-5min.patch b/0065-change-default-subscribe-timeout-to-5min.patch new file mode 100644 index 0000000..7e4d067 --- /dev/null +++ b/0065-change-default-subscribe-timeout-to-5min.patch @@ -0,0 +1,76 @@ +From 5c89c23f5e0de06a17a9263114430674221a1ee0 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Wed, 17 Apr 2024 06:59:08 +0000 +Subject: [PATCH 65/69] change default subscribe timeout to 5min + +Signed-off-by: jikai <jikai11@huawei.com> +--- + src/daemon/mailbox/message_queue.c | 4 ++-- + src/utils/cutils/blocking_queue.c | 13 +++++++------ + src/utils/cutils/blocking_queue.h | 2 +- + 3 files changed, 10 insertions(+), 9 deletions(-) + +diff --git a/src/daemon/mailbox/message_queue.c b/src/daemon/mailbox/message_queue.c +index 7e53301e..699ea0bc 100644 +--- a/src/daemon/mailbox/message_queue.c ++++ b/src/daemon/mailbox/message_queue.c +@@ -20,8 +20,8 @@ + + #include "utils.h" + +-// default set subscriber timeout to 1000ms, maybe could be configured later +-const int64_t subscribe_timeout = 1000; ++// default set subscriber timeout to 300s, maybe could be configured later ++const int64_t subscribe_timeout = 300; + + static void message_queue_subscriber_free(void *key, void *val) + { +diff --git a/src/utils/cutils/blocking_queue.c b/src/utils/cutils/blocking_queue.c +index 7c9c5f50..9bdb2ca3 100644 +--- a/src/utils/cutils/blocking_queue.c ++++ b/src/utils/cutils/blocking_queue.c +@@ -55,12 +55,11 @@ blocking_queue *blocking_queue_create(int64_t timeout, void (*release)(void *)) + queue->release = release; + + if (timeout >= 0) { +- queue->timeout.tv_sec = timeout / (Time_Second / Time_Milli); +- queue->timeout.tv_nsec = (timeout % (Time_Second / Time_Milli) ) * Time_Milli; ++ queue->timeout = timeout; + } else { +- queue->timeout.tv_sec = -1; ++ queue->timeout = -1; + } +- ++ + return isula_transfer_ptr(queue); + } + +@@ -112,8 +111,10 @@ int blocking_queue_pop(blocking_queue *queue, void **data) { + lock = &queue->lock; + + while (queue->head->next == NULL) { +- if (queue->timeout.tv_sec >= 0) { +- int ret = pthread_cond_timedwait(&queue->not_empty, &queue->lock, &queue->timeout); ++ if (queue->timeout >= 0) { ++ struct timespec timeout = { 0 }; ++ timeout.tv_sec = queue->timeout + time(NULL); ++ int ret = pthread_cond_timedwait(&queue->not_empty, &queue->lock, &timeout); + if (ret != 0) { + if (ret != ETIMEDOUT) { + ERROR("Failed to wait cond"); +diff --git a/src/utils/cutils/blocking_queue.h b/src/utils/cutils/blocking_queue.h +index 1c52a9d3..257779c3 100644 +--- a/src/utils/cutils/blocking_queue.h ++++ b/src/utils/cutils/blocking_queue.h +@@ -37,7 +37,7 @@ typedef struct blocking_queue { + blocking_node *head; + blocking_node *tail; + pthread_mutex_t lock; +- struct timespec timeout; ++ int64_t timeout; + pthread_cond_t not_empty; + void (*release)(void *); + } blocking_queue; +-- +2.34.1 + diff --git a/0066-cdi-support-modules-version-spec-spec_dirs-device.patch b/0066-cdi-support-modules-version-spec-spec_dirs-device.patch new file mode 100644 index 0000000..6aff31b --- /dev/null +++ b/0066-cdi-support-modules-version-spec-spec_dirs-device.patch @@ -0,0 +1,1254 @@ +From 684f0aa648061a13e777153e297a7064ba33ea17 Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Thu, 11 Apr 2024 11:00:44 +0800 +Subject: [PATCH 66/69] cdi:support modules version/spec/spec_dirs/device + +--- + .../modules/device/cdi/behavior/cdi_device.c | 100 +++++++- + .../modules/device/cdi/behavior/cdi_device.h | 12 +- + .../modules/device/cdi/behavior/cdi_spec.c | 220 ++++++++++++++++-- + .../modules/device/cdi/behavior/cdi_spec.h | 16 +- + .../device/cdi/behavior/cdi_spec_dirs.c | 80 +++++++ + .../device/cdi/behavior/cdi_spec_dirs.h | 2 +- + .../modules/device/cdi/behavior/cdi_version.c | 173 ++++++++++++-- + .../modules/device/cdi/behavior/cdi_version.h | 1 - + src/daemon/modules/device/cdi/cdi_cache.c | 37 +-- + src/daemon/modules/device/cdi/cdi_cache.h | 1 + + .../network/cni_operator/libcni/libcni_api.c | 98 +------- + src/utils/cutils/utils_version.c | 147 ++++++++++++ + src/utils/cutils/utils_version.h | 36 +++ + 13 files changed, 748 insertions(+), 175 deletions(-) + create mode 100644 src/utils/cutils/utils_version.c + create mode 100644 src/utils/cutils/utils_version.h + +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_device.c b/src/daemon/modules/device/cdi/behavior/cdi_device.c +index 0fef8f42..aec3d7c0 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_device.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_device.c +@@ -14,27 +14,109 @@ + ******************************************************************************/ + #include "cdi_device.h" + ++#include <isula_libutils/log.h> ++#include <isula_libutils/auto_cleanup.h> ++ ++#include "error.h" ++#include "cdi_parser.h" ++#include "cdi_spec.h" ++ ++static int cdi_device_validate(struct cdi_cache_device *d); ++ + void free_cdi_cache_device(struct cdi_cache_device *d) + { +- (void)d; ++ if (d == NULL) { ++ return; ++ } ++ ++ /* ++ * free_cdi_cache_device should not be recursively free raw_device. ++ * Otherwise, the function conflicts with the raw_spec free raw_device ++ * when cdi_cache_spec free raw_spec, triggering double free. ++ */ ++ d->raw_device = NULL; ++ ++ /* ++ * free_cdi_cache_device should not be recursively free cache_spec. ++ * Otherwise, the function conflicts with the cache free specs, ++ * triggering double free. ++ */ ++ d->cache_spec = NULL; ++ ++ free(d); ++} ++ ++struct cdi_cache_device *cdi_device_new_device(struct cdi_cache_spec *spec, cdi_device *d) ++{ ++ struct cdi_cache_device *dev = NULL; ++ ++ if (spec == NULL || d == NULL) { ++ ERROR("Invalid params"); ++ return NULL; ++ } ++ ++ dev = util_common_calloc_s(sizeof(*dev)); ++ if (dev == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ dev->raw_device = d; ++ dev->cache_spec = spec; ++ ++ if (cdi_device_validate(dev) != 0) { ++ free_cdi_cache_device(dev); ++ return NULL; ++ } ++ return dev; + } + +-struct cdi_cache_device *cdi_device_new_device(struct cdi_cache_spec *spec, cdi_device *d, char **error) ++const struct cdi_cache_spec *cdi_device_get_spec(const struct cdi_cache_device *d) + { +- return NULL; ++ if (d == NULL) { ++ ERROR("Invalid params"); ++ return NULL; ++ } ++ return d->cache_spec; + } + +-struct cdi_cache_spec *cdi_device_get_spec(struct cdi_cache_device *d) ++char *cdi_device_get_qualified_name(const struct cdi_cache_device *d) + { +- return NULL; ++ if (d == NULL || d->raw_device == NULL) { ++ ERROR("Invalid params"); ++ return NULL; ++ } ++ return cdi_parser_qualified_name(cdi_spec_get_vendor(d->cache_spec), ++ cdi_spec_get_class(d->cache_spec), d->raw_device->name); + } + +-char *cdi_device_get_qualified_name(struct cdi_cache_device *d) ++cdi_container_edits *cdi_device_get_edits(const struct cdi_cache_device *d) + { +- return NULL; ++ if (d == NULL || d->raw_device == NULL) { ++ ERROR("Invalid params"); ++ return NULL; ++ } ++ return d->raw_device->container_edits; + } + +-cdi_container_edits *cdi_device_get_edits(struct cdi_cache_device *d) ++static int cdi_device_validate(struct cdi_cache_device *d) + { +- return NULL; ++ cdi_container_edits *edits = NULL; ++ ++ if (cdi_parser_validate_device_name(d->raw_device->name) != 0) { ++ ERROR("Failed to validate device name"); ++ return -1; ++ } ++ ++ // ignore validate annotations ++ ++ edits = cdi_device_get_edits(d); ++ if (cdi_container_edits_is_empty(edits)) { ++ ERROR("Invalid device, empty device edits"); ++ return -1; ++ } ++ if (cdi_container_edits_validate(edits) != 0) { ++ ERROR("Invalid device %s", d->raw_device->name); ++ return -1; ++ } ++ return 0; + } +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_device.h b/src/daemon/modules/device/cdi/behavior/cdi_device.h +index 5d63a576..9b0a5eab 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_device.h ++++ b/src/daemon/modules/device/cdi/behavior/cdi_device.h +@@ -28,16 +28,16 @@ extern "C" { + struct cdi_cache_spec; + + struct cdi_cache_device { +- cdi_device *raw_device; +- struct cdi_cache_spec *cache_spec; ++ const cdi_device *raw_device; ++ const struct cdi_cache_spec *cache_spec; + }; + + void free_cdi_cache_device(struct cdi_cache_device *d); + +-struct cdi_cache_device *cdi_device_new_device(struct cdi_cache_spec *spec, cdi_device *d, char **error); +-struct cdi_cache_spec *cdi_device_get_spec(struct cdi_cache_device *d); +-char *cdi_device_get_qualified_name(struct cdi_cache_device *d); +-cdi_container_edits *cdi_device_get_edits(struct cdi_cache_device *d); ++struct cdi_cache_device *cdi_device_new_device(struct cdi_cache_spec *spec, cdi_device *d); ++const struct cdi_cache_spec *cdi_device_get_spec(const struct cdi_cache_device *d); ++char *cdi_device_get_qualified_name(const struct cdi_cache_device *d); ++cdi_container_edits *cdi_device_get_edits(const struct cdi_cache_device *d); + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec.c b/src/daemon/modules/device/cdi/behavior/cdi_spec.c +index f79b5a44..235b1863 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_spec.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec.c +@@ -14,47 +14,235 @@ + ******************************************************************************/ + #include "cdi_spec.h" + ++#include <stdlib.h> ++#include <isula_libutils/log.h> ++#include <isula_libutils/auto_cleanup.h> ++ ++#include "utils.h" ++#include "utils_version.h" ++#include "error.h" ++#include "path.h" ++#include "cdi_version.h" ++#include "cdi_parser.h" ++#include "cdi_device.h" ++ ++static int cdi_spec_init(struct cdi_cache_spec *s); ++ + void free_cdi_cache_spec(struct cdi_cache_spec *s) + { +- (void)s; ++ if (s == NULL) { ++ return; ++ } ++ ++ free_cdi_spec(s->raw_spec); ++ s->raw_spec = NULL; ++ free(s->vendor); ++ s->vendor = NULL; ++ free(s->class); ++ s->class = NULL; ++ free(s->path); ++ s->path = NULL; ++ map_free(s->devices); ++ s->devices = NULL; ++ ++ free(s); + } + +-struct cdi_cache_spec *cdi_spec_read_spec(const char *path, int priority, char **error) ++struct cdi_cache_spec *cdi_spec_read_spec(const char *path, int priority) + { +- return NULL; ++ cdi_spec *raw_spec = NULL; ++ __isula_auto_free parser_error err = NULL; ++ char cleanpath[PATH_MAX] = { 0 }; ++ ++ if (util_clean_path(path, cleanpath, sizeof(cleanpath)) == NULL) { ++ ERROR("Failed to get clean path %s", path); ++ return NULL; ++ } ++ ++ raw_spec = cdi_spec_parse_file(cleanpath, NULL, &err); ++ if (raw_spec == NULL) { ++ ERROR("Failed to read CDI Spec %s: %s", cleanpath, err); ++ return NULL; ++ } ++ DEBUG("Read cdi spec %s", cleanpath); ++ ++ return cdi_spec_new_spec(raw_spec, cleanpath, priority); + } + +-struct cdi_cache_spec *cdi_spec_new_spec(cdi_spec *raw, const char *path, int priority, char **error) ++struct cdi_cache_spec *cdi_spec_new_spec(cdi_spec *raw, const char *path, int priority) + { ++ struct cdi_cache_spec *spec = NULL; ++ __isula_auto_free char *checked_path = NULL; ++ ++ if (raw == NULL) { ++ ERROR("Invalid param"); ++ return NULL; ++ } ++ ++ if (!util_has_suffix(path, ".json")) { ++ checked_path = util_string_append(path, CDI_DEFAULT_SPEC_EXT); ++ if (checked_path == NULL) { ++ ERROR("Failed to append %s to path %s", CDI_DEFAULT_SPEC_EXT, path); ++ return NULL; ++ } ++ } else { ++ checked_path = util_strdup_s(path); ++ } ++ spec = util_common_calloc_s(sizeof(*spec)); ++ if (spec == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ spec->raw_spec = raw; ++ spec->path = checked_path; ++ checked_path = NULL; ++ spec->priority = priority; ++ ++ if (cdi_parser_parse_qualifier(raw->kind, &spec->vendor, &spec->class) != 0) { ++ ERROR("Failed to parse kind %s", raw->kind); ++ goto error_out; ++ } ++ if (cdi_spec_init(spec) != 0) { ++ ERROR("Invalid CDI Spec"); ++ goto error_out; ++ } ++ ++ return spec; ++ ++error_out: ++ free_cdi_cache_spec(spec); + return NULL; + } + +-const char *cdi_spec_get_vendor(struct cdi_cache_spec *s) ++const char *cdi_spec_get_vendor(const struct cdi_cache_spec *s) + { +- return NULL; ++ if (s == NULL) { ++ ERROR("Invalid params"); ++ return NULL; ++ } ++ return s->vendor; + } + +-const char *cdi_spec_get_class(struct cdi_cache_spec *s) ++const char *cdi_spec_get_class(const struct cdi_cache_spec *s) + { +- return NULL; ++ if (s == NULL) { ++ ERROR("Invalid params"); ++ return NULL; ++ } ++ return s->class; + } + +-struct cdi_cache_device *cdi_spec_get_cache_device(struct cdi_cache_spec *s, const char *name) ++struct cdi_cache_device *cdi_spec_get_cache_device(const struct cdi_cache_spec *s, const char *name) + { +- return NULL; ++ if (s == NULL) { ++ ERROR("Invalid params"); ++ return NULL; ++ } ++ return map_search(s->devices, (void *)name);; + } + +-const char *cdi_spec_get_path(struct cdi_cache_spec *s) ++const char *cdi_spec_get_path(const struct cdi_cache_spec *s) + { +- return NULL; ++ if (s == NULL) { ++ ERROR("Invalid params"); ++ return NULL; ++ } ++ return s->path; + } + +-int cdi_spec_get_priority(struct cdi_cache_spec *s) ++int cdi_spec_get_priority(const struct cdi_cache_spec *s) + { +- return 0; ++ if (s == NULL) { ++ ERROR("Invalid params"); ++ return -1; ++ } ++ return s->priority; + } + +-cdi_container_edits *cdi_spec_get_edits(struct cdi_cache_spec *s) ++cdi_container_edits *cdi_spec_get_edits(const struct cdi_cache_spec *s) + { +- return NULL; ++ if (s == NULL || s->raw_spec == NULL) { ++ ERROR("Invalid params"); ++ return NULL; ++ } ++ return s->raw_spec->container_edits; ++} ++ ++static void map_cdi_cache_device_kvfree(void *key, void *value) ++{ ++ free(key); ++ free_cdi_cache_device((struct cdi_cache_device *)value); ++} ++ ++static int cdi_spec_init(struct cdi_cache_spec *s) ++{ ++ const char *min_version = NULL; ++ __isula_auto_free char *spec_version = NULL; ++ cdi_container_edits *edits = NULL; ++ struct cdi_cache_device *dev = NULL; ++ cdi_device *d = NULL; ++ size_t i; ++ bool version_result = true; ++ ++ if (!cdi_is_valid_version(s->raw_spec->cdi_version)) { ++ ERROR("Failed to validate cdi spec version: %s", s->raw_spec->cdi_version); ++ return -1; ++ } ++ ++ min_version = cdi_minimum_required_version(s->raw_spec); ++ if (min_version == NULL) { ++ ERROR("Could not determine minimum required version"); ++ return -1; ++ } ++ if (util_version_greater_than(min_version, s->raw_spec->cdi_version, &version_result) != 0) { ++ ERROR("Failed to compare version %s and %s", min_version, s->raw_spec->cdi_version); ++ return -1; ++ } ++ if (version_result) { ++ ERROR("The spec version must be at least v%s", min_version); ++ return -1; ++ } ++ ++ if (cdi_parser_validate_vendor_name(s->vendor) != 0) { ++ return -1; ++ } ++ if (cdi_parser_validate_class_name(s->class) != 0) { ++ return -1; ++ } ++ ++ // ignore validate annotations ++ ++ edits = cdi_spec_get_edits(s); ++ if (cdi_container_edits_validate(edits) != 0) { ++ return -1; ++ } ++ ++ s->devices = map_new(MAP_STR_PTR, MAP_DEFAULT_CMP_FUNC, map_cdi_cache_device_kvfree); ++ if (s->devices == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ for (i = 0; i < s->raw_spec->devices_len; i++) { ++ d = s->raw_spec->devices[i]; ++ dev = cdi_device_new_device(s, d); ++ if (dev == NULL) { ++ ERROR("Could not determine minimum required version"); ++ goto error_out; ++ } ++ if (map_search(s->devices, (void *)d->name) != NULL) { ++ ERROR("Invalid spec, multiple device %s", d->name); ++ goto error_out; ++ } ++ if (!map_insert(s->devices, (void *)d->name, dev)) { ++ ERROR("Failed to insert device %s", d->name); ++ goto error_out; ++ } ++ } ++ ++ return 0; ++ ++error_out: ++ map_free(s->devices); ++ s->devices = NULL; ++ return -1; + } +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec.h b/src/daemon/modules/device/cdi/behavior/cdi_spec.h +index 87248041..ca7b2cfa 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_spec.h ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec.h +@@ -40,14 +40,14 @@ struct cdi_cache_spec { + + void free_cdi_cache_spec(struct cdi_cache_spec *s); + +-struct cdi_cache_spec *cdi_spec_read_spec(const char *path, int priority, char **error); +-struct cdi_cache_spec *cdi_spec_new_spec(cdi_spec *raw, const char *path, int priority, char **error); +-const char *cdi_spec_get_vendor(struct cdi_cache_spec *s); +-const char *cdi_spec_get_class(struct cdi_cache_spec *s); +-struct cdi_cache_device *cdi_spec_get_cache_device(struct cdi_cache_spec *s, const char *name); +-const char *cdi_spec_get_path(struct cdi_cache_spec *s); +-int cdi_spec_get_priority(struct cdi_cache_spec *s); +-cdi_container_edits *cdi_spec_get_edits(struct cdi_cache_spec *s); ++struct cdi_cache_spec *cdi_spec_read_spec(const char *path, int priority); ++struct cdi_cache_spec *cdi_spec_new_spec(cdi_spec *raw, const char *path, int priority); ++const char *cdi_spec_get_vendor(const struct cdi_cache_spec *s); ++const char *cdi_spec_get_class(const struct cdi_cache_spec *s); ++struct cdi_cache_device *cdi_spec_get_cache_device(const struct cdi_cache_spec *s, const char *name); ++const char *cdi_spec_get_path(const struct cdi_cache_spec *s); ++int cdi_spec_get_priority(const struct cdi_cache_spec *s); ++cdi_container_edits *cdi_spec_get_edits(const struct cdi_cache_spec *s); + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c +index e340abc0..cafb52b8 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c +@@ -14,6 +14,17 @@ + ******************************************************************************/ + #include "cdi_spec_dirs.h" + ++#include <sys/stat.h> ++#include <isula_libutils/log.h> ++#include <isula_libutils/auto_cleanup.h> ++ ++#include "utils.h" ++#include "path.h" ++#include "error.h" ++#include "utils_file.h" ++#include "utils_array.h" ++#include "cdi_spec.h" ++ + #define DEFAULT_SPEC_DIRS_LEN 2 + static char *default_spec_dirs_items[DEFAULT_SPEC_DIRS_LEN] = {CDI_DEFAULT_STATIC_DIR, CDI_DEFAULT_DYNAMIC_DIR}; + +@@ -23,7 +34,76 @@ string_array g_default_spec_dirs = { + .cap = DEFAULT_SPEC_DIRS_LEN, + }; + ++struct scan_spec_dir_cb_args { ++ struct cdi_scan_fn_maps *scan_fn_maps; ++ cdi_scan_spec_func scan_fn; ++ int priority; ++}; ++ ++static bool scan_spec_dir_cb(const char *dir, const struct dirent *pdirent, void *context) ++{ ++ struct scan_spec_dir_cb_args *args = (struct scan_spec_dir_cb_args *)context; ++ struct cdi_scan_fn_maps *scan_fn_maps = args->scan_fn_maps; ++ cdi_scan_spec_func scan_fn = args->scan_fn; ++ int priority = args->priority; ++ struct stat st = { 0 }; ++ __isula_auto_free char *file_path = NULL; ++ struct cdi_cache_spec *cache_spec = NULL; ++ ++ file_path = util_path_join(dir, pdirent->d_name); ++ if (file_path == NULL) { ++ ERROR("Failed to get path %s/%s", dir, pdirent->d_name); ++ goto error_out; ++ } ++ ++ if (lstat(file_path, &st) != 0) { ++ ERROR("Failed to lstat %s", file_path); ++ goto error_out; ++ } ++ if (S_ISDIR(st.st_mode)) { ++ DEBUG("Skip dir %s", file_path); ++ return true; ++ } ++ ++ if (!util_has_suffix(file_path, ".json")) { ++ DEBUG("Skip file %s", file_path); ++ return true; ++ } ++ ++ cache_spec = cdi_spec_read_spec(file_path, priority); ++ if (cache_spec == NULL) { ++ ERROR("Failed to read spec %s", file_path); ++ goto error_out; ++ } ++ scan_fn(scan_fn_maps, file_path, priority, cache_spec); ++ return true; ++ ++error_out: ++ *(scan_fn_maps->refresh_error_flag) = true; ++ return true; ++} ++ + int cdi_scan_spec_dirs(string_array *dirs, struct cdi_scan_fn_maps *scan_fn_maps, cdi_scan_spec_func scan_fn) + { ++ size_t i; ++ int nret = 0; ++ ++ for (i = 0; i < dirs->len; i++) { ++ struct scan_spec_dir_cb_args args = { ++ .scan_fn_maps = scan_fn_maps, ++ .scan_fn = scan_fn, ++ .priority = i, ++ }; ++ if (!util_dir_exists(dirs->items[i])) { ++ WARN("Cdi dir %s not exists", dirs->items[i]); ++ continue; ++ } ++ nret = util_scan_subdirs(dirs->items[i], scan_spec_dir_cb, &args); ++ if (nret != 0) { ++ ERROR("Failed to scan dir %s", dirs->items[i]); ++ return -1; ++ } ++ } ++ + return 0; + } +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h +index eedcabad..b17a0cd8 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h +@@ -35,7 +35,7 @@ struct cdi_scan_fn_maps { + bool *refresh_error_flag; + }; + typedef void(*cdi_scan_spec_func)(struct cdi_scan_fn_maps *scan_fn_maps, const char *path, int priority, +- struct cdi_cache_spec *spec, char *error); ++ struct cdi_cache_spec *spec); + + int cdi_scan_spec_dirs(string_array *dirs, struct cdi_scan_fn_maps *scan_fn_maps, cdi_scan_spec_func scan_fn); + +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_version.c b/src/daemon/modules/device/cdi/behavior/cdi_version.c +index 3e87c111..882a965e 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_version.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_version.c +@@ -14,27 +14,174 @@ + ******************************************************************************/ + #include "cdi_version.h" + +-#define CDI_V_CURRENT_VERSION "v"##CDI_CURRENT_VERSION +- +-#define CDI_V010 "v0.1.0" +-#define CDI_V020 "v0.2.0" +-#define CDI_V030 "v0.3.0" +-#define CDI_V040 "v0.4.0" +-#define CDI_V050 "v0.5.0" +-#define CDI_V060 "v0.6.0" ++#include <ctype.h> ++#include <isula_libutils/log.h> ++#include <isula_libutils/auto_cleanup.h> ++ ++#include "error.h" ++#include "utils_version.h" ++#include "utils_string.h" ++#include "cdi_container_edits.h" ++#include "cdi_parser.h" ++ ++#define CDI_V010 "0.1.0" ++#define CDI_V020 "0.2.0" ++#define CDI_V030 "0.3.0" ++#define CDI_V040 "0.4.0" ++#define CDI_V050 "0.5.0" ++#define CDI_V060 "0.6.0" + #define CDI_V_EARLIEST CDI_V030 + +-const char *cdi_minimum_required_version(cdi_spec *spec) ++typedef bool (*required_version_cb)(cdi_spec *spec); ++ ++struct required_version_map { ++ char *version; ++ required_version_cb cb; ++}; ++ ++static bool requires_v060(cdi_spec *spec) ++{ ++ size_t i; ++ int ret = 0; ++ __isula_auto_free char *vendor = NULL; ++ __isula_auto_free char *class = NULL; ++ ++ // The 0.6.0 spec allows annotations to be specified at a spec level ++ if (spec->annotations != NULL) { ++ return true; ++ } ++ ++ // The 0.6.0 spec allows annotations to be specified at a device level ++ if (spec->devices != NULL) { ++ for (i = 0; i < spec->devices_len; i++) { ++ if (spec->devices[i]->annotations != NULL) { ++ return true; ++ } ++ } ++ } ++ ++ // The 0.6.0 spec allows dots "." in Kind name label (class) ++ ret = cdi_parser_parse_qualifier(spec->kind, &vendor, &class); ++ if (ret == 0 && vendor != NULL) { ++ if (util_strings_count(class, '.') > 0) { ++ return true; ++ } ++ } ++ return false; ++} ++ ++static bool check_host_path(cdi_container_edits *e) + { +- return NULL; ++ size_t i; ++ ++ if (e == NULL) { ++ return false; ++ } ++ for (i = 0; i < e->device_nodes_len; i++) { ++ // The HostPath field was added in 0.5.0 ++ if (e->device_nodes[i]->host_path != NULL) { ++ return true; ++ } ++ } ++ return false; + } + +-bool cdi_is_greater_than_version(const char *v, const char *o) ++static bool requires_v050(cdi_spec *spec) + { +- return true; ++ size_t i; ++ ++ for (i = 0; i < spec->devices_len; i++) { ++ // The 0.5.0 spec allowed device names to start with a digit instead of requiring a letter ++ if (spec->devices[i]->name != NULL && strlen(spec->devices[i]->name) > 0 && ++ !isalpha(spec->devices[i]->name[0])) { ++ return true; ++ } ++ if (check_host_path(spec->devices[i]->container_edits)) { ++ return true; ++ } ++ } ++ ++ return check_host_path(spec->container_edits); ++} ++ ++static bool check_mount_type(cdi_container_edits *e) ++{ ++ size_t i; ++ ++ if (e == NULL) { ++ return false; ++ } ++ for (i = 0; i < e->mounts_len; i++) { ++ // The Type field was added in 0.4.0 ++ if (e->mounts[i]->type != NULL) { ++ return true; ++ } ++ } ++ return false; ++} ++ ++static bool requires_v040(cdi_spec *spec) ++{ ++ size_t i; ++ ++ for (i = 0; i < spec->devices_len; i++) { ++ if (check_mount_type(spec->devices[i]->container_edits)) { ++ return true; ++ } ++ } ++ ++ return check_mount_type(spec->container_edits); ++} ++ ++#define VALID_SPEC_VERSIONS_LEN 6 ++static struct required_version_map g_valid_spec_versions[VALID_SPEC_VERSIONS_LEN] = { ++ {CDI_V010, NULL}, ++ {CDI_V020, NULL}, ++ {CDI_V030, NULL}, ++ {CDI_V040, requires_v060}, ++ {CDI_V050, requires_v050}, ++ {CDI_V060, requires_v040} ++}; ++ ++const char *cdi_minimum_required_version(cdi_spec *spec) ++{ ++ const char *min_version = CDI_V_EARLIEST; ++ int i; ++ bool result = true; ++ ++ if (spec == NULL) { ++ return NULL; ++ } ++ ++ for (i = 0; i < VALID_SPEC_VERSIONS_LEN; i++) { ++ if (g_valid_spec_versions[i].cb == NULL) { ++ continue; ++ } ++ if (g_valid_spec_versions[i].cb(spec)) { ++ if (util_version_greater_than(g_valid_spec_versions[i].version, min_version, &result) != 0) { ++ ERROR("Failed to compare version %s and %s", g_valid_spec_versions[i].version, min_version); ++ return NULL; ++ } ++ if (result) { ++ min_version = g_valid_spec_versions[i].version; ++ } ++ } ++ if (strcmp(min_version, CDI_CURRENT_VERSION)) { ++ break; ++ } ++ } ++ ++ return min_version; + } + + bool cdi_is_valid_version(const char *spec_version) + { +- return true; ++ int i; ++ ++ for (i = 0; i < VALID_SPEC_VERSIONS_LEN; i++) { ++ if (strcmp(g_valid_spec_versions[i].version, spec_version) == 0) { ++ return true; ++ } ++ } ++ return false; + } +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_version.h b/src/daemon/modules/device/cdi/behavior/cdi_version.h +index 99b7e692..6bd86340 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_version.h ++++ b/src/daemon/modules/device/cdi/behavior/cdi_version.h +@@ -24,7 +24,6 @@ extern "C" { + #define CDI_CURRENT_VERSION "0.6.0" + + const char *cdi_minimum_required_version(cdi_spec *spec); +-bool cdi_is_greater_than_version(const char *v, const char *o); + bool cdi_is_valid_version(const char *spec_version); + + #ifdef __cplusplus +diff --git a/src/daemon/modules/device/cdi/cdi_cache.c b/src/daemon/modules/device/cdi/cdi_cache.c +index 37767855..e637f7cd 100644 +--- a/src/daemon/modules/device/cdi/cdi_cache.c ++++ b/src/daemon/modules/device/cdi/cdi_cache.c +@@ -206,13 +206,12 @@ static void map_cdi_cache_specs_kvfree(void *key, void *value) + static void map_cdi_cache_device_kvfree(void *key, void *value) + { + free(key); +- free_cdi_cache_device((struct cdi_cache_device *)value); +-} +- +-static void set_refresh_error_flag(bool *refresh_error_flag, const char *error, const char *path) +-{ +- *refresh_error_flag = true; +- ERROR("Cdi refresh error: %s, spec %s", error, path); ++ /* ++ * map_cdi_cache_device_kvfree should not be recursively free cdi_cache_device. ++ * Otherwise, the function conflicts with the cdi_cache_specs free devices, ++ * triggering double free. ++ */ ++ (void)value; + } + + static bool resolve_conflict(struct cdi_scan_fn_maps *scan_fn_maps, const char *name, +@@ -220,8 +219,8 @@ static bool resolve_conflict(struct cdi_scan_fn_maps *scan_fn_maps, const char * + { + map_t *conflicts = scan_fn_maps->conflicts; + bool *refresh_error_flag = scan_fn_maps->refresh_error_flag; +- struct cdi_cache_spec *dev_spec = NULL; +- struct cdi_cache_spec *old_spec = NULL; ++ const struct cdi_cache_spec *dev_spec = NULL; ++ const struct cdi_cache_spec *old_spec = NULL; + int dev_prio; + int old_prio; + bool val = true; +@@ -251,7 +250,7 @@ static bool resolve_conflict(struct cdi_scan_fn_maps *scan_fn_maps, const char * + } + + static void refresh_scan_spec_func(struct cdi_scan_fn_maps *scan_fn_maps, const char *path, +- int priority, struct cdi_cache_spec *spec, char *error) ++ int priority, struct cdi_cache_spec *spec) + { + map_t *specs = scan_fn_maps->specs; + map_t *devices = scan_fn_maps->devices; +@@ -267,12 +266,6 @@ static void refresh_scan_spec_func(struct cdi_scan_fn_maps *scan_fn_maps, const + + if (util_clean_path(path, clean_path, sizeof(clean_path)) == NULL) { + ERROR("Failed to get clean path %s", path); +- format_errorf(&tmp_error, "Failed to get clean path %s", path); +- return; +- } +- if (error != NULL) { +- ERROR("Failed to load CDI Spec %s", error); +- format_errorf(&tmp_error, "Failed to load CDI Spec %s", error); + goto error_out; + } + +@@ -282,18 +275,15 @@ static void refresh_scan_spec_func(struct cdi_scan_fn_maps *scan_fn_maps, const + spec_array = util_common_array_new(1, (free_common_array_item_cb)free_cdi_cache_spec, util_clone_ptr); + if (spec_array == NULL) { + ERROR("Out of memory"); +- tmp_error = util_strdup_s("Out of memory"); + goto error_out; + } + if (!map_insert(specs, (void *)vendor, spec_array)) { + ERROR("Failed to insert spec array to specs"); +- tmp_error = util_strdup_s("Failed to insert spec array to specs"); + goto error_out; + } + } + if (util_append_common_array(spec_array, spec) != 0) { + ERROR("Failed to append spec"); +- tmp_error = util_strdup_s("Failed to append spec"); + goto error_out; + } + spec_array = NULL; +@@ -301,7 +291,6 @@ static void refresh_scan_spec_func(struct cdi_scan_fn_maps *scan_fn_maps, const + itor = map_itor_new(spec->devices); + if (itor == NULL) { + ERROR("Out of memory, create new map itor failed"); +- tmp_error = util_strdup_s("Out of memory, create new map itor failed"); + goto error_out; + } + for (; map_itor_valid(itor); map_itor_next(itor)) { +@@ -315,7 +304,6 @@ static void refresh_scan_spec_func(struct cdi_scan_fn_maps *scan_fn_maps, const + } + if (!map_replace(devices, (void *)qualified, dev)) { + ERROR("Failed to insert device to devices by name %s", qualified); +- format_errorf(&tmp_error, "Failed to insert device to devices by name %s", qualified); + goto error_out; + } + free(qualified); +@@ -324,7 +312,7 @@ static void refresh_scan_spec_func(struct cdi_scan_fn_maps *scan_fn_maps, const + goto out; + + error_out: +- set_refresh_error_flag(refresh_error_flag, tmp_error, path); ++ *refresh_error_flag = true; + out: + map_itor_free(itor); + return; +@@ -333,7 +321,6 @@ out: + static int refresh(struct cdi_cache *c) + { + int ret = 0; +- __isula_auto_free char *error = NULL; + map_t *specs = NULL; + map_t *devices = NULL; + map_t *conflicts = NULL; +@@ -559,8 +546,6 @@ error_out: + + static void watch_setup(struct cdi_watch *w, string_array *dirs) + { +- __isula_auto_free char *error = NULL; +- + if (w == NULL || dirs == NULL || dirs->len == 0) { + ERROR("Invalid param"); + return; +@@ -735,7 +720,6 @@ static void update_add_watch_dir(struct cdi_watch *w, const char *dir, bool *upd + { + int wd = -1; + bool tmp_value = true; +- __isula_auto_free char *error = NULL; + + wd = inotify_add_watch(w->watcher_fd, dir, CDI_WATCH_EVENTS); + if (wd < 0) { +@@ -770,7 +754,6 @@ static bool watch_update(struct cdi_watch *w, const char *removed, int wd) + bool *ok = NULL; + bool update = false; + map_itor *itor = NULL; +- __isula_auto_free char *error = NULL; + + itor = map_itor_new(w->tracked); + if (itor == NULL) { +diff --git a/src/daemon/modules/device/cdi/cdi_cache.h b/src/daemon/modules/device/cdi/cdi_cache.h +index da315de2..638e954e 100644 +--- a/src/daemon/modules/device/cdi/cdi_cache.h ++++ b/src/daemon/modules/device/cdi/cdi_cache.h +@@ -52,6 +52,7 @@ struct cdi_cache { + pthread_mutex_t mutex; + string_array *spec_dirs; // cdi-spec-dirs will scan for CDI Spec files + map_t *specs; // MAP_STR_PTR specs[vendor] = common_array of cdi_cache_spec* ++ // This map holding the reference to cdi device, the devices will not released when the map is freed. + map_t *devices; // MAP_STR_PTR devices[cdi_device.name] = cdi_cache_device* + bool refresh_error_flag; + bool auto_refresh; +diff --git a/src/daemon/modules/network/cni_operator/libcni/libcni_api.c b/src/daemon/modules/network/cni_operator/libcni/libcni_api.c +index 068881be..7ba983af 100644 +--- a/src/daemon/modules/network/cni_operator/libcni/libcni_api.c ++++ b/src/daemon/modules/network/cni_operator/libcni/libcni_api.c +@@ -28,6 +28,7 @@ + #include <isula_libutils/auto_cleanup.h> + + #include "utils.h" ++#include "utils_version.h" + #include "utils_network.h" + #include "libcni_cached.h" + #include "libcni_conf.h" +@@ -544,97 +545,6 @@ free_out: + return ret; + } + +-struct parse_version { +- int major; +- int minor; +- int micro; +-}; +- +-static bool do_parse_version(const char **splits, size_t splits_len, struct parse_version *ret) +-{ +- if (util_safe_int(splits[0], &ret->major) != 0) { +- ERROR("failed to convert major version part: %s", splits[0]); +- return false; +- } +- +- if (splits_len >= 2 && util_safe_int(splits[1], &ret->minor) != 0) { +- ERROR("failed to convert minor version part: %s", splits[1]); +- return false; +- } +- +- if (splits_len >= 3 && util_safe_int(splits[2], &ret->micro) != 0) { +- ERROR("failed to convert micro version part: %s", splits[2]); +- return false; +- } +- +- return true; +-} +- +-static bool parse_version_from_str(const char *src_version, struct parse_version *result) +-{ +- char **splits = NULL; +- const size_t max_len = 4; +- size_t tlen = 0; +- bool ret = false; +- +- splits = util_string_split(src_version, '.'); +- if (splits == NULL) { +- ERROR("Split version: \"%s\" failed", src_version); +- return false; +- } +- tlen = util_array_len((const char **)splits); +- if (tlen < 1 || tlen >= max_len) { +- ERROR("Invalid version: \"%s\"", src_version); +- goto out; +- } +- +- ret = do_parse_version((const char **)splits, tlen, result); +- +-out: +- util_free_array(splits); +- return ret; +-} +- +-static bool do_compare_version(const struct parse_version *p_first, const struct parse_version *p_second) +-{ +- bool ret = false; +- +- if (p_first->major > p_second->major) { +- ret = true; +- } else if (p_first->major == p_second->major) { +- if (p_first->minor > p_second->minor) { +- ret = true; +- } else if (p_first->minor == p_second->minor && p_first->micro >= p_second->micro) { +- ret = true; +- } +- } +- +- return ret; +-} +- +-static int version_greater_than_or_equal_to(const char *first, const char *second, bool *result) +-{ +- struct parse_version first_parsed = { 0 }; +- struct parse_version second_parsed = { 0 }; +- +- if (result == NULL) { +- ERROR("Invalid argument"); +- return -1; +- } +- +- if (!parse_version_from_str(first, &first_parsed)) { +- return -1; +- } +- +- if (!parse_version_from_str(second, &second_parsed)) { +- return -1; +- } +- +- *result = do_compare_version(&first_parsed, &second_parsed); +- +- return 0; +-} +- + static inline bool check_add_network_args(const cni_net_conf *net, const struct runtime_conf *rc) + { + return (net == NULL || rc == NULL); +@@ -690,7 +600,7 @@ int cni_add_network_list(const struct cni_network_list_conf *list, const struct + } + + if (*pret != NULL && +- version_greater_than_or_equal_to((*pret)->cniversion, SUPPORT_CACHE_AND_CHECK_VERSION, &greater) != 0) { ++ util_version_greater_than_or_equal_to((*pret)->cniversion, SUPPORT_CACHE_AND_CHECK_VERSION, &greater) != 0) { + return 0; + } + +@@ -741,7 +651,7 @@ int cni_del_network_list(const struct cni_network_list_conf *list, const struct + return -1; + } + +- if (version_greater_than_or_equal_to(list->list->cni_version, SUPPORT_CACHE_AND_CHECK_VERSION, &greater) != 0) { ++ if (util_version_greater_than_or_equal_to(list->list->cni_version, SUPPORT_CACHE_AND_CHECK_VERSION, &greater) != 0) { + return -1; + } + +@@ -807,7 +717,7 @@ int cni_check_network_list(const struct cni_network_list_conf *list, const struc + return -1; + } + +- if (version_greater_than_or_equal_to(list->list->cni_version, SUPPORT_CACHE_AND_CHECK_VERSION, &greater) != 0) { ++ if (util_version_greater_than_or_equal_to(list->list->cni_version, SUPPORT_CACHE_AND_CHECK_VERSION, &greater) != 0) { + return -1; + } + +diff --git a/src/utils/cutils/utils_version.c b/src/utils/cutils/utils_version.c +new file mode 100644 +index 00000000..9dea5b09 +--- /dev/null ++++ b/src/utils/cutils/utils_version.c +@@ -0,0 +1,147 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2022. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-4-7 ++ * Description: provide version functions ++ ********************************************************************************/ ++ ++#define _GNU_SOURCE ++#include "utils_version.h" ++ ++#include <isula_libutils/log.h> ++ ++#include "utils.h" ++#include "utils_string.h" ++ ++struct parse_version { ++ int major; ++ int minor; ++ int micro; ++}; ++ ++static bool do_parse_version(const char **splits, size_t splits_len, struct parse_version *ret) ++{ ++ if (util_safe_int(splits[0], &ret->major) != 0) { ++ ERROR("Failed to convert major version part: %s", splits[0]); ++ return false; ++ } ++ ++ if (splits_len >= 2 && util_safe_int(splits[1], &ret->minor) != 0) { ++ ERROR("Failed to convert minor version part: %s", splits[1]); ++ return false; ++ } ++ ++ if (splits_len >= 3 && util_safe_int(splits[2], &ret->micro) != 0) { ++ ERROR("Failed to convert micro version part: %s", splits[2]); ++ return false; ++ } ++ ++ return true; ++} ++ ++static bool parse_version_from_str(const char *src_version, struct parse_version *result) ++{ ++ __isula_auto_array_t char **splits = NULL; ++ const size_t max_len = 4; ++ size_t tlen = 0; ++ bool ret = false; ++ ++ splits = util_string_split(src_version, '.'); ++ if (splits == NULL) { ++ ERROR("Split version: \"%s\" failed", src_version); ++ return false; ++ } ++ tlen = util_array_len((const char **)splits); ++ if (tlen < 1 || tlen >= max_len) { ++ ERROR("Invalid version: \"%s\"", src_version); ++ return false; ++ } ++ ++ ret = do_parse_version((const char **)splits, tlen, result); ++ ++ return ret; ++} ++ ++static int do_compare_version(const struct parse_version *p_first, const struct parse_version *p_second) ++{ ++ if (p_first->major != p_second->major) { ++ return p_first->major - p_second->major; ++ } ++ if (p_first->minor != p_second->minor) { ++ return p_first->minor - p_second->minor; ++ } ++ if (p_first->micro != p_second->micro) { ++ return p_first->micro - p_second->micro; ++ } ++ ++ return 0; ++} ++ ++int util_version_compare(const char *first, const char *second, int *diff_value) ++{ ++ struct parse_version first_parsed = { 0 }; ++ struct parse_version second_parsed = { 0 }; ++ ++ if (first == NULL || second == NULL || diff_value == NULL) { ++ ERROR("Invalid argument"); ++ return -1; ++ } ++ ++ if (!parse_version_from_str(first, &first_parsed)) { ++ return -1; ++ } ++ ++ if (!parse_version_from_str(second, &second_parsed)) { ++ return -1; ++ } ++ ++ *diff_value = do_compare_version(&first_parsed, &second_parsed); ++ ++ return 0; ++} ++ ++int util_version_greater_than(const char *first, const char *second, bool *result) ++{ ++ int ret; ++ int diff_value = 0; ++ ++ if (result == NULL) { ++ ERROR("Invalid argument"); ++ return -1; ++ } ++ ++ ret = util_version_compare(first, second, &diff_value); ++ if (ret != 0) { ++ return ret; ++ } ++ ++ *result = (diff_value > 0); ++ return ret; ++} ++ ++int util_version_greater_than_or_equal_to(const char *first, const char *second, bool *result) ++{ ++ int ret; ++ int diff_value = 0; ++ ++ if (result == NULL) { ++ ERROR("Invalid argument"); ++ return -1; ++ } ++ ++ ret = util_version_compare(first, second, &diff_value); ++ if (ret != 0) { ++ return ret; ++ } ++ ++ *result = (diff_value >= 0); ++ return ret; ++} +diff --git a/src/utils/cutils/utils_version.h b/src/utils/cutils/utils_version.h +new file mode 100644 +index 00000000..f9b543b8 +--- /dev/null ++++ b/src/utils/cutils/utils_version.h +@@ -0,0 +1,36 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2022. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-4-7 ++ * Description: provide version functions ++ ********************************************************************************/ ++ ++#ifndef UTILS_CUTILS_UTILS_VERSION_H ++#define UTILS_CUTILS_UTILS_VERSION_H ++ ++#include <stdbool.h> ++#include <stddef.h> ++#include <stdint.h> ++#include <sys/types.h> ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++int util_version_compare(const char *first, const char *second, int *diff_value); ++int util_version_greater_than(const char *first, const char *second, bool *result); ++int util_version_greater_than_or_equal_to(const char *first, const char *second, bool *result); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif // UTILS_CUTILS_UTILS_VERSION_H +\ No newline at end of file +-- +2.34.1 + diff --git a/0067-cdi-support-modules-container_edits-parser.patch b/0067-cdi-support-modules-container_edits-parser.patch new file mode 100644 index 0000000..901de2b --- /dev/null +++ b/0067-cdi-support-modules-container_edits-parser.patch @@ -0,0 +1,1210 @@ +From d0442316761717849deb248b5da45240f7bb1d38 Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Thu, 11 Apr 2024 11:01:44 +0800 +Subject: [PATCH 67/69] cdi:support modules container_edits/parser + +--- + src/daemon/modules/api/specs_api.h | 13 + + .../device/cdi/behavior/cdi_container_edits.c | 622 +++++++++++++++++- + .../device/cdi/behavior/cdi_container_edits.h | 2 +- + .../device/cdi/behavior/parser/cdi_parser.c | 196 +++++- + .../device/cdi/behavior/parser/cdi_parser.h | 6 +- + src/daemon/modules/spec/specs.c | 230 +++++++ + 6 files changed, 1052 insertions(+), 17 deletions(-) + +diff --git a/src/daemon/modules/api/specs_api.h b/src/daemon/modules/api/specs_api.h +index f54c0d31..0999836b 100644 +--- a/src/daemon/modules/api/specs_api.h ++++ b/src/daemon/modules/api/specs_api.h +@@ -55,6 +55,19 @@ const oci_runtime_spec *get_readonly_default_oci_spec(bool system_container); + + int spec_module_init(void); + ++#ifdef ENABLE_CDI ++int defs_process_add_multiple_env(defs_process *dp, const char **envs, size_t env_len); ++int spec_add_multiple_process_env(oci_runtime_spec *oci_spec, const char **envs, size_t env_len); ++int spec_add_device(oci_runtime_spec *oci_spec, defs_device *device); ++int spec_add_linux_resources_device(oci_runtime_spec *oci_spec, bool allow, const char *dev_type, ++ int64_t major, int64_t minor, const char *access); ++void spec_remove_mount(oci_runtime_spec *oci_spec, const char *dest); ++int spec_add_mount(oci_runtime_spec *oci_spec, defs_mount *mnt); ++int spec_add_prestart_hook(oci_runtime_spec *oci_spec, defs_hook *prestart_hook); ++int spec_add_poststart_hook(oci_runtime_spec *oci_spec, defs_hook *poststart_hook); ++int spec_add_poststop_hook(oci_runtime_spec *oci_spec, defs_hook *poststop_hook); ++#endif /* ENABLE_CDI */ ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c +index 590118b1..816b9c2d 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c +@@ -14,36 +14,642 @@ + ******************************************************************************/ + #include "cdi_container_edits.h" + ++#include <sys/stat.h> ++#include <sys/sysmacros.h> ++#include <isula_libutils/auto_cleanup.h> ++#include <isula_libutils/log.h> ++#include <isula_libutils/utils_array.h> ++ ++#include "error.h" ++#include "path.h" ++#include "specs_extend.h" ++#include "utils.h" ++#include "utils_array.h" ++#include "specs_api.h" ++ ++/* ++ * The OCI being used by the iSulad not supportes ++ * createRuntime/createContainer/startContainer currently. ++ */ + // PRESTART_HOOK is the name of the OCI "prestart" hook. + #define PRESTART_HOOK "prestart" +-// CREATE_RUNTIME_HOOK is the name of the OCI "createRuntime" hook. +-#define CREATE_RUNTIME_HOOK "createRuntime" +-// CREATE_CONTAINER_HOOK is the name of the OCI "createContainer" hook. +-#define CREATE_CONTAINER_HOOK "createContainer" +-// START_CONTAINER_HOOK is the name of the OCI "startContainer" hook. +-#define START_CONTAINER_HOOK "startContainer" + // POSTSTART_HOOK is the name of the OCI "poststart" hook. + #define POSTSTART_HOOK "poststart" + // POSTSTOP_HOOK is the name of the OCI "poststop" hook. + #define POSTSTOP_HOOK "poststop" + ++#define VALID_HOOK_NAME_LEN 3 ++static const char* g_valid_hook_names[VALID_HOOK_NAME_LEN] = { ++ PRESTART_HOOK, POSTSTART_HOOK, POSTSTOP_HOOK ++}; ++ ++static int cdi_validate_env(char **envs, size_t envs_len); ++static int cdi_validate_device_node(cdi_device_node *d); ++static int cdi_validate_hook(cdi_hook *h); ++static int cdi_validate_mount(cdi_mount *m); ++ ++#define BLOCK_DEVICE "b" ++#define CHAR_DEVICE "c" ++#define FIFO_DEVICE "p" ++static int device_info_from_path(const char *path, char **dev_type, int64_t *major, int64_t *minor) ++{ ++ struct stat stat = { 0 }; ++ int ret = 0; ++ ++ ret = lstat(path, &stat); ++ if (ret != 0) { ++ ERROR("Failed to stat %s", path); ++ return -1; ++ } ++ ++ if (S_ISBLK(stat.st_mode)) { ++ *dev_type = util_strdup_s(BLOCK_DEVICE); ++ } else if (S_ISCHR(stat.st_mode)) { ++ *dev_type = util_strdup_s(CHAR_DEVICE); ++ } else if (S_ISFIFO(stat.st_mode)) { ++ *dev_type = util_strdup_s(FIFO_DEVICE); ++ } else { ++ *dev_type = NULL; ++ *major = 0; ++ *minor = 0; ++ ERROR("Not a device node"); ++ return -1; ++ } ++ ++ *major = (int64_t)major(stat.st_rdev); ++ *minor = (int64_t)minor(stat.st_rdev); ++ return 0; ++} ++ ++static int fill_device_node_info(cdi_device_node *d) ++{ ++ __isula_auto_free char *dev_type = NULL; ++ int64_t major; ++ int64_t minor; ++ ++ if (d->host_path == NULL) { ++ d->host_path = util_strdup_s(d->path); ++ } ++ ++ if (d->type != NULL && (d->major != 0 || strcmp(d->type, FIFO_DEVICE) == 0)) { ++ return 0; ++ } ++ ++ if (device_info_from_path(d->host_path, &dev_type, &major, &minor) != 0) { ++ ERROR("Failed to stat CDI host device %s", d->host_path); ++ return -1; ++ } ++ ++ if (d->type == NULL) { ++ d->type = dev_type; ++ dev_type = NULL; ++ } else { ++ if (strcmp(d->type, dev_type) != 0) { ++ ERROR("CDI device (%s, %s), host type mismatch (%s, %s)", ++ d->path, d->host_path, d->type, dev_type); ++ return -1; ++ } ++ } ++ if (d->major == 0 && strcmp(d->type, FIFO_DEVICE) != 0) { ++ d->major = major; ++ d->minor = minor; ++ } ++ return 0; ++} ++ ++static cdi_device_node *clone_cdi_device_node(cdi_device_node *d) ++{ ++ cdi_device_node *device_node = NULL; ++ ++ device_node = util_common_calloc_s(sizeof(*device_node)); ++ if (device_node == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ device_node->path = util_strdup_s(d->path); ++ device_node->host_path = util_strdup_s(d->host_path); ++ device_node->type = util_strdup_s(d->type); ++ device_node->major = d->major; ++ device_node->minor = d->minor; ++ device_node->file_mode = d->file_mode; ++ device_node->permissions = util_strdup_s(d->permissions); ++ device_node->uid = d->uid; ++ device_node->gid = d->gid; ++ return device_node; ++} ++ ++static cdi_hook *clone_cdi_hook(cdi_hook *h) ++{ ++ cdi_hook *hook = NULL; ++ ++ hook = util_common_calloc_s(sizeof(*hook)); ++ if (hook == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ hook->hook_name = util_strdup_s(h->hook_name); ++ hook->path = util_strdup_s(h->path); ++ if (h->args_len != 0) { ++ hook->args = util_copy_array_by_len(h->args, h->args_len); ++ if (hook->args == NULL) { ++ ERROR("Failed to copy args"); ++ goto error_out; ++ } ++ hook->args_len = h->args_len; ++ } ++ if (h->env_len != 0) { ++ hook->env = util_copy_array_by_len(h->env, h->env_len); ++ if (hook->env == NULL) { ++ ERROR("Failed to copy env"); ++ goto error_out; ++ } ++ hook->env_len = h->env_len; ++ } ++ hook->timeout = h->timeout; ++ ++ return hook; ++ ++error_out: ++ free_cdi_hook(hook); ++ return NULL; ++} ++ ++static cdi_mount *clone_cdi_mount(cdi_mount *m) ++{ ++ cdi_mount *mount = NULL; ++ ++ mount = util_common_calloc_s(sizeof(*mount)); ++ if (mount == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ mount->host_path = util_strdup_s(m->host_path); ++ mount->container_path = util_strdup_s(m->container_path); ++ if (m->options_len != 0) { ++ mount->options = util_copy_array_by_len(m->options, m->options_len); ++ if (mount->options == NULL) { ++ ERROR("Failed to copy options"); ++ free_cdi_mount(mount); ++ return NULL; ++ } ++ mount->options_len = m->options_len; ++ } ++ mount->type = util_strdup_s(m->type); ++ ++ return mount; ++} ++ ++static defs_hook *cdi_hook_to_oci(cdi_hook *h) ++{ ++ defs_hook *oci_hook = NULL; ++ ++ oci_hook = util_common_calloc_s(sizeof(*oci_hook)); ++ if (oci_hook == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ oci_hook->path = util_strdup_s(h->path); ++ if (h->args_len != 0) { ++ oci_hook->args = util_copy_array_by_len(h->args, h->args_len); ++ if (oci_hook->args == NULL) { ++ ERROR("Failed to copy args"); ++ goto error_out; ++ } ++ oci_hook->args_len = h->args_len; ++ } ++ if (h->env_len != 0) { ++ oci_hook->env = util_copy_array_by_len(h->env, h->env_len); ++ if (oci_hook->env == NULL) { ++ ERROR("Failed to copy env"); ++ goto error_out; ++ } ++ oci_hook->env_len = h->env_len; ++ } ++ oci_hook->timeout = h->timeout; ++ return oci_hook; ++ ++error_out: ++ free_defs_hook(oci_hook); ++ return NULL; ++} ++ ++static defs_mount *cdi_mount_to_oci(cdi_mount *m) ++{ ++ defs_mount *oci_mount = NULL; ++ ++ oci_mount = util_common_calloc_s(sizeof(*oci_mount)); ++ if (oci_mount == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ oci_mount->source = util_strdup_s(m->host_path); ++ oci_mount->destination = util_strdup_s(m->container_path); ++ if (m->options_len != 0) { ++ oci_mount->options = util_copy_array_by_len(m->options, m->options_len); ++ if (oci_mount->options == NULL) { ++ ERROR("Failed to copy options"); ++ free_defs_mount(oci_mount); ++ return NULL; ++ } ++ oci_mount->options_len = m->options_len; ++ } ++ oci_mount->type = util_strdup_s(m->type); ++ ++ return oci_mount; ++} ++ ++static defs_device *cdi_device_node_to_oci(cdi_device_node *d) ++{ ++ defs_device *oci_device = NULL; ++ ++ oci_device = util_common_calloc_s(sizeof(*oci_device)); ++ if (oci_device == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ oci_device->path = util_strdup_s(d->path); ++ oci_device->type = util_strdup_s(d->type); ++ oci_device->major = d->major; ++ oci_device->minor = d->minor; ++ oci_device->file_mode = d->file_mode; ++ oci_device->uid = d->uid; ++ oci_device->gid = d->gid; ++ ++ return oci_device; ++} ++ ++static int apply_cdi_device_nodes(cdi_container_edits *e, oci_runtime_spec *spec) ++{ ++ size_t i; ++ defs_device *dev = NULL; ++ cdi_device_node *dn = NULL; ++ const char *access = NULL; ++ ++ for (i = 0; i < e->device_nodes_len; i++) { ++ dn = clone_cdi_device_node(e->device_nodes[i]); ++ if (dn == NULL) { ++ ERROR("Failed to copy device node"); ++ goto error_out; ++ } ++ ++ if (fill_device_node_info(dn) != 0) { ++ goto error_out; ++ } ++ dev = cdi_device_node_to_oci(dn); ++ if (dev == NULL) { ++ ERROR("Failed to generate oci device"); ++ goto error_out; ++ } ++ /* Currently, for uid and gid, isulad cannot distinguish ++ * 0 and unspecified. Here, 0 is processed as unspecified. ++ */ ++ if (dev->uid == 0 && spec->process != NULL) { ++ if (spec->process->user->uid > 0) { ++ dev->uid = spec->process->user->uid; ++ } ++ } ++ if (dev->gid == 0 && spec->process != NULL) { ++ if (spec->process->user->gid > 0) { ++ dev->gid = spec->process->user->gid; ++ } ++ } ++ ++ if (spec_add_device(spec, dev) != 0) { ++ goto error_out; ++ } ++ ++ if (strcmp(dev->type, BLOCK_DEVICE) == 0 || strcmp(dev->type, CHAR_DEVICE) == 0) { ++ if (e->device_nodes[i]->permissions != NULL) { ++ access = e->device_nodes[i]->permissions; ++ } else { ++ access = "rwm"; ++ } ++ if (spec_add_linux_resources_device(spec, true, dev->type, ++ dev->major, dev->minor, access)) { ++ dev = NULL; ++ goto error_out; ++ } ++ } ++ free_cdi_device_node(dn); ++ dn = NULL; ++ dev = NULL; ++ } ++ ++ return 0; ++ ++error_out: ++ free_cdi_device_node(dn); ++ free_defs_device(dev); ++ return -1; ++} ++ ++static int defs_mount_parts(defs_mount *m) ++{ ++ char cleanpath[PATH_MAX] = { 0 }; ++ if (util_clean_path(m->destination, cleanpath, sizeof(cleanpath)) == NULL) { ++ return -1; ++ } ++ ++ return util_strings_count(cleanpath, '/'); ++} ++ ++static inline int defs_mount_cmp(defs_mount **first, defs_mount **second) ++{ ++ int first_part = defs_mount_parts(*first); ++ int second_part = defs_mount_parts(*second); ++ ++ if (first_part < second_part) { ++ return -1; ++ } ++ if (first_part > second_part) { ++ return 1; ++ } ++ ++ return strcmp((*first)->destination, (*second)->destination); ++} ++ ++static int apply_cdi_mounts(cdi_container_edits *e, oci_runtime_spec *spec) ++{ ++ size_t i; ++ defs_mount *mnt = NULL; ++ ++ if (e->mounts_len == 0) { ++ return 0; ++ } ++ ++ for (i = 0; i < e->mounts_len; i++) { ++ spec_remove_mount(spec, e->mounts[i]->container_path); ++ mnt = cdi_mount_to_oci(e->mounts[i]); ++ if (spec_add_mount(spec, mnt) != 0) { ++ free_defs_mount(mnt); ++ return -1; ++ } ++ } ++ ++ qsort(spec->mounts, spec->mounts_len, ++ sizeof(defs_mount *), (int (*)(const void *, const void *))defs_mount_cmp); ++ return 0; ++} ++ ++static int apply_cdi_hooks(cdi_container_edits *e, oci_runtime_spec *spec) ++{ ++ size_t i; ++ int ret = 0; ++ ++ for (i = 0; i < e->hooks_len; i++) { ++ defs_hook *oci_hook = cdi_hook_to_oci(e->hooks[i]); ++ if (strcmp(e->hooks[i]->hook_name, PRESTART_HOOK)) { ++ ret = spec_add_prestart_hook(spec, oci_hook); ++ } else if (strcmp(e->hooks[i]->hook_name, POSTSTART_HOOK)) { ++ ret = spec_add_poststart_hook(spec, oci_hook); ++ } else if (strcmp(e->hooks[i]->hook_name, POSTSTOP_HOOK)) { ++ ret = spec_add_poststop_hook(spec, oci_hook); ++ } else { ++ /* ++ * The OCI being used by the iSulad not supportes ++ * createRuntime/createContainer/startContainer currently. ++ */ ++ ERROR("Unknown hook name %s", e->hooks[i]->hook_name); ++ free_defs_hook(oci_hook); ++ return -1; ++ } ++ if (ret != 0) { ++ ERROR("Failed add hook %s", e->hooks[i]->hook_name); ++ free_defs_hook(oci_hook); ++ return -1; ++ } ++ } ++ return ret; ++} ++ + int cdi_container_edits_apply(cdi_container_edits *e, oci_runtime_spec *spec) + { ++ if (spec == NULL) { ++ ERROR("Can't edit nil OCI Spec"); ++ return -1; ++ } ++ if (e == NULL) { ++ WARN("Cdi container edits is nil"); ++ return 0; ++ } ++ ++ if (e->env_len > 0) { ++ if (spec_add_multiple_process_env(spec, (const char **)e->env, e->env_len) != 0) { ++ ERROR("Failed to merge envs"); ++ return -1; ++ } ++ } ++ ++ if (apply_cdi_device_nodes(e, spec) != 0) { ++ ERROR("Failed to apply device nodes"); ++ return -1; ++ } ++ ++ if (apply_cdi_mounts(e, spec) != 0) { ++ ERROR("Failed to apply mounts"); ++ return -1; ++ } ++ ++ if (apply_cdi_hooks(e, spec) != 0) { ++ ERROR("Failed to apply hooks"); ++ return -1; ++ } ++ + return 0; + } + +-int cdi_container_edits_validate(cdi_container_edits *e, char **error) ++int cdi_container_edits_validate(cdi_container_edits *e) + { ++ size_t i; ++ ++ if (e == NULL) { ++ WARN("Cdi container edits is nil"); ++ return 0; ++ } ++ ++ if (cdi_validate_env(e->env, e->env_len) != 0) { ++ ERROR("Invalid container edits"); ++ return -1; ++ } ++ for (i = 0; i < e->device_nodes_len; i++) { ++ if (cdi_validate_device_node(e->device_nodes[i]) != 0) { ++ ERROR("Invalid container device node"); ++ return -1; ++ } ++ } ++ for (i = 0; i < e->hooks_len; i++) { ++ if (cdi_validate_hook(e->hooks[i]) != 0) { ++ ERROR("Invalid container hook"); ++ return -1; ++ } ++ } ++ for (i = 0; i < e->mounts_len; i++) { ++ if (cdi_validate_mount(e->mounts[i]) != 0) { ++ ERROR("Invalid container mount"); ++ return -1; ++ } ++ } ++ + return 0; + } + ++#define EDITS_APPEND_ITEM_DEF(item) \ ++ static int append_##item(cdi_container_edits *e, cdi_container_edits *o, clone_common_array_item_cb cb) \ ++ { \ ++ common_array e_array = { \ ++ .items = (void **)e->item, \ ++ .len = e->item##_len, \ ++ .cap = e->item##_len, \ ++ .free_item_cb = NULL, \ ++ .clone_item_cb = cb \ ++ }; \ ++ common_array o_array = { \ ++ .items = (void **)o->item, \ ++ .len = o->item##_len, \ ++ .cap = o->item##_len, \ ++ .free_item_cb = NULL, \ ++ .clone_item_cb = cb \ ++ }; \ ++ if (util_merge_common_array(&e_array, &o_array) != 0) { \ ++ ERROR("Out of memory"); \ ++ return -1; \ ++ } \ ++ e->item = (void *)e_array.items; \ ++ e->item##_len += o->item##_len; \ ++ return 0; \ ++ } ++ ++EDITS_APPEND_ITEM_DEF(env) ++EDITS_APPEND_ITEM_DEF(device_nodes) ++EDITS_APPEND_ITEM_DEF(hooks) ++EDITS_APPEND_ITEM_DEF(mounts) ++ + int cdi_container_edits_append(cdi_container_edits *e, cdi_container_edits *o) + { ++ if (o == NULL) { ++ return 0; ++ } ++ if (e == NULL) { ++ ERROR("Invalid params"); ++ return -1; ++ } ++ ++ if (append_env(e, o, (clone_common_array_item_cb)util_strdup_s) != 0) { ++ return -1; ++ } ++ if (append_device_nodes(e, o, (clone_common_array_item_cb)clone_cdi_device_node) != 0) { ++ return -1; ++ } ++ if (append_hooks(e, o, (clone_common_array_item_cb)clone_cdi_hook) != 0) { ++ return -1; ++ } ++ if (append_mounts(e, o, (clone_common_array_item_cb)clone_cdi_mount) != 0) { ++ return -1; ++ } ++ + return 0; + } + + bool cdi_container_edits_is_empty(cdi_container_edits *e) + { +- return true; ++ if (e == NULL) { ++ return false; ++ } ++ return e->env_len + e->device_nodes_len + e->hooks_len + e->mounts_len == 0; + } + ++static int cdi_validate_env(char **envs, size_t envs_len) ++{ ++ size_t i; ++ char *ptr = NULL; ++ ++ for (i = 0; i < envs_len; i++) { ++ ptr = strchr(envs[i], '='); ++ if (ptr == NULL || ptr == envs[i]) { ++ ERROR("Invalid environment variable %s", envs[i]); ++ return -1; ++ } ++ } ++ return 0; ++} ++ ++static int cdi_validate_device_node(cdi_device_node *d) ++{ ++ char *p = NULL; ++ ++ if (d == NULL) { ++ ERROR("Device node is nil"); ++ return -1; ++ } ++ ++ if (d->path == NULL) { ++ ERROR("Invalid (empty) device path"); ++ return -1; ++ } ++ if (d->type != NULL && strcmp(d->type, BLOCK_DEVICE) != 0 && ++ strcmp(d->type, CHAR_DEVICE) != 0 && strcmp(d->type, FIFO_DEVICE) != 0) { ++ ERROR("Device %s: invalid type %s", d->path, d->type); ++ return -1; ++ } ++ for (p = d->permissions; p != NULL && *p != '\0'; p++) { ++ if (*p != 'r' && *p != 'w' && *p != 'm') { ++ ERROR("Device %s: invalid permissions %s", d->path, d->permissions); ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ ++static int cdi_validate_hook(cdi_hook *h) ++{ ++ size_t i; ++ ++ if (h == NULL) { ++ ERROR("Hook is nil"); ++ return -1; ++ } ++ ++ for (i = 0; i < VALID_HOOK_NAME_LEN; i++) { ++ if (strcmp(h->hook_name, g_valid_hook_names[i]) == 0) { ++ break; ++ } ++ } ++ if (i == VALID_HOOK_NAME_LEN) { ++ ERROR("Invalid hook name %s", h->hook_name); ++ return -1; ++ } ++ if (h->path == NULL) { ++ ERROR("Invalid hook %s with empty path", h->hook_name); ++ return -1; ++ } ++ if (cdi_validate_env(h->env, h->env_len) != 0) { ++ ERROR("Invalid hook %s", h->hook_name); ++ return -1; ++ } ++ return 0; ++} ++ ++static int cdi_validate_mount(cdi_mount *m) ++{ ++ if (m == NULL) { ++ ERROR("Mount is nil"); ++ return -1; ++ } ++ ++ if (m->host_path == NULL) { ++ ERROR("Invalid mount, empty host path"); ++ return -1; ++ } ++ if (m->container_path == NULL) { ++ ERROR("Invalid mount, empty container path"); ++ return -1; ++ } ++ return 0; ++} +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_container_edits.h b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.h +index ea921e37..ddceec25 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_container_edits.h ++++ b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.h +@@ -28,7 +28,7 @@ extern "C" { + #endif + + int cdi_container_edits_apply(cdi_container_edits *e, oci_runtime_spec *spec); +-int cdi_container_edits_validate(cdi_container_edits *e, char **error); ++int cdi_container_edits_validate(cdi_container_edits *e); + int cdi_container_edits_append(cdi_container_edits *e, cdi_container_edits *o); + bool cdi_container_edits_is_empty(cdi_container_edits *e); + +diff --git a/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c +index 14293c72..8824d29c 100644 +--- a/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c ++++ b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c +@@ -14,42 +14,228 @@ + ******************************************************************************/ + #include "cdi_parser.h" + ++#include <stdio.h> ++#include <ctype.h> ++#include <string.h> ++#include <isula_libutils/log.h> ++#include <isula_libutils/auto_cleanup.h> ++ ++#include "error.h" ++#include "utils_string.h" ++ ++/* cdi_parser_qualified_name returns the qualified name for a device. ++ * The syntax for a qualified device names is ++ * ++ * "<vendor>/<class>=<name>". ++ * ++ * A valid vendor and class name may contain the following runes: ++ * ++ * 'A'-'Z', 'a'-'z', '0'-'9', '.', '-', '_'. ++ * ++ * A valid device name may contain the following runes: ++ * ++ * 'A'-'Z', 'a'-'z', '0'-'9', '-', '_', '.', ':' ++ */ + char *cdi_parser_qualified_name(const char *vendor, const char *class, const char *name) + { +- return NULL; ++ char device_name[PATH_MAX] = { 0 }; ++ int nret; ++ ++ if (vendor == NULL || class == NULL || name == NULL) { ++ ERROR("Invalid argument"); ++ return NULL; ++ } ++ ++ nret = snprintf(device_name, sizeof(device_name), "%s/%s=%s", ++ vendor, class, name); ++ if (nret < 0 || (size_t)nret >= sizeof(device_name)) { ++ ERROR("Device name is too long"); ++ return NULL; ++ } ++ return util_strdup_s(device_name); + } + ++// cdi_parser_is_qualified_name tests if a device name is qualified. + bool cdi_parser_is_qualified_name(const char *device) + { +- return true; ++ __isula_auto_free char *vendor = NULL; ++ __isula_auto_free char *class = NULL; ++ __isula_auto_free char *name = NULL; ++ ++ return cdi_parser_parse_qualified_name(device, &vendor, &class, &name) == 0; + } + ++// cdi_parser_parse_qualified_name splits a qualified name into device vendor, class, and name. + int cdi_parser_parse_qualified_name(const char *device, char **vendor, char **class, char **name) + { ++ int ret = 0; ++ ++ ret = cdi_parser_parse_device(device, vendor, class, name); ++ if (ret != 0) { ++ if (*vendor == NULL) { ++ ERROR("Unqualified device %s, missing vendor", device); ++ return -1; ++ } ++ if (*class == NULL) { ++ ERROR("Unqualified device %s, missing class", device); ++ return -1; ++ } ++ if (*name == NULL) { ++ ERROR("Unqualified device %s, missing name", device); ++ return -1; ++ } ++ ERROR("Unqualified device %s", device); ++ return -1; ++ } ++ ++ if (cdi_parser_validate_vendor_name(*vendor) != 0) { ++ ERROR("Invalid device %s", device); ++ goto err_out; ++ } ++ if (cdi_parser_validate_class_name(*class) != 0) { ++ ERROR("Invalid device %s", device); ++ goto err_out; ++ } ++ if (cdi_parser_validate_device_name(*name) != 0) { ++ ERROR("Invalid device %s", device); ++ goto err_out; ++ } ++ + return 0; ++ ++err_out: ++ free(*vendor); ++ *vendor = NULL; ++ free(*class); ++ *class = NULL; ++ free(*name); ++ *name = NULL; ++ return -1; + } + ++// cdi_parser_parse_device tries to split a device name into vendor, class, and name. + int cdi_parser_parse_device(const char *device, char **vendor, char **class, char **name) + { ++ __isula_auto_array_t char **parts = NULL; ++ ++ if (vendor == NULL || class == NULL || name == NULL || ++ device == NULL || device[0] == '/') { ++ ERROR("Invalid argument"); ++ return -1; ++ } ++ ++ parts = util_string_split_n(device, '=', 2); ++ if (parts == NULL || util_array_len((const char **)parts) != 2 || parts[0] == NULL || parts[1] == NULL) { ++ return -1; ++ } ++ ++ *name = parts[1]; ++ parts[1] = NULL; ++ (void)cdi_parser_parse_qualifier(parts[0], vendor, class); ++ if (*vendor == NULL) { ++ ERROR("Failed to parse device qualifier: %s", parts[0]); ++ return -1; ++ } ++ + return 0; + } + ++/* cdi_parser_parse_qualifier splits a device qualifier into vendor and class. ++ * The syntax for a device qualifier is ++ * ++ * "<vendor>/<class>" ++ */ + int cdi_parser_parse_qualifier(const char *kind, char **vendor, char **class) + { ++ __isula_auto_array_t char **parts = NULL; ++ ++ if (kind == NULL || vendor == NULL || class == NULL) { ++ ERROR("Invalid argument"); ++ return -1; ++ } ++ ++ parts = util_string_split_n(kind, '/', 2); ++ if (parts == NULL || util_array_len((const char **)parts) != 2 || parts[0] == NULL || parts[1] == NULL) { ++ return -1; ++ } ++ *vendor = parts[0]; ++ parts[0] = NULL; ++ *class = parts[1]; ++ parts[1] = NULL; ++ ++ return 0; ++} ++ ++static int validate_vendor_or_class_name(const char *name) ++{ ++ int i = 0; ++ ++ if (name == NULL) { ++ ERROR("Empty name"); ++ return -1; ++ } ++ ++ if (!isalpha(name[0])) { ++ ERROR("%s, should start with letter", name); ++ return -1; ++ } ++ for (i = 1; name[i] != '\0'; i++) { ++ if (!(isalnum(name[i]) || name[i] == '_' || name[i] == '-' || name[i] == '.')) { ++ ERROR("Invalid character '%c' in name %s", name[i], name); ++ return -1; ++ } ++ } ++ if (!isalnum(name[i - 1])) { ++ ERROR("%s, should end with a letter or digit", name); ++ return -1; ++ } ++ + return 0; + } + +-int cdi_parser_validate_vendor_name(const char *vendor, char **error) ++int cdi_parser_validate_vendor_name(const char *vendor) + { ++ if (validate_vendor_or_class_name(vendor) != 0) { ++ ERROR("Invalid vendor"); ++ return -1; ++ } + return 0; + } + +-int cdi_parser_validate_class_name(const char *class, char **error) ++int cdi_parser_validate_class_name(const char *class) + { ++ if (validate_vendor_or_class_name(class) != 0) { ++ ERROR("Invalid class."); ++ return -1; ++ } + return 0; + } + +-int cdi_parser_validate_device_name(const char *name, char **error) ++int cdi_parser_validate_device_name(const char *name) + { ++ size_t i; ++ ++ if (name == NULL) { ++ ERROR("Invalid (empty) device name"); ++ return -1; ++ } ++ if (!isalnum(name[0])) { ++ ERROR("Invalid class %s, should start with a letter or digit", name); ++ return -1; ++ } ++ if (strlen(name) == 1) { ++ return 0; ++ } ++ for (i = 1; name[i] != '\0'; i++) { ++ if (!(isalnum(name[i]) || name[i] == '_' || name[i] == '-' || name[i] == '.' || name[i] == ':')) { ++ ERROR("Invalid character '%c' in device name %s", name[i], name); ++ return -1; ++ } ++ } ++ if (!isalnum(name[i - 1])) { ++ ERROR("Invalid name %s, should end with a letter or digit", name); ++ return -1; ++ } ++ + return 0; + } +diff --git a/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.h b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.h +index 467641a1..3658e29b 100644 +--- a/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.h ++++ b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.h +@@ -27,9 +27,9 @@ bool cdi_parser_is_qualified_name(const char *device); + int cdi_parser_parse_qualified_name(const char *device, char **vendor, char **class, char **name); + int cdi_parser_parse_device(const char *device, char **vendor, char **class, char **name); + int cdi_parser_parse_qualifier(const char *kind, char **vendor, char **class); +-int cdi_parser_validate_vendor_name(const char *vendor, char **error); +-int cdi_parser_validate_class_name(const char *class, char **error); +-int cdi_parser_validate_device_name(const char *name, char **error); ++int cdi_parser_validate_vendor_name(const char *vendor); ++int cdi_parser_validate_class_name(const char *class); ++int cdi_parser_validate_device_name(const char *name); + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index b4d2b0f6..77ca70f9 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -29,6 +29,7 @@ + #include <isula_libutils/host_config.h> + #include <isula_libutils/log.h> + #include <isula_libutils/auto_cleanup.h> ++#include <isula_libutils/utils_array.h> + + #include "specs_api.h" + #include "utils.h" +@@ -2595,3 +2596,232 @@ int spec_module_init(void) + return 0; + } + ++#ifdef ENABLE_CDI ++static int add_env(defs_process *dp, const char *env, const char *key) ++{ ++ size_t i; ++ char *oci_key = NULL; ++ char *oci_value = NULL; ++ char *saveptr = NULL; ++ __isula_auto_free char *tmp_env = NULL; ++ ++ for (i = 0; i < dp->env_len; i++) { ++ tmp_env = util_strdup_s(dp->env[i]); ++ oci_key = strtok_r(tmp_env, "=", &saveptr); ++ oci_value = strtok_r(NULL, "=", &saveptr); ++ if (oci_key == NULL || oci_value == NULL) { ++ ERROR("Bad env format"); ++ return -1; ++ } ++ if (strcmp(key, oci_key) == 0) { ++ free(dp->env[i]); ++ dp->env[i] = util_strdup_s(env); ++ return 0; ++ } ++ free(tmp_env); ++ tmp_env = NULL; ++ } ++ if (util_mem_realloc((void **)&dp->env, (dp->env_len + 1) * sizeof(char *), ++ (void *)dp->env, dp->env_len * sizeof(char *)) != 0) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ dp->env[dp->env_len] = util_strdup_s(env); ++ dp->env_len++; ++ return 0; ++} ++ ++int defs_process_add_multiple_env(defs_process *dp, const char **envs, size_t env_len) ++{ ++ size_t i; ++ char *key = NULL; ++ char *value = NULL; ++ char *saveptr = NULL; ++ __isula_auto_free char *tmp_env = NULL; ++ ++ if (envs == NULL || env_len == 0) { ++ DEBUG("empty envs"); ++ return 0; ++ } ++ if (dp == NULL) { ++ ERROR("Invalid params"); ++ return -1; ++ } ++ ++ for (i = 0; i < env_len; i++) { ++ tmp_env = util_strdup_s(envs[i]); ++ key = strtok_r(tmp_env, "=", &saveptr); ++ value = strtok_r(NULL, "=", &saveptr); ++ if (key == NULL || value == NULL) { ++ ERROR("Bad env format: %s", tmp_env); ++ return -1; ++ } ++ if (add_env(dp, envs[i], key) != 0) { ++ return -1; ++ } ++ free(tmp_env); ++ tmp_env = NULL; ++ } ++ ++ return 0; ++} ++ ++int spec_add_multiple_process_env(oci_runtime_spec *oci_spec, const char **envs, size_t env_len) ++{ ++ int ret = 0; ++ ++ if (envs == NULL || env_len == 0) { ++ DEBUG("empty envs"); ++ return 0; ++ } ++ if (oci_spec == NULL) { ++ ERROR("Invalid params"); ++ return -1; ++ } ++ ++ ret = make_sure_oci_spec_process(oci_spec); ++ if (ret < 0) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ ++ ret = defs_process_add_multiple_env(oci_spec->process, envs, env_len); ++ if (ret < 0) { ++ ERROR("Failed to add envs"); ++ } ++ ++ return ret; ++} ++ ++int spec_add_device(oci_runtime_spec *oci_spec, defs_device *device) ++{ ++ int ret = 0; ++ size_t i; ++ ++ if (device == NULL) { ++ return -1; ++ } ++ ret = make_sure_oci_spec_linux(oci_spec); ++ if (ret < 0) { ++ return -1; ++ } ++ ++ for (i = 0; i < oci_spec->linux->devices_len; i++) { ++ if (strcmp(oci_spec->linux->devices[i]->path, device->path) == 0) { ++ free_defs_device(oci_spec->linux->devices[i]); ++ oci_spec->linux->devices[i] = device; ++ return 0; ++ } ++ } ++ ++ if (util_mem_realloc((void **)&oci_spec->linux->devices, (oci_spec->linux->devices_len + 1) * sizeof(char *), ++ (void *)oci_spec->linux->devices, oci_spec->linux->devices_len * sizeof(char *)) != 0) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ oci_spec->linux->devices[oci_spec->linux->devices_len] = device; ++ oci_spec->linux->devices_len++; ++ ++ return 0; ++} ++ ++int spec_add_linux_resources_device(oci_runtime_spec *oci_spec, bool allow, const char *dev_type, ++ int64_t major, int64_t minor, const char *access) ++{ ++ int ret = 0; ++ defs_device_cgroup *device = NULL; ++ ++ ret = make_sure_oci_spec_linux_resources(oci_spec); ++ if (ret < 0) { ++ return -1; ++ } ++ ++ device = util_common_calloc_s(sizeof(*device)); ++ if (device == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ device->allow = allow; ++ device->type = util_strdup_s(dev_type); ++ device->access = util_strdup_s(access); ++ device->major = major; ++ device->minor = minor; ++ ++ if (util_mem_realloc((void **)&oci_spec->linux->resources->devices, (oci_spec->linux->resources->devices_len + 1) * sizeof(char *), ++ (void *)oci_spec->linux->resources->devices, oci_spec->linux->resources->devices_len * sizeof(char *)) != 0) { ++ ERROR("Out of memory"); ++ free_defs_device_cgroup(device); ++ return -1; ++ } ++ oci_spec->linux->resources->devices[oci_spec->linux->resources->devices_len] = device; ++ oci_spec->linux->resources->devices_len++; ++ ++ return 0; ++} ++ ++void spec_remove_mount(oci_runtime_spec *oci_spec, const char *dest) ++{ ++ size_t i; ++ ++ if (oci_spec == NULL || oci_spec->mounts == NULL || dest == NULL) { ++ return; ++ } ++ ++ for (i = 0; i < oci_spec->mounts_len; i++) { ++ if (strcmp(oci_spec->mounts[i]->destination, dest) == 0) { ++ free_defs_mount(oci_spec->mounts[i]); ++ (void)memcpy((void **)&oci_spec->mounts[i], (void **)&oci_spec->mounts[i + 1], ++ (oci_spec->mounts_len - i - 1) * sizeof(void *)); ++ oci_spec->mounts_len--; ++ return; ++ } ++ } ++} ++ ++int spec_add_mount(oci_runtime_spec *oci_spec, defs_mount *mnt) ++{ ++ if (oci_spec == NULL || mnt == NULL) { ++ return -1; ++ } ++ ++ if (util_mem_realloc((void **)&oci_spec->mounts, (oci_spec->mounts_len + 1) * sizeof(char *), ++ (void *)oci_spec->mounts, oci_spec->mounts_len * sizeof(char *)) != 0) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ oci_spec->mounts[oci_spec->mounts_len] = mnt; ++ oci_spec->mounts_len++; ++ ++ return 0; ++} ++ ++#define SPEC_ADD_HOOKS_ITEM_DEF(hooktype) \ ++ int spec_add_##hooktype##_hook(oci_runtime_spec *oci_spec, defs_hook *hooktype##_hook) \ ++ { \ ++ int ret = 0; \ ++ if (oci_spec == NULL || hooktype##_hook == NULL) { \ ++ return -1; \ ++ } \ ++ ret = make_sure_oci_spec_hooks(oci_spec); \ ++ if (ret < 0) { \ ++ return -1; \ ++ } \ ++ if (util_mem_realloc((void **)&oci_spec->hooks->hooktype, (oci_spec->hooks->hooktype##_len + 1) * sizeof(char *), \ ++ (void *)oci_spec->hooks->hooktype, oci_spec->hooks->hooktype##_len * sizeof(char *)) != 0) { \ ++ ERROR("Out of memory"); \ ++ return -1; \ ++ } \ ++ oci_spec->hooks->hooktype[oci_spec->hooks->hooktype##_len] = hooktype##_hook; \ ++ oci_spec->hooks->hooktype##_len++; \ ++ return 0; \ ++ } ++ ++/* ++* The OCI being used by the iSulad not supportes ++* createRuntime/createContainer/startContainer currently. ++*/ ++SPEC_ADD_HOOKS_ITEM_DEF(prestart) ++SPEC_ADD_HOOKS_ITEM_DEF(poststart) ++SPEC_ADD_HOOKS_ITEM_DEF(poststop) ++ ++#endif /* ENABLE_CDI */ +\ No newline at end of file +-- +2.34.1 + diff --git a/0068-cdi-invoke-cdi-operate-when-init-isulad-and-create-c.patch b/0068-cdi-invoke-cdi-operate-when-init-isulad-and-create-c.patch new file mode 100644 index 0000000..3633616 --- /dev/null +++ b/0068-cdi-invoke-cdi-operate-when-init-isulad-and-create-c.patch @@ -0,0 +1,371 @@ +From 0674bfac4dd1ab812432334c779ab718dc54bc8b Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Thu, 11 Apr 2024 11:02:19 +0800 +Subject: [PATCH 68/69] cdi:invoke cdi operate when init isulad and create + container + +--- + src/cmd/isulad/main.c | 11 +++ + src/daemon/common/cri/v1/v1_cri_helpers.cc | 79 +++++++++++++++++++ + src/daemon/common/cri/v1/v1_cri_helpers.h | 3 + + src/daemon/config/daemon_arguments.c | 4 + + src/daemon/config/isulad_config.c | 8 ++ + .../v1/v1_cri_container_manager_service.cc | 8 ++ + .../executor/container_cb/execution_create.c | 9 +++ + .../modules/service/service_container.c | 10 +++ + src/daemon/modules/spec/specs_mount.c | 43 +++++++++- + src/daemon/modules/spec/specs_mount.h | 4 + + src/daemon/modules/spec/verify.c | 2 +- + 11 files changed, 179 insertions(+), 2 deletions(-) + +diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c +index 9fa87bdb..3e2249d7 100644 +--- a/src/cmd/isulad/main.c ++++ b/src/cmd/isulad/main.c +@@ -83,6 +83,9 @@ + #endif + #include "id_name_manager.h" + #include "cgroup.h" ++#ifdef ENABLE_CDI ++#include "cdi_operate_api.h" ++#endif /* ENABLE_CDI */ + + sem_t g_daemon_shutdown_sem; + sem_t g_daemon_wait_shutdown_sem; +@@ -1400,6 +1403,14 @@ static int isulad_server_init_common() + } + #endif + ++#ifdef ENABLE_CDI ++ if (args->json_confs->enable_cdi && ++ cdi_operate_registry_init(args->json_confs->cdi_spec_dirs, args->json_confs->cdi_spec_dirs_len) != 0) { ++ ERROR("Failed to init CDI module"); ++ goto out; ++ } ++#endif /* ENABLE_CDI */ ++ + if (spec_module_init() != 0) { + ERROR("Failed to init spec module"); + goto out; +diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.cc b/src/daemon/common/cri/v1/v1_cri_helpers.cc +index ea5c8bb5..520d23d4 100644 +--- a/src/daemon/common/cri/v1/v1_cri_helpers.cc ++++ b/src/daemon/common/cri/v1/v1_cri_helpers.cc +@@ -22,6 +22,7 @@ + + #include <isula_libutils/log.h> + #include <isula_libutils/parse_common.h> ++#include <isula_libutils/auto_cleanup.h> + + #include "v1_cri_security_context.h" + #include "cri_helpers.h" +@@ -33,6 +34,9 @@ + #include "isulad_config.h" + #include "sha256.h" + #include "v1_naming.h" ++#ifdef ENABLE_CDI ++#include "cdi_operate_api.h" ++#endif /* ENABLE_CDI */ + + namespace CRIHelpersV1 { + +@@ -666,4 +670,79 @@ std::unique_ptr<runtime::v1::ContainerStatus> GetContainerStatus(service_executo + return contStatus; + } + ++#ifdef ENABLE_CDI ++static int InsertCDIDevices(std::unordered_set<std::string> &fromCRI, const std::string &devName, ++ string_array *requested, Errors &err) ++{ ++ if (fromCRI.find(devName) == fromCRI.end()) { ++ fromCRI.insert(devName); ++ if (util_append_string_array(requested, devName.c_str()) != 0) { ++ ERROR("Out of memory"); ++ err.Errorf("Out of memory"); ++ return -1; ++ } ++ DEBUG("Appended device: %s", devName.c_str()); ++ } else { ++ INFO("Skipping duplicate CDI device %s", devName.c_str()); ++ } ++ return 0; ++} ++ ++void GenerateCDIRequestedDevices(const runtime::v1::ContainerConfig &config, host_config *hostconfig, Errors &err) ++{ ++ std::unordered_set<std::string> fromCRI; ++ __isula_auto_string_array_t string_array *requested = nullptr; ++ __isula_auto_string_array_t string_array *keys = nullptr; ++ __isula_auto_string_array_t string_array *devices = nullptr; ++ json_map_string_string *annotations = nullptr; ++ __isula_auto_free char *error = nullptr; ++ ++ if (hostconfig == nullptr) { ++ ERROR("Invalid input arguments"); ++ err.Errorf("Invalid input arguments"); ++ return; ++ } ++ ++ if (config.cdi_devices().empty() && config.annotations().empty()) { ++ return; ++ } ++ requested = (string_array *)util_common_calloc_s(sizeof(*requested)); ++ if (requested == nullptr) { ++ ERROR("Out of memory"); ++ err.Errorf("Out of memory"); ++ return; ++ } ++ if (!config.cdi_devices().empty()) { ++ for (int i = 0; i < config.cdi_devices().size(); i++) { ++ if (InsertCDIDevices(fromCRI, config.cdi_devices(i).name(), requested, err) != 0) { ++ goto free_out; ++ } ++ } ++ } ++ if (!config.annotations().empty()) { ++ annotations = CRIHelpers::MakeAnnotations(config.annotations(), err); ++ if (err.NotEmpty()) { ++ goto free_out; ++ } ++ if (cdi_operate_parse_annotations(annotations, &keys, &devices, &error) != 0) { ++ ERROR("Failed to parse CDI annotations: %s", error); ++ err.Errorf("Failed to parse CDI annotations: %s", error); ++ goto free_out; ++ } ++ for (size_t i = 0; i < devices->len; i++) { ++ if (InsertCDIDevices(fromCRI, std::string(devices->items[i]), requested, err) != 0) { ++ goto free_out; ++ } ++ } ++ } ++ hostconfig->cdi_requested_devices = requested->items; ++ requested->items = nullptr; ++ hostconfig->cdi_requested_devices_len = requested->len; ++ requested->len = 0; ++ ++free_out: ++ free_json_map_string_string(annotations); ++} ++#endif /* ENABLE_CDI */ ++ + } // v1 namespace CRIHelpers +diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.h b/src/daemon/common/cri/v1/v1_cri_helpers.h +index 1578c428..22cffd0d 100644 +--- a/src/daemon/common/cri/v1/v1_cri_helpers.h ++++ b/src/daemon/common/cri/v1/v1_cri_helpers.h +@@ -79,6 +79,9 @@ std::string CRISandboxerConvert(const std::string &runtime); + + void ApplySandboxSecurityContextToHostConfig(const runtime::v1::LinuxSandboxSecurityContext &context, host_config *hc, + Errors &error); ++#ifdef ENABLE_CDI ++void GenerateCDIRequestedDevices(const runtime::v1::ContainerConfig &config, host_config *hostconfig, Errors &err); ++#endif /* ENABLE_CDI */ + + auto GetContainerStatus(service_executor_t *m_cb, const std::string &containerID, Errors &error) + -> std::unique_ptr<runtime::v1::ContainerStatus>; +diff --git a/src/daemon/config/daemon_arguments.c b/src/daemon/config/daemon_arguments.c +index 0ae6268a..ef15934a 100644 +--- a/src/daemon/config/daemon_arguments.c ++++ b/src/daemon/config/daemon_arguments.c +@@ -173,6 +173,10 @@ int service_arguments_init(struct service_arguments *args) + goto free_out; + } + ++#ifdef ENABLE_CDI ++ args->json_confs->enable_cdi = false; ++#endif /* ENABLE_CDI */ ++ + ret = 0; + + free_out: +diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c +index 778ff921..695a0d95 100644 +--- a/src/daemon/config/isulad_config.c ++++ b/src/daemon/config/isulad_config.c +@@ -1830,6 +1830,14 @@ int merge_json_confs_into_global(struct service_arguments *args) + args->json_confs->metrics_port = tmp_json_confs->metrics_port; + #endif + ++#ifdef ENABLE_CDI ++ args->json_confs->enable_cdi = tmp_json_confs->enable_cdi; ++ args->json_confs->cdi_spec_dirs = tmp_json_confs->cdi_spec_dirs; ++ tmp_json_confs->cdi_spec_dirs = NULL; ++ args->json_confs->cdi_spec_dirs_len = tmp_json_confs->cdi_spec_dirs_len; ++ tmp_json_confs->cdi_spec_dirs_len = 0; ++#endif /* ENABLE_CDI */ ++ + out: + free(err); + free_isulad_daemon_configs(tmp_json_confs); +diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +index e86dafae..1097c32c 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +@@ -199,6 +199,14 @@ auto ContainerManagerService::GenerateCreateContainerHostConfig( + } + } + ++#ifdef ENABLE_CDI ++ CRIHelpersV1::GenerateCDIRequestedDevices(containerConfig, hostconfig, error); ++ if (error.NotEmpty()) { ++ ERROR("Failed to generate CDI requested devices"); ++ goto cleanup; ++ } ++#endif /* ENABLE_CDI */ ++ + return hostconfig; + + cleanup: +diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c +index a9102226..785b4e27 100644 +--- a/src/daemon/executor/container_cb/execution_create.c ++++ b/src/daemon/executor/container_cb/execution_create.c +@@ -63,6 +63,7 @@ + #include "runtime_api.h" + #include "id_name_manager.h" + #include "mailbox.h" ++#include "specs_mount.h" + + #ifdef ENABLE_CRI_API_V1 + static bool validate_sandbox_info(const container_sandbox_info *sandbox) +@@ -512,6 +513,14 @@ static oci_runtime_spec *generate_oci_config(host_config *host_spec, const char + goto error_out; + } + ++#ifdef ENABLE_CDI ++ ret = inject_CDI_devcies_for_oci_spec(oci_spec, host_spec); ++ if (ret != 0) { ++ ERROR("Failed to inject CDI devices"); ++ goto error_out; ++ } ++#endif /* ENABLE_CDI */ ++ + return oci_spec; + + error_out: +diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c +index eb7ce4f4..b19a134a 100644 +--- a/src/daemon/modules/service/service_container.c ++++ b/src/daemon/modules/service/service_container.c +@@ -2003,6 +2003,16 @@ static defs_process *make_exec_process_spec(const container_config *container_sp + } + + spec->no_new_privileges = oci_spec->process->no_new_privileges; ++ ++#ifdef ENABLE_CDI ++ // extend step: merge env from oci_spec which comes from injected devices ++ ret = defs_process_add_multiple_env(spec, (const char **)oci_spec->process->env, ++ oci_spec->process->env_len); ++ if (ret != 0) { ++ ERROR("Failed to dup oci env for exec process spec"); ++ goto err_out; ++ } ++#endif /* ENABLE_CDI */ + } + + // for oci runtime: +diff --git a/src/daemon/modules/spec/specs_mount.c b/src/daemon/modules/spec/specs_mount.c +index 50ee9a85..12bd261b 100644 +--- a/src/daemon/modules/spec/specs_mount.c ++++ b/src/daemon/modules/spec/specs_mount.c +@@ -28,6 +28,7 @@ + #include <isula_libutils/container_config_v2.h> + #include <isula_libutils/json_common.h> + #include <isula_libutils/oci_runtime_config_linux.h> ++#include <isula_libutils/auto_cleanup.h> + #include <limits.h> + #include <stdint.h> + +@@ -54,6 +55,9 @@ + #include "volume_api.h" + #include "parse_volume.h" + #include "specs_api.h" ++#ifdef ENABLE_CDI ++#include "cdi_operate_api.h" ++#endif /* ENABLE_CDI */ + + enum update_rw { + update_rw_untouch, +@@ -3582,6 +3586,15 @@ int update_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostcon + oci_spec->linux->resources->devices_len += 1; + } + ++ // extend step: inject CDI devcies ++#ifdef ENABLE_CDI ++ ret = inject_CDI_devcies_for_oci_spec(oci_spec, hostconfig); ++ if (ret != 0) { ++ ERROR("Failed to inject CDI devices"); ++ return -1; ++ } ++#endif /* ENABLE_CDI */ ++ + // Step8: do update devices and cgroup device rules at here + if (hostconfig->privileged) { + // Step8.1: for priviledged container, we should merge all devices under /dev +@@ -3592,4 +3605,32 @@ int update_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostcon + } + + return ret; +-} +\ No newline at end of file ++} ++ ++#ifdef ENABLE_CDI ++int inject_CDI_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostconfig) ++{ ++ int ret = 0; ++ string_array devices_array = { 0 }; ++ __isula_auto_free char *error = NULL; ++ ++ if (oci_spec == NULL || hostconfig == NULL) { ++ ERROR("Invalid params"); ++ return -1; ++ } ++ if (hostconfig->cdi_requested_devices == NULL) { ++ return 0; ++ } ++ devices_array.items = hostconfig->cdi_requested_devices; ++ devices_array.len = hostconfig->cdi_requested_devices_len; ++ devices_array.cap = hostconfig->cdi_requested_devices_len; ++ if (cdi_operate_refresh() != 0) { ++ WARN("CDI registry has errors, please check past logs"); ++ } ++ if (cdi_operate_inject_devices(oci_spec, &devices_array) != 0) { ++ ERROR("Failed to inject CDI devices"); ++ ret = -1; ++ } ++ return ret; ++} ++#endif /* ENABLE_CDI */ +\ No newline at end of file +diff --git a/src/daemon/modules/spec/specs_mount.h b/src/daemon/modules/spec/specs_mount.h +index b742ca35..1406c557 100644 +--- a/src/daemon/modules/spec/specs_mount.h ++++ b/src/daemon/modules/spec/specs_mount.h +@@ -49,6 +49,10 @@ int setup_ipc_dirs(host_config *host_spec, container_config_v2_common_config *v2 + + int update_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostconfig); + ++#ifdef ENABLE_CDI ++int inject_CDI_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostconfig); ++#endif /* ENABLE_CDI */ ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/modules/spec/verify.c b/src/daemon/modules/spec/verify.c +index af790d6e..617b7f23 100644 +--- a/src/daemon/modules/spec/verify.c ++++ b/src/daemon/modules/spec/verify.c +@@ -1518,7 +1518,7 @@ static int verify_custom_mount(defs_mount **mounts, size_t len) + + for (i = 0; i < len; ++i) { + iter = *(mounts + i); +- if (iter == NULL || strcmp(iter->type, MOUNT_TYPE_BIND)) { ++ if (iter == NULL || iter->type == NULL || strcmp(iter->type, MOUNT_TYPE_BIND)) { + continue; + } + +-- +2.34.1 + diff --git a/0069-bugfix-fix-cni_operate_ut-ut.patch b/0069-bugfix-fix-cni_operate_ut-ut.patch new file mode 100644 index 0000000..538fcff --- /dev/null +++ b/0069-bugfix-fix-cni_operate_ut-ut.patch @@ -0,0 +1,24 @@ +From a1f75fd089309d0f8620195ce7e517294be2c410 Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Fri, 19 Apr 2024 18:37:05 +0800 +Subject: [PATCH 69/69] bugfix:fix cni_operate_ut ut + +--- + test/network/cni_operate/CMakeLists.txt | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/test/network/cni_operate/CMakeLists.txt b/test/network/cni_operate/CMakeLists.txt +index 5b4d7c7d..752e5199 100644 +--- a/test/network/cni_operate/CMakeLists.txt ++++ b/test/network/cni_operate/CMakeLists.txt +@@ -14,6 +14,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_string.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_array.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_regex.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_version.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/namespace.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/sha256/sha256.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/path.c +-- +2.34.1 + diff --git a/0070-isolate-sandboxer-code-by-using-macro.patch b/0070-isolate-sandboxer-code-by-using-macro.patch new file mode 100644 index 0000000..4ab8cb2 --- /dev/null +++ b/0070-isolate-sandboxer-code-by-using-macro.patch @@ -0,0 +1,143 @@ +From c1d445e178cd610f8a6d9156012c6c7922eed9c5 Mon Sep 17 00:00:00 2001 +From: xuxuepeng <xuxuepeng1@huawei.com> +Date: Sat, 20 Apr 2024 11:24:18 +0800 +Subject: [PATCH 1/2] isolate sandboxer code by using macro + +Signed-off-by: xuxuepeng <xuxuepeng1@huawei.com> +--- + cmake/options.cmake | 2 +- + src/daemon/common/cri/v1/v1_cri_helpers.cc | 7 +++++++ + src/daemon/config/isulad_config.c | 2 ++ + src/daemon/sandbox/controller/CMakeLists.txt | 2 +- + src/daemon/sandbox/controller/controller_manager.cc | 6 ++++++ + src/daemon/sandbox/controller/controller_manager.h | 2 ++ + 6 files changed, 19 insertions(+), 2 deletions(-) + +diff --git a/cmake/options.cmake b/cmake/options.cmake +index c1eac472..a15b8194 100644 +--- a/cmake/options.cmake ++++ b/cmake/options.cmake +@@ -51,7 +51,7 @@ if (ENABLE_CDI STREQUAL "ON") + endif() + endif() + +-option(ENABLE_SANDBOXER "Enable sandbox API" ON) ++option(ENABLE_SANDBOXER "Enable sandbox API" OFF) + if (ENABLE_SANDBOXER STREQUAL "ON") + add_definitions(-DENABLE_SANDBOXER) + set(ENABLE_SANDBOXER 1) +diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.cc b/src/daemon/common/cri/v1/v1_cri_helpers.cc +index 520d23d4..1f797ad7 100644 +--- a/src/daemon/common/cri/v1/v1_cri_helpers.cc ++++ b/src/daemon/common/cri/v1/v1_cri_helpers.cc +@@ -391,6 +391,7 @@ void GetContainerSandboxID(const std::string &containerID, std::string &realCont + realContainerID = info->id; + } + ++#ifdef ENABLE_SANDBOXER + std::string CRISandboxerConvert(const std::string &runtime) + { + std::string sandboxer; +@@ -429,6 +430,12 @@ out: + (void)isulad_server_conf_unlock(); + return sandboxer; + } ++#else ++std::string CRISandboxerConvert(const std::string &runtime) ++{ ++ return DEFAULT_SANDBOXER_NAME; ++} ++#endif + + void ApplySandboxSecurityContextToHostConfig(const runtime::v1::LinuxSandboxSecurityContext &context, host_config *hc, + Errors &error) +diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c +index 695a0d95..617db7a2 100644 +--- a/src/daemon/config/isulad_config.c ++++ b/src/daemon/config/isulad_config.c +@@ -1757,8 +1757,10 @@ int merge_json_confs_into_global(struct service_arguments *args) + args->json_confs->runtimes = tmp_json_confs->runtimes; + tmp_json_confs->runtimes = NULL; + #ifdef ENABLE_CRI_API_V1 ++#ifdef ENABLE_SANDBOXER + args->json_confs->cri_sandboxers = tmp_json_confs->cri_sandboxers; + tmp_json_confs->cri_sandboxers = NULL; ++#endif + args->json_confs->enable_cri_v1 = tmp_json_confs->enable_cri_v1; + args->json_confs->enable_pod_events = tmp_json_confs->enable_pod_events; + #endif +diff --git a/src/daemon/sandbox/controller/CMakeLists.txt b/src/daemon/sandbox/controller/CMakeLists.txt +index f846657a..8764c05b 100644 +--- a/src/daemon/sandbox/controller/CMakeLists.txt ++++ b/src/daemon/sandbox/controller/CMakeLists.txt +@@ -9,7 +9,7 @@ set(local_sandbox_controller_top_incs + ${CMAKE_CURRENT_SOURCE_DIR} + ) + +-if (ENABLE_SANDBOXER) ++if (ENABLE_CRI_API_V1 AND ENABLE_SANDBOXER) + add_subdirectory(sandboxer) + list (APPEND local_sandbox_controller_top_srcs + ${CONTROLLER_SANDBOXER_SRCS} +diff --git a/src/daemon/sandbox/controller/controller_manager.cc b/src/daemon/sandbox/controller/controller_manager.cc +index 21c6f5fe..91c98d26 100644 +--- a/src/daemon/sandbox/controller/controller_manager.cc ++++ b/src/daemon/sandbox/controller/controller_manager.cc +@@ -20,7 +20,9 @@ + #include <isula_libutils/defs.h> + + #include "shim_controller.h" ++#ifdef ENABLE_SANDBOXER + #include "sandboxer_controller.h" ++#endif + #include "isulad_config.h" + #include "daemon_arguments.h" + +@@ -44,10 +46,12 @@ bool ControllerManager::Init(Errors &error) + return false; + } + ++#ifdef ENABLE_SANDBOXER + // Initialize sandboxer controller + if (!RegisterAllSandboxerControllers(error)) { + return false; + } ++#endif + return true; + } + +@@ -75,6 +79,7 @@ auto ControllerManager::RegisterShimController(Errors &error) -> bool + return true; + } + ++#ifdef ENABLE_SANDBOXER + auto ControllerManager::RegisterAllSandboxerControllers(Errors &error) -> bool + { + std::map<std::string, std::string> config; +@@ -160,6 +165,7 @@ auto ControllerManager::RegisterSandboxerController(const std::string &sandboxer + INFO("Sandboxer controller initialized successfully, sandboxer: %s", sandboxer.c_str()); + return true; + } ++#endif + + auto ControllerManager::GetController(const std::string &name) -> std::shared_ptr<Controller> + { +diff --git a/src/daemon/sandbox/controller/controller_manager.h b/src/daemon/sandbox/controller/controller_manager.h +index 28b52c2f..3fd547cf 100644 +--- a/src/daemon/sandbox/controller/controller_manager.h ++++ b/src/daemon/sandbox/controller/controller_manager.h +@@ -31,9 +31,11 @@ public: + auto GetController(const std::string &name) -> std::shared_ptr<Controller>; + private: + auto RegisterShimController(Errors &error) -> bool; ++#ifdef ENABLE_SANDBOXER + auto RegisterAllSandboxerControllers(Errors &error) -> bool; + auto LoadSandboxerControllersConfig(std::map<std::string, std::string> &config) -> bool; + auto RegisterSandboxerController(const std::string &sandboxer, const std::string &address, Errors &error) -> bool; ++#endif + + protected: + std::map<std::string, std::shared_ptr<Controller>> m_controllers; +-- +2.34.1 + diff --git a/0071-Remove-sandboxer-ut-if-sandboxer-is-not-enabled.patch b/0071-Remove-sandboxer-ut-if-sandboxer-is-not-enabled.patch new file mode 100644 index 0000000..dbfbe5d --- /dev/null +++ b/0071-Remove-sandboxer-ut-if-sandboxer-is-not-enabled.patch @@ -0,0 +1,28 @@ +From 7c0c79e8ad4680f97651dd52721344961c803c15 Mon Sep 17 00:00:00 2001 +From: xuxuepeng <xuxuepeng1@huawei.com> +Date: Sat, 20 Apr 2024 11:48:56 +0800 +Subject: [PATCH 71/73] Remove sandboxer ut if sandboxer is not enabled + +Signed-off-by: xuxuepeng <xuxuepeng1@huawei.com> +--- + test/sandbox/CMakeLists.txt | 8 +++++--- + 1 file changed, 5 insertions(+), 3 deletions(-) + +diff --git a/test/sandbox/CMakeLists.txt b/test/sandbox/CMakeLists.txt +index 68397477..38d7ccb9 100644 +--- a/test/sandbox/CMakeLists.txt ++++ b/test/sandbox/CMakeLists.txt +@@ -1,5 +1,7 @@ + project(iSulad_UT) + +-add_subdirectory(controller) +-add_subdirectory(sandbox_manager) +-add_subdirectory(sandbox) ++if (ENABLE_SANDBOXER) ++ add_subdirectory(controller) ++ add_subdirectory(sandbox_manager) ++ add_subdirectory(sandbox) ++endif() +-- +2.34.1 + diff --git a/0072-cdi-design-doc.patch b/0072-cdi-design-doc.patch new file mode 100644 index 0000000..78c3116 --- /dev/null +++ b/0072-cdi-design-doc.patch @@ -0,0 +1,391 @@ +From 72246a9e83ada3af7560817534013a93f40d5bae Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Sat, 20 Apr 2024 11:46:11 +0800 +Subject: [PATCH 72/73] cdi:design doc + +Signed-off-by: liuxu <liuxu156@huawei.com> +--- + docs/design/README.md | 3 + + docs/design/README_zh.md | 3 + + docs/design/detailed/CDI/cdi_design_zh.md | 341 ++++++++++++++++++++++ + 3 files changed, 347 insertions(+) + create mode 100644 docs/design/detailed/CDI/cdi_design_zh.md + +diff --git a/docs/design/README.md b/docs/design/README.md +index cf29c0a1..d2a3702d 100644 +--- a/docs/design/README.md ++++ b/docs/design/README.md +@@ -18,6 +18,9 @@ This section contains some design documents for users who want to learn more abo + + - You can see how the CRI startup process is refactored in [cri_cni_refactor](./detailed/CRI/cri_cni_refactor.md). + ++## CDI ++- You can see how the CDI is refactored in [cdi_design](./detailed/CDI/cdi_design_zh.md.md)。 ++ + ## Events + + - You can see how the events modules are designed in [events_design](./detailed/Events/events_design.md). +diff --git a/docs/design/README_zh.md b/docs/design/README_zh.md +index 3382bfbe..c6172b6f 100644 +--- a/docs/design/README_zh.md ++++ b/docs/design/README_zh.md +@@ -24,6 +24,9 @@ + + - 查看 CRI的启动程序的重构文档： [cri_cni_refactor](./detailed/CRI/cri_cni_refactor_zh.md) 。 + ++## CDI ++- 查看 CDI 的设计文档： [cdi_design](./detailed/CDI/cdi_design_zh.md.md)。 ++ + ## Events + + - 查看 events 模块的设计文档： [events_design](./detailed/Events/events_design_zh.md) 。 +diff --git a/docs/design/detailed/CDI/cdi_design_zh.md b/docs/design/detailed/CDI/cdi_design_zh.md +new file mode 100644 +index 00000000..15c88f93 +--- /dev/null ++++ b/docs/design/detailed/CDI/cdi_design_zh.md +@@ -0,0 +1,341 @@ ++| Author | liuxu | ++| ------ | --------------------- | ++| Date | 2024-02-27 | ++| Email | liuxu156@huawei.com | ++ ++# 背景介绍 ++## What is CDI? ++CDI（容器设备接口）是容器运行时的一种规范，用于支持第三方设备。 ++ ++它引入了device作为资源的抽象概念。device由完全限定的名称唯一指定，该名称由vendor ID、一个device class和在每个vendor ID-device class对中唯一的name构成。 ++``` ++vendor.com/class=unique_name ++``` ++vendor ID和device class （上例中的vendor.com/class）的组合称为device kind. ++ ++CDI只关心使容器能够感知设备。CDI明确地忽略了诸如资源管理之类的领域（并期望由编排器处理）。由于这个原因，CDI规范实现起来很简单，并且为运行时和编排器提供了极大的灵活性。 ++ ++注：CDI模型基于容器网络接口（CNI）模型和规范。 ++ ++## Why is CDI needed? ++在Linux上，使容器具有设备感知能力过去只需在该容器中暴露一个设备节点。但是，随着设备和软件变得越来越复杂，供应商希望执行更多的操作，例如： ++- 向容器公开设备可能需要公开多个设备节点、从运行时命名空间挂载文件或隐藏procfs条目。 ++- 执行容器和设备之间的兼容性检查（例如：检查容器是否可以在指定设备上运行）。 ++- 执行特定于运行时的操作（例如：虚拟机与基于Linux容器的运行时）。 ++- 执行特定于设备的操作（例如：清理GPU的内存或重新配置FPGA）。 ++ ++在缺乏第三方设备标准的情况下，供应商通常不得不为不同的运行时编写和维护多个插件，甚至直接在运行时中贡献特定于供应商的代码。此外，运行时不统一地暴露插件系统（甚至根本不暴露插件系统），导致在更高级别的抽象（例如Kubernetes设备插件）中重复功能。 ++ ++## How does CDI work? ++要使CDI正常工作，需要完成以下操作： ++- JSON格式的CDI文件应位于CDI规范目录中，它的作用是更新OCI spec。默认目录为/etc/cdi和/var/run/cdi ++- 应使用CRI的annotations与CDI_devices（本次特性支持）将唯一的设备名称传递给运行时 ++- 容器运行时应该能够通过设备名称找到CDI文件在内存中对应的缓存信息，并使用缓存的内容更新容器配置。 ++ ++## How to configure CDI? ++### iSulad ++daemon.json中开启cri-v1和cdi配置： ++ ++当 cdi-spec-dirs 不指定时，默认为"/etc/cdi", "/var/run/cdi" ++```json ++"enable-cri-v1": true, ++"cdi-spec-dirs": ["/etc/cdi", "/var/run/cdi"], # 指定CDI规范所在目录 ++"enable-cdi": true # 打开CDI特性 ++``` ++ ++在CRI创建容器的参数中使用CDI，以下两种方式均可： ++1. annotations中指定设备 ++```json ++{ ++ ... ... ++ "annotations": [ ++ ... ... ++ // key值格式要求含有cdi.k8s.io作为前缀，后面跟随pluginName ++ {"cdi.k8s.io/test": "vendor.com/device=myDevice"}, ++ ... ... ++ ] ++ ... ... ++} ++``` ++2. CDI_Devices中指定设备 ++```json ++{ ++ ... ... ++ "CDI_Devices": [ ++ ... ... ++ {"Name": "vendor.com/device=myDevice"}, ++ ... ... ++ ] ++ ... ... ++} ++``` ++ ++# 方案目标 ++## 概述 ++容器设备接口（Container Device Interface，简称CDI）描述了容器运行时创建能够与第三方设备交互的容器的机制。 ++ ++对于第三方设备，与这些设备进行交互通常需要容器运行时公开多个设备节点。例如，第三方设备可能需要加载内核模块、装载主机库、暴露/屏蔽特定procfs路径。 ++ ++容器设备接口描述了一种允许第三方供应商执行这些操作的机制，从而不需要更改容器运行时。 ++ ++使用的机制是一个JSON文件（类似于容器网络接口(CNI）)，它允许供应商描述容器运行时应该对容器的OCI规范执行的操作。 ++ ++CDI 支持以下两个流程： ++ ++A.设备安装 ++1. 用户在机器上安装第三方设备驱动程序（和第三方设备/被测试的设备）。 ++2. 设备驱动程序安装软件会在一个已知路径（/etc/cdi/vendor.json）上写入一个 JSON 文件。 ++ ++B.容器运行时 ++1. 用户在创建容器时，CRI中指定设备名称 ++2. 容器运行时会读取 JSON 文件。 ++3. 容器运行时会验证 JSON 文件中是否描述了设备。 ++4. 容器运行时会根据 JSON 文件中的指令转换 OCI 规范，并插入OCI Spec中 ++# 总体设计 ++**iSulad支持CDI功能目前仅支持CRI方式调用，CLI方式暂不支持** ++## 整体结构 ++ ++```mermaid ++flowchart TD ++classDef unFinish fill:#c19,stroke:#216,stroke-width:2px,color:#fff,stroke-dasharray: 5 5; ++subgraph isulad ++ subgraph CDIOperate ++ RegistryOps ++ AnnotationsOps ++ ...... ++ end ++ ++ subgraph CDI ++ D[ContainerEdits] ++ E[Device] ++ F[Registry] ++ G[Cache] ++ H[Spec] ++ end ++ CDIOperate --> CDI ++ ++ OA[isula module] --> |call| CDIOperate ++ OB[CRI module] --> |call| CDIOperate ++ CDIOperate:::unFinish ++ CDI:::unFinish ++end ++ PA[CLI] --> OA ++ PB[CRI] --> OB ++``` ++ ++- CDIOperate封装了CDI模块，对外提供更合理的CDI功能的相关接口。 ++- CDI负责实现CDI Specs的读取、校验、解析、devices注入OCI Spec等具体功能。 ++## 时序设计 ++在isulad启动后，以创建一个容器为例。 ++ ++图中isulad启动后，拉起一个新的线程isulad-cdi_watcher，负责监控cdi-spec-dirs，当cdi-spec-dirs中的cdi Spec文件发生修改、删除等动作时，重新扫描cdi-spec-dirs中的cdi Spec文件。 ++ ++```mermaid ++sequenceDiagram ++ participant CRI ++ participant isulad ++ participant cdi_watcher ++ isulad ->> cdi_watcher: init cdi ++ par ++ loop ++ cdi_watcher ->> cdi_watcher:wait for inotify event ++ cdi_watcher ->> cdi_watcher:refresh cdi cache ++ end ++ and ++ opt CRI ++ CRI ->> isulad: create container ++ isulad ->> isulad:parse cdi annotations/CDI_devices ++ isulad ->> isulad:generates an OCI specification with cdi info ++ end ++ end ++``` ++ ++# 接口描述 ++## 3.1 结构体和常量说明 ++ ++```c ++// 实现CDI Spec规范中的json字段，参考 https://github1s.com/containerd/containerd/blob/main/vendor/tags.cncf.io/container-device-interface/specs-go/config.go#L9 ++typedef struct {} cdi_spec; ++typedef struct {} cdi_spec_device; ++typedef struct {} cdi_spec_container_edits; ++typedef struct {} cdi_spec_device_node; ++typedef struct {} cdi_spec_mount; ++typedef struct {} cdi_spec_hook; ++ ++struct cdi_cache_device { ++ const cdi_device *raw_device; ++ const struct cdi_cache_spec *cache_spec; ++}; ++ ++struct cdi_cache_spec { ++ cdi_spec *raw_spec; ++ char *vendor; ++ char *class; ++ char *path; ++ int priority; ++ map_t *devices; // MAP_STR_PTR devices[cdi_device.name] = cdi_cache_device* ++}; ++ ++struct cdi_cache_ops { ++ // injecting CDI devices into an OCI Spec. ++ // Resolver ++ int (*inject_devices)(struct cdi_cache *c, oci_runtime_spec *spec, string_array *devices); ++ ++ // refreshing the cache of CDI Specs and devices. ++ // Refresher ++ int (*configure)(struct cdi_cache *c, string_array *spec_dirs); ++ int (*refresh)(struct cdi_cache *c); ++}; ++ ++struct cdi_watch { ++ int watcher_fd; // inotify fd ++ map_t *tracked; // MAP_STR_BOOL tracked[spec_dirs[i]] = bool ++ map_t *wd_dirs; // MAP_INT_STR wd_dirs[wd] = spec_dirs[i] ++}; ++ ++// Cache stores CDI Specs loaded from Spec directories. ++struct cdi_cache { ++ pthread_mutex_t mutex; ++ string_array *spec_dirs; // cdi-spec-dirs will scan for CDI Spec files ++ map_t *specs; // MAP_STR_PTR specs[vendor] = common_array of cdi_cache_spec* ++ // This map holding the reference to cdi device, the devices will not released when the map is freed. ++ map_t *devices; // MAP_STR_PTR devices[cdi_device.name] = cdi_cache_device* ++ bool refresh_error_flag; ++ bool auto_refresh; ++ struct cdi_watch *watch; ++}; ++ ++struct cdi_registry { ++ struct cdi_cache *cdi_cache; ++ struct cdi_cache_ops *ops; ++}; ++``` ++涉及修改现有的结构体： ++```c ++// 用于从CRI向executor传递devices数据 ++typedef struct { ++ ... ... ++ char **cdi_requested_devices; ++ size_t cdi_requested_devices_len; ++} host_config; ++ ++// isulad的daemon.json增加cdi相关的基本配置 ++typedef struct { ++ ... ... ++ char **cdi_spec_dirs; ++ size_t cdi_spec_dirs_len; ++ bool enable_cdi; ++} isulad_daemon_configs; ++``` ++ ++## 3.2 接口说明 ++CDIOperate的设计目标：使得CDI规范有关的内容不过多的对外暴露，降低CDI模块和外部的耦合。 ++### RegistryOps ++```c ++int cdi_operate_registry_init(char **specs_dirs, size_t specs_dirs_len); ++ ++int cdi_operate_refresh(void); ++ ++int cdi_operate_inject_devices(oci_runtime_spec *spec, string_array *devices); ++ ++int cdi_operate_parse_annotations(json_map_string_string *annotations, string_array **keys, ++ string_array **devices, char **error); ++``` ++ ++# 详细设计 ++## daemon.json ++daemon.json中增加cdi配置： ++```json ++"cdi-spec-dirs": ["/etc/cdi", "/var/run/cdi"], ++"enable-cdi": true ++``` ++## CDIOperate被调用点 ++### CreateContainer ++创建新的Container时需要将devices插入OCI Spec。 ++本次开发暂不涉及isula支持CDI。 ++ ++为什么已经有isulad-cdi_watcher线程了，在create的时候还需要refresh? ++1. isulad-cdi_watcher在触发inotify event后，如果为Rename、Remove、Write事件，会直接执行refresh，将重新扫描cdi-spec-dirs中的CDI Specs到内存。 ++2. 而在create container时，cdi_refresh先检查tracked，tracked标记了是否所有目录都已被跟踪，如果不是才会触发refresh。tracked在cache生成时初始化为未跟踪。 ++ ++```mermaid ++flowchart TD ++O((begin)) ++A(ContainerManagerService::CreateContainer <br> CRI创建容器) ++B(ContainerManagerService::CDIParseDevices <br> 解析CDI有关的字段) ++C(container_cb::container_create_cb <br> 创建容器的前置准备) ++D(spec::merge_all_specs <br> 准备生成OCI Spec) ++E(spec::spec_inject_cdi_devices <br> 解析需要的设备名) ++F(CDIOperate::cdi_refresh <br> 刷新cdi的缓存) ++G(CDIOperate::cdi_inject_devices <br> 将容器需要的设备的相关信息插入OCI Spec) ++Z((create end)) ++O --> A --> B --> C --> D --> E --> F --> G --> Z ++``` ++ ++ContainerManagerService::CDIParseDevices 需要支持两种CDI devices解析方式： ++1. CDIOperate::cdi_parse_annotations 解析ContainerConfig.annotations() ++2. 解析ContainerConfig.CDI_devices() ++解析后放入container_create_request ++### main ++isulad 启动时，读取所有CDI Specs，初始化cache ++```mermaid ++flowchart TD ++O((begin)) ++A(main) ++B(isulad_server_init_common <br> isulad服务端初始化) ++C(cdi::cdi_registry_init <br> 读取现有的CDI Specs <br> 并拉起新的线程监控CDI Specs的变化) ++Z(end) ++O --> A --> B --> C --> Z ++``` ++ ++## CDI 模块设计 ++```mermaid ++flowchart TD ++ subgraph CDIoperate ++ operate ++ end ++ ++ subgraph CDI ++ annotations ++ registry ++ cache ++ subgraph behavior ++ spec-dirs ++ spec ++ device ++ container-edits ++ version ++ parser ++ end ++ end ++ ++ operate --> registry ++ operate --> annotations ++ registry --> cache ++ cache --> spec-dirs ++ cache --> device ++ cache --> spec ++ cache --> container-edits ++ device <--> spec ++ device --> parser ++ device --> container-edits ++ device --> parser ++ spec --> container-edits ++ spec --> version ++ version --> parser ++``` ++ ++### new cache ++```mermaid ++flowchart TD ++O((begin)) ++A(ContainerManagerService::CreateContainer <br> CRI创建容器) ++B(ContainerManagerService::CDIParseDevices <br> 解析CDI有关的字段) ++C(container_cb::container_create_cb <br> 创建容器的前置准备) ++D(spec::merge_all_specs <br> 准备生成OCI Spec) ++E(spec::spec_inject_cdi_devices <br> 解析需要的设备名) ++F(CDIOperate::cdi_refresh <br> 刷新cdi的缓存) ++G(CDIOperate::cdi_inject_devices <br> 将容器需要的设备的相关信息插入OCI Spec) ++Z((create end)) ++O --> A --> B --> C --> D --> E --> F --> G --> Z ++``` +-- +2.34.1 + diff --git a/0073-bugfix-cdi-version-check.patch b/0073-bugfix-cdi-version-check.patch new file mode 100644 index 0000000..00c1b9a --- /dev/null +++ b/0073-bugfix-cdi-version-check.patch @@ -0,0 +1,52 @@ +From 4d8a89e0a3700253db044f6641d91ab10ad6ce10 Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Sat, 20 Apr 2024 19:36:55 +0800 +Subject: [PATCH 73/73] bugfix: cdi version check + +Signed-off-by: liuxu <liuxu156@huawei.com> +--- + src/daemon/modules/device/cdi/behavior/cdi_spec.c | 2 +- + src/daemon/modules/device/cdi/behavior/cdi_version.c | 6 +++--- + 2 files changed, 4 insertions(+), 4 deletions(-) + +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec.c b/src/daemon/modules/device/cdi/behavior/cdi_spec.c +index 235b1863..8783debc 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_spec.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec.c +@@ -199,7 +199,7 @@ static int cdi_spec_init(struct cdi_cache_spec *s) + return -1; + } + if (version_result) { +- ERROR("The spec version must be at least v%s", min_version); ++ ERROR("The %s spec version must be at least v%s", s->path, min_version); + return -1; + } + +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_version.c b/src/daemon/modules/device/cdi/behavior/cdi_version.c +index 882a965e..550f3107 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_version.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_version.c +@@ -138,9 +138,9 @@ static struct required_version_map g_valid_spec_versions[VALID_SPEC_VERSIONS_LEN + {CDI_V010, NULL}, + {CDI_V020, NULL}, + {CDI_V030, NULL}, +- {CDI_V040, requires_v060}, ++ {CDI_V040, requires_v040}, + {CDI_V050, requires_v050}, +- {CDI_V060, requires_v040} ++ {CDI_V060, requires_v060} + }; + + const char *cdi_minimum_required_version(cdi_spec *spec) +@@ -166,7 +166,7 @@ const char *cdi_minimum_required_version(cdi_spec *spec) + min_version = g_valid_spec_versions[i].version; + } + } +- if (strcmp(min_version, CDI_CURRENT_VERSION)) { ++ if (strcmp(min_version, CDI_CURRENT_VERSION) == 0) { + break; + } + } +-- +2.34.1 + diff --git a/0074-bugfix-of-background-execution-exec-error-command.patch b/0074-bugfix-of-background-execution-exec-error-command.patch new file mode 100644 index 0000000..6aca398 --- /dev/null +++ b/0074-bugfix-of-background-execution-exec-error-command.patch @@ -0,0 +1,33 @@ +From 2c86e55d98b0d62c534ff5810c1eb1d327d6425a Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Tue, 23 Apr 2024 17:44:00 +1400 +Subject: [PATCH 74/75] bugfix of background execution exec error command + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/modules/runtime/isula/isula_rt_ops.c | 9 +++++++++ + 1 file changed, 9 insertions(+) + +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 47a14b1d..854752ea 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -1099,6 +1099,15 @@ static int get_container_process_pid(const char *workdir) + util_usleep_nointerupt(100000); + continue; + } ++ // If isulad does not read the container process pid, but isulad-shim reads the pid, ++ // and the container process exits, isulad-shim exits accordingly. ++ // At this time, exec should return true, because the container process has been created successfully ++ // and exec is successful, just because The process executes too fast causing isulad to not be read correctly ++ file_read_int(fname, &pid); ++ if (pid != 0) { ++ DEBUG("Process exit and isulad-shim exit"); ++ return pid; ++ } + ERROR("failed read pid from dead shim %s", workdir); + return -1; + } +-- +2.25.1 + diff --git a/0075-bugfix-for-setting-cpu-rt-to-a-negative-value-when-e.patch b/0075-bugfix-for-setting-cpu-rt-to-a-negative-value-when-e.patch new file mode 100644 index 0000000..7ea77a4 --- /dev/null +++ b/0075-bugfix-for-setting-cpu-rt-to-a-negative-value-when-e.patch @@ -0,0 +1,35 @@ +From bd18051dade97d4f75346cb67beea551a38ca13e Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Tue, 23 Apr 2024 20:52:04 +0800 +Subject: [PATCH 75/75] bugfix for setting cpu-rt to a negative value when env + not supports cpu-rt + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/modules/spec/verify.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/src/daemon/modules/spec/verify.c b/src/daemon/modules/spec/verify.c +index 617b7f23..57e16ef9 100644 +--- a/src/daemon/modules/spec/verify.c ++++ b/src/daemon/modules/spec/verify.c +@@ -381,14 +381,14 @@ static int verify_cpu_realtime(const sysinfo_t *sysinfo, int64_t realtime_period + { + int ret = 0; + +- if (realtime_period > 0 && !(sysinfo->cgcpuinfo.cpu_rt_period)) { ++ if (realtime_period != 0 && !(sysinfo->cgcpuinfo.cpu_rt_period)) { + ERROR("Invalid --cpu-rt-period: Your kernel does not support cgroup rt period"); + isulad_set_error_message("Invalid --cpu-rt-period: Your kernel does not support cgroup rt period"); + ret = -1; + goto out; + } + +- if (realtime_runtime > 0 && !(sysinfo->cgcpuinfo.cpu_rt_runtime)) { ++ if (realtime_runtime != 0 && !(sysinfo->cgcpuinfo.cpu_rt_runtime)) { + ERROR("Invalid --cpu-rt-runtime: Your kernel does not support cgroup rt runtime"); + isulad_set_error_message("Invalid --cpu-rt-period: Your kernel does not support cgroup rt runtime"); + ret = -1; +-- +2.25.1 + diff --git a/0076-cdi-add-UT.patch b/0076-cdi-add-UT.patch new file mode 100644 index 0000000..678b578 --- /dev/null +++ b/0076-cdi-add-UT.patch @@ -0,0 +1,741 @@ +From 0cd088174c94c56ee86506dab9a6a33f6e8fdaa4 Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Thu, 25 Apr 2024 10:52:20 +0800 +Subject: [PATCH 76/78] cdi:add UT + +Signed-off-by: liuxu <liuxu156@huawei.com> +--- + CI/make-and-install.sh | 4 +- + test/cutils/CMakeLists.txt | 1 + + test/cutils/utils_array/utils_array_ut.cc | 128 ++++++++++ + test/cutils/utils_utils/utils_utils_ut.cc | 12 + + test/cutils/utils_version/CMakeLists.txt | 17 ++ + test/cutils/utils_version/utils_version_ut.cc | 71 ++++++ + .../image/oci/oci_config_merge/CMakeLists.txt | 1 + + test/mocks/cdi_operate_api_mock.cc | 58 +++++ + test/mocks/cdi_operate_api_mock.h | 35 +++ + test/sandbox/controller/shim/CMakeLists.txt | 1 + + test/specs/specs/CMakeLists.txt | 1 + + test/specs/specs/specs_ut.cc | 241 ++++++++++++++++++ + test/specs/specs_extend/CMakeLists.txt | 1 + + 13 files changed, 569 insertions(+), 2 deletions(-) + create mode 100644 test/cutils/utils_version/CMakeLists.txt + create mode 100644 test/cutils/utils_version/utils_version_ut.cc + create mode 100644 test/mocks/cdi_operate_api_mock.cc + create mode 100644 test/mocks/cdi_operate_api_mock.h + +diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh +index 9bb984cd..9d4c5533 100755 +--- a/CI/make-and-install.sh ++++ b/CI/make-and-install.sh +@@ -72,7 +72,7 @@ cd $ISULAD_COPY_PATH + sed -i 's/fd == STDIN_FILENO || fd == STDOUT_FILENO || fd == STDERR_FILENO/fd == 0 || fd == 1 || fd == 2 || fd >= 1000/g' ./src/utils/cutils/utils.c + rm -rf build + mkdir build && cd build +-cmake -DCMAKE_BUILD_TYPE=Debug -DENABLE_UT=ON -DENABLE_CRI_API_V1=ON -DENABLE_SHIM_V2=ON -DENABLE_METRICS=ON .. ++cmake -DCMAKE_BUILD_TYPE=Debug -DENABLE_UT=ON -DENABLE_CRI_API_V1=ON -DENABLE_CDI=ON -DENABLE_SHIM_V2=ON -DENABLE_METRICS=ON .. + make -j $(nproc) + make install + ctest -E "driver_devmapper_ut" -T memcheck --output-on-failure +@@ -103,7 +103,7 @@ rm -rf build + mkdir build + cd build + if [[ ${enable_gcov} -ne 0 ]]; then +- cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_CRI_API_V1=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON -DENABLE_GRPC_REMOTE_CONNECT=ON .. ++ cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_CRI_API_V1=ON -DENABLE_CDI=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON -DENABLE_GRPC_REMOTE_CONNECT=ON .. + else + cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON -DENABLE_CRI_API_V1=ON -DENABLE_GRPC_REMOTE_CONNECT=ON .. + fi +diff --git a/test/cutils/CMakeLists.txt b/test/cutils/CMakeLists.txt +index 9e681cc9..bd9def02 100644 +--- a/test/cutils/CMakeLists.txt ++++ b/test/cutils/CMakeLists.txt +@@ -34,4 +34,5 @@ add_subdirectory(utils_utils) + add_subdirectory(utils_verify) + add_subdirectory(utils_network) + add_subdirectory(utils_transform) ++add_subdirectory(utils_version) + add_subdirectory(map) +diff --git a/test/cutils/utils_array/utils_array_ut.cc b/test/cutils/utils_array/utils_array_ut.cc +index 7bd13c25..6c6e76fa 100644 +--- a/test/cutils/utils_array/utils_array_ut.cc ++++ b/test/cutils/utils_array/utils_array_ut.cc +@@ -50,6 +50,33 @@ TEST(utils_array, test_util_free_array) + util_free_array(array); + } + ++TEST(utils_array, test_util_copy_array_by_len) ++{ ++ char **array = nullptr; ++ char **array_copy = nullptr; ++ size_t len = 3; ++ ++ array = (char **)util_common_calloc_s(4 * sizeof(char *)); ++ ASSERT_NE(array, nullptr); ++ array[0] = util_strdup_s("test1"); ++ array[1] = util_strdup_s("test2"); ++ array[2] = util_strdup_s("test3"); ++ ++ array_copy = util_copy_array_by_len(array, len); ++ ASSERT_NE(array_copy, nullptr); ++ for (size_t i = 0; i < len; i++) { ++ ASSERT_EQ(strcmp(array_copy[i], array[i]), 0); ++ free(array[i]); ++ free(array_copy[i]); ++ } ++ ++ ASSERT_EQ(util_copy_array_by_len(array, 0), nullptr); ++ ASSERT_EQ(util_copy_array_by_len(nullptr, len), nullptr); ++ ++ free(array); ++ free(array_copy); ++} ++ + TEST(utils_array, test_util_grow_array) + { + char **array = nullptr; +@@ -229,6 +256,34 @@ TEST(utils_array, test_util_append_string_array) + sarray = nullptr; + } + ++TEST(utils_array, test_util_copy_string_array) ++{ ++ __isula_auto_string_array_t string_array *sarray_copy = nullptr; ++ __isula_auto_string_array_t string_array *sarray = (string_array *)util_common_calloc_s(sizeof(string_array)); ++ ASSERT_NE(sarray, nullptr); ++ int ret; ++ ++ ret = util_append_string_array(sarray, "1234567890"); ++ ASSERT_EQ(ret, 0); ++ ret = util_append_string_array(sarray, "abc"); ++ ASSERT_EQ(ret, 0); ++ ret = util_append_string_array(sarray, "bcd"); ++ ASSERT_EQ(ret, 0); ++ ASSERT_EQ(sarray->len, 3); ++ ++ sarray_copy = util_copy_string_array(sarray); ++ ASSERT_NE(sarray_copy, nullptr); ++ ASSERT_EQ(sarray_copy->len, sarray->len); ++ for (size_t i = 0; i < sarray_copy->len; i++) { ++ ASSERT_EQ(strcmp(sarray_copy->items[i], sarray->items[i]), 0); ++ } ++ ++ ASSERT_EQ(util_copy_string_array(nullptr), nullptr); ++ sarray->cap = 0; ++ ASSERT_EQ(util_copy_string_array(sarray), nullptr); ++ sarray->cap = sarray->len; ++} ++ + TEST(utils_array, test_util_string_array_contain) + { + string_array *sarray = (string_array *)util_common_calloc_s(sizeof(string_array)); +@@ -299,3 +354,76 @@ TEST(utils_array, test_util_common_array_append_pointer) + delete element1; + delete element2; + } ++ ++static void common_array_free_mock(void *ptr) ++{ ++ (void)ptr; ++ return; ++} ++ ++TEST(utils_array, test_util_append_common_array) ++{ ++ __isula_auto_common_array_t common_array *carray = nullptr; ++ int ret; ++ int value1 = 1; ++ int value2 = 2; ++ int value3 = 3; ++ ++ carray = util_common_array_new(1, common_array_free_mock, util_clone_ptr); ++ ASSERT_NE(carray, nullptr); ++ ++ ret = util_append_common_array(carray, &value1); ++ ASSERT_EQ(ret, 0); ++ ASSERT_EQ(carray->items[0], &value1); ++ ASSERT_EQ(carray->len, 1); ++ ++ ret = util_append_common_array(carray, &value2); ++ ASSERT_EQ(ret, 0); ++ ret = util_append_common_array(carray, &value3); ++ ASSERT_EQ(ret, 0); ++ ASSERT_EQ(carray->items[1], &value2); ++ ASSERT_EQ(carray->items[2], &value3); ++ ASSERT_EQ(carray->len, 3); ++ ++ carray->clone_item_cb = nullptr; ++ ASSERT_EQ(util_append_common_array(carray, &value1), -1); ++ carray->clone_item_cb = util_clone_ptr; ++ ASSERT_EQ(util_append_common_array(carray, nullptr), 0); ++} ++ ++TEST(utils_array, test_util_merge_common_array) ++{ ++ __isula_auto_common_array_t common_array *carray1 = nullptr; ++ __isula_auto_common_array_t common_array *carray2 = nullptr; ++ int ret; ++ int value1 = 1; ++ int value2 = 2; ++ ++ carray1 = util_common_array_new(1, common_array_free_mock, util_clone_ptr); ++ ASSERT_NE(carray1, nullptr); ++ carray2 = util_common_array_new(1, common_array_free_mock, util_clone_ptr); ++ ASSERT_NE(carray2, nullptr); ++ ++ ret = util_append_common_array(carray1, &value1); ++ ASSERT_EQ(ret, 0); ++ ASSERT_EQ(carray1->items[0], &value1); ++ ASSERT_EQ(carray1->len, 1); ++ ret = util_append_common_array(carray2, &value2); ++ ASSERT_EQ(ret, 0); ++ ASSERT_EQ(carray2->items[0], &value2); ++ ASSERT_EQ(carray2->len, 1); ++ ++ ret = util_merge_common_array(carray1, carray2); ++ ASSERT_EQ(ret, 0); ++ ASSERT_EQ(carray1->items[1], &value2); ++ ASSERT_EQ(carray1->len, 2); ++ ++ ASSERT_EQ(util_merge_common_array(nullptr, carray2), -1); ++ ASSERT_EQ(util_merge_common_array(carray1, nullptr), -1); ++ carray1->clone_item_cb = nullptr; ++ ASSERT_EQ(util_merge_common_array(carray1, carray2), -1); ++ carray1->clone_item_cb = util_clone_ptr; ++ carray2->clone_item_cb = nullptr; ++ ASSERT_EQ(util_merge_common_array(carray1, carray2), -1); ++ carray2->clone_item_cb = util_clone_ptr; ++} +\ No newline at end of file +diff --git a/test/cutils/utils_utils/utils_utils_ut.cc b/test/cutils/utils_utils/utils_utils_ut.cc +index 0720d1b1..a61e5a21 100644 +--- a/test/cutils/utils_utils/utils_utils_ut.cc ++++ b/test/cutils/utils_utils/utils_utils_ut.cc +@@ -54,6 +54,18 @@ static int status_to_exit_code(int status) + return exit_code; + } + ++TEST(utils_utils, test_util_swap_ptr) ++{ ++ int val1 = 1; ++ int val2 = 2; ++ int *ptr1 = &val1; ++ int *ptr2 = &val2; ++ ++ util_swap_ptr((void **)&ptr1, (void **)&ptr2); ++ ASSERT_EQ(*ptr1, val2); ++ ASSERT_EQ(*ptr2, val1); ++} ++ + TEST(utils_utils, test_util_mem_realloc) + { + char *old = nullptr; +diff --git a/test/cutils/utils_version/CMakeLists.txt b/test/cutils/utils_version/CMakeLists.txt +new file mode 100644 +index 00000000..1ada8e93 +--- /dev/null ++++ b/test/cutils/utils_version/CMakeLists.txt +@@ -0,0 +1,17 @@ ++project(iSulad_UT) ++ ++SET(EXE utils_version_ut) ++ ++add_executable(${EXE} ++ utils_version_ut.cc) ++ ++target_include_directories(${EXE} PUBLIC ++ ${GTEST_INCLUDE_DIR} ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../include ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils ++ ) ++ ++target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) ++add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_version/utils_version_ut.cc b/test/cutils/utils_version/utils_version_ut.cc +new file mode 100644 +index 00000000..d1fc0932 +--- /dev/null ++++ b/test/cutils/utils_version/utils_version_ut.cc +@@ -0,0 +1,71 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-04-25 ++ * Description: utils version unit test ++ *******************************************************************************/ ++ ++#include <stdlib.h> ++#include <stdio.h> ++#include <climits> ++#include <gtest/gtest.h> ++#include "mock.h" ++#include "utils_version.h" ++#include "utils.h" ++ ++TEST(utils_version, test_util_version_compare) ++{ ++ const char *version1 = "1.1.1"; ++ const char *version2 = "1.1.2"; ++ int diff_value = 0; ++ ++ ASSERT_EQ(util_version_compare(version1, version2, &diff_value), 0); ++ ASSERT_TRUE(diff_value < 0); ++ ASSERT_EQ(util_version_compare(version1, version1, &diff_value), 0); ++ ASSERT_TRUE(diff_value == 0); ++ ASSERT_EQ(util_version_compare(version2, version1, &diff_value), 0); ++ ASSERT_TRUE(diff_value > 0); ++ ++ ASSERT_EQ(util_version_compare(version1, nullptr, &diff_value), -1); ++ ASSERT_EQ(util_version_compare(nullptr, version2, &diff_value), -1); ++ ASSERT_EQ(util_version_compare(version1, version2, nullptr), -1); ++ ASSERT_EQ(util_version_compare("1.1.1.1", version2, nullptr), -1); ++ ASSERT_EQ(util_version_compare(version1, "a.b.1.1", nullptr), -1); ++} ++ ++TEST(utils_version, test_util_version_greater_than) ++{ ++ const char *version1 = "0.6.0"; ++ const char *version2 = "1.0.0"; ++ bool result = true; ++ ++ ASSERT_EQ(util_version_greater_than(version1, version2, &result), 0); ++ ASSERT_FALSE(result); ++ ASSERT_EQ(util_version_greater_than(version1, version1, &result), 0); ++ ASSERT_FALSE(result); ++ ASSERT_EQ(util_version_greater_than(version2, version1, &result), 0); ++ ASSERT_TRUE(result); ++} ++ ++TEST(utils_version, test_util_version_greater_than_or_equal_to) ++{ ++ const char *version1 = "0.6.0"; ++ const char *version2 = "1.0.0"; ++ bool result = true; ++ ++ ASSERT_EQ(util_version_greater_than_or_equal_to(version1, version2, &result), 0); ++ ASSERT_FALSE(result); ++ ASSERT_EQ(util_version_greater_than_or_equal_to(version1, version1, &result), 0); ++ ASSERT_TRUE(result); ++ ASSERT_EQ(util_version_greater_than_or_equal_to(version2, version1, &result), 0); ++ ASSERT_TRUE(result); ++} ++ +diff --git a/test/image/oci/oci_config_merge/CMakeLists.txt b/test/image/oci/oci_config_merge/CMakeLists.txt +index ffd3999d..38ade4ec 100644 +--- a/test/image/oci/oci_config_merge/CMakeLists.txt ++++ b/test/image/oci/oci_config_merge/CMakeLists.txt +@@ -47,6 +47,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/selinux_label_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/storage_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/image_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/cdi_operate_api_mock.cc + oci_config_merge_ut.cc) + + target_include_directories(${EXE} PUBLIC +diff --git a/test/mocks/cdi_operate_api_mock.cc b/test/mocks/cdi_operate_api_mock.cc +new file mode 100644 +index 00000000..d8f9f9d4 +--- /dev/null ++++ b/test/mocks/cdi_operate_api_mock.cc +@@ -0,0 +1,58 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2023-10-20 ++ * Description: provide image api mock ++ ******************************************************************************/ ++ ++#include "cdi_operate_api_mock.h" ++ ++namespace { ++std::shared_ptr<MockCdiOperateApi> g_cdi_operate_api_mock = nullptr; ++} ++ ++void MockCdiOperateApi_SetMock(std::shared_ptr<MockCdiOperateApi> mock) ++{ ++ g_cdi_operate_api_mock = mock; ++} ++ ++int cdi_operate_registry_init(char **specs_dirs, size_t specs_dirs_len) ++{ ++ if (g_cdi_operate_api_mock != nullptr) { ++ return g_cdi_operate_api_mock->CdiOperateRegistryInit(specs_dirs, specs_dirs_len); ++ } ++ return 0; ++} ++ ++int cdi_operate_refresh(void) ++{ ++ if (g_cdi_operate_api_mock != nullptr) { ++ return g_cdi_operate_api_mock->CdiOperateRefresh(); ++ } ++ return 0; ++} ++ ++int cdi_operate_inject_devices(oci_runtime_spec *spec, string_array *devices) ++{ ++ if (g_cdi_operate_api_mock != nullptr) { ++ return g_cdi_operate_api_mock->CdiOperateInjectDevices(spec, devices); ++ } ++ return 0; ++} ++ ++int cdi_operate_parse_annotations(json_map_string_string *annotations, string_array **keys, ++ string_array **devices, char **error) ++{ ++ if (g_cdi_operate_api_mock != nullptr) { ++ return g_cdi_operate_api_mock->CdiOperateParseAnnotations(annotations, keys, devices, error); ++ } ++ return 0; ++} +\ No newline at end of file +diff --git a/test/mocks/cdi_operate_api_mock.h b/test/mocks/cdi_operate_api_mock.h +new file mode 100644 +index 00000000..c118ee7a +--- /dev/null ++++ b/test/mocks/cdi_operate_api_mock.h +@@ -0,0 +1,35 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-04-24 ++ * Description: provide cdi api mock ++ ******************************************************************************/ ++ ++#ifndef ISULAD_TEST_MOCKS_CDI_OPERATE_API_MOCK_H ++#define ISULAD_TEST_MOCKS_CDI_OPERATE_API_MOCK_H ++ ++#include <gmock/gmock.h> ++#include <memory> ++ ++#include "cdi_operate_api.h" ++ ++class MockCdiOperateApi { ++public: ++ MOCK_METHOD2(CdiOperateRegistryInit, int(char **specs_dirs, size_t specs_dirs_len)); ++ MOCK_METHOD0(CdiOperateRefresh, int(void)); ++ MOCK_METHOD2(CdiOperateInjectDevices, int(oci_runtime_spec *spec, string_array *devices)); ++ MOCK_METHOD4(CdiOperateParseAnnotations, int(json_map_string_string *annotations, string_array **keys, ++ string_array **devices, char **error)); ++}; ++ ++void MockCdiOperateApi_SetMock(std::shared_ptr<MockCdiOperateApi> mock); ++ ++#endif +diff --git a/test/sandbox/controller/shim/CMakeLists.txt b/test/sandbox/controller/shim/CMakeLists.txt +index 26a66e51..d18d1861 100644 +--- a/test/sandbox/controller/shim/CMakeLists.txt ++++ b/test/sandbox/controller/shim/CMakeLists.txt +@@ -21,6 +21,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/callback_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/image_api_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/service_container_api_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/cdi_operate_api_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/sandbox/controller/controller_common.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/sandbox/controller/shim/shim_controller_ut.cc + ) +diff --git a/test/specs/specs/CMakeLists.txt b/test/specs/specs/CMakeLists.txt +index 12c11f51..892d44d7 100644 +--- a/test/specs/specs/CMakeLists.txt ++++ b/test/specs/specs/CMakeLists.txt +@@ -44,6 +44,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/image_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/cdi_operate_api_mock.cc + specs_ut.cc) + + target_include_directories(${EXE} PUBLIC +diff --git a/test/specs/specs/specs_ut.cc b/test/specs/specs/specs_ut.cc +index 6c42216d..47836e5b 100644 +--- a/test/specs/specs/specs_ut.cc ++++ b/test/specs/specs/specs_ut.cc +@@ -564,6 +564,247 @@ TEST_F(SpecsUnitTest, test_update_devcies_for_oci_spec) + free(err); + } + ++#ifdef ENABLE_CDI ++TEST_F(SpecsUnitTest, test_defs_process_add_multiple_env) ++{ ++ size_t env_len = 2; ++ char **envs = (char **)util_common_calloc_s(sizeof(char *) * env_len); ++ ASSERT_NE(envs, nullptr); ++ defs_process *dp = (defs_process *)util_common_calloc_s(sizeof(defs_process)); ++ ASSERT_NE(dp, nullptr); ++ dp->env_len = 1; ++ dp->env = (char **)util_common_calloc_s(sizeof(char *) * dp->env_len); ++ ASSERT_NE(dp->env, nullptr); ++ ++ envs[0] = util_strdup_s("key0=value0"); ++ envs[1] = util_strdup_s("key1=value1"); ++ dp->env[0] = util_strdup_s("key0=value0_old"); ++ ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), 0); ++ ASSERT_EQ(dp->env_len, 2); ++ ASSERT_EQ(strcmp(dp->env[0], envs[0]), 0); ++ ASSERT_EQ(strcmp(dp->env[1], envs[1]), 0); ++ ++ ASSERT_EQ(defs_process_add_multiple_env(dp, nullptr, env_len), 0); ++ ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, 0), 0); ++ ASSERT_EQ(defs_process_add_multiple_env(nullptr, (const char **)envs, env_len), -1); ++ ++ free(envs[0]); ++ envs[0] = util_strdup_s("=value0"); ++ ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1); ++ free(envs[0]); ++ envs[0] = util_strdup_s("key0="); ++ ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1); ++ free(envs[0]); ++ envs[0] = util_strdup_s("key0xxxx"); ++ ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1); ++ ++ free(dp->env[0]); ++ dp->env[0] = util_strdup_s("=value0"); ++ ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1); ++ free(dp->env[0]); ++ dp->env[0] = util_strdup_s("key0="); ++ ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1); ++ free(dp->env[0]); ++ dp->env[0] = util_strdup_s("key0xxxx"); ++ ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1); ++ ++ free_defs_process(dp); ++ free(envs[0]); ++ free(envs[1]); ++ free(envs); ++} ++ ++TEST_F(SpecsUnitTest, test_spec_add_multiple_process_env) ++{ ++ size_t env_len = 2; ++ char **envs = (char **)util_common_calloc_s(sizeof(char *) * env_len); ++ ASSERT_NE(envs, nullptr); ++ oci_runtime_spec *oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec)); ++ ASSERT_NE(oci_spec, nullptr); ++ oci_spec->process = (defs_process *)util_common_calloc_s(sizeof(defs_process)); ++ ASSERT_NE(oci_spec->process, nullptr); ++ oci_spec->process->env_len = 1; ++ oci_spec->process->env = (char **)util_common_calloc_s(sizeof(char *) * oci_spec->process->env_len); ++ ASSERT_NE(oci_spec->process->env, nullptr); ++ ++ envs[0] = util_strdup_s("key0=value0"); ++ envs[1] = util_strdup_s("key1=value1"); ++ oci_spec->process->env[0] = util_strdup_s("key0=value0_old"); ++ ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), 0); ++ ASSERT_EQ(oci_spec->process->env_len, 2); ++ ASSERT_EQ(strcmp(oci_spec->process->env[0], envs[0]), 0); ++ ASSERT_EQ(strcmp(oci_spec->process->env[1], envs[1]), 0); ++ ++ ASSERT_EQ(spec_add_multiple_process_env(oci_spec, nullptr, env_len), 0); ++ ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, 0), 0); ++ ASSERT_EQ(spec_add_multiple_process_env(nullptr, (const char **)envs, env_len), -1); ++ ++ free(envs[0]); ++ envs[0] = util_strdup_s("=value0"); ++ ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1); ++ free(envs[0]); ++ envs[0] = util_strdup_s("key0="); ++ ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1); ++ free(envs[0]); ++ envs[0] = util_strdup_s("key0xxxx"); ++ ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1); ++ ++ free(oci_spec->process->env[0]); ++ oci_spec->process->env[0] = util_strdup_s("=value0"); ++ ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1); ++ free(oci_spec->process->env[0]); ++ oci_spec->process->env[0] = util_strdup_s("key0="); ++ ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1); ++ free(oci_spec->process->env[0]); ++ oci_spec->process->env[0] = util_strdup_s("key0xxxx"); ++ ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1); ++ ++ free_oci_runtime_spec(oci_spec); ++ free(envs[0]); ++ free(envs[1]); ++ free(envs); ++} ++ ++TEST_F(SpecsUnitTest, test_spec_add_device) ++{ ++ defs_device *device = (defs_device *)util_common_calloc_s(sizeof(defs_device)); ++ ASSERT_NE(device, nullptr); ++ oci_runtime_spec *oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec)); ++ ASSERT_NE(oci_spec, nullptr); ++ oci_spec->linux = (oci_runtime_config_linux *)util_common_calloc_s(sizeof(oci_runtime_config_linux)); ++ ASSERT_NE(oci_spec->linux, nullptr); ++ oci_spec->linux->devices_len = 1; ++ oci_spec->linux->devices = (defs_device **)util_common_calloc_s(sizeof(defs_device *) * oci_spec->linux->devices_len); ++ ASSERT_NE(oci_spec->linux->devices, nullptr); ++ ++ device->path = util_strdup_s("/device/path"); ++ oci_spec->linux->devices[0] = (defs_device *)util_common_calloc_s(sizeof(defs_device)); ++ ASSERT_NE(oci_spec->linux->devices[0], nullptr); ++ oci_spec->linux->devices[0]->path = util_strdup_s("/device/path"); ++ ASSERT_EQ(spec_add_device(oci_spec, device), 0); ++ ASSERT_EQ(oci_spec->linux->devices[0], device); ++ ++ oci_spec->linux->devices[0] = nullptr; ++ oci_spec->linux->devices_len = 0; ++ ASSERT_EQ(spec_add_device(oci_spec, device), 0); ++ ASSERT_EQ(oci_spec->linux->devices_len, 1); ++ ASSERT_EQ(oci_spec->linux->devices[0], device); ++ ++ ASSERT_EQ(spec_add_device(oci_spec, nullptr), -1); ++ ASSERT_EQ(spec_add_device(nullptr, device), -1); ++ ++ free_oci_runtime_spec(oci_spec); ++} ++ ++TEST_F(SpecsUnitTest, test_spec_add_linux_resources_device) ++{ ++ oci_runtime_spec *oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec)); ++ ASSERT_NE(oci_spec, nullptr); ++ oci_spec->linux = (oci_runtime_config_linux *)util_common_calloc_s(sizeof(oci_runtime_config_linux)); ++ ASSERT_NE(oci_spec->linux, nullptr); ++ oci_spec->linux->resources = (defs_resources *)util_common_calloc_s(sizeof(defs_resources)); ++ ASSERT_NE(oci_spec->linux->resources, nullptr); ++ oci_spec->linux->resources->devices_len = 1; ++ oci_spec->linux->resources->devices = (defs_device_cgroup **)util_common_calloc_s(sizeof(defs_device_cgroup *) * oci_spec->linux->resources->devices_len); ++ ASSERT_NE(oci_spec->linux->resources->devices, nullptr); ++ ++ oci_spec->linux->resources->devices[0] = (defs_device_cgroup *)util_common_calloc_s(sizeof(defs_device_cgroup)); ++ ASSERT_NE(oci_spec->linux->resources->devices[0], nullptr); ++ ASSERT_EQ(spec_add_linux_resources_device(oci_spec, true, "bind", 10, 9, "rwm"), 0); ++ ASSERT_EQ(oci_spec->linux->resources->devices_len, 2); ++ ASSERT_EQ(oci_spec->linux->resources->devices[1]->allow, true); ++ ASSERT_EQ(strcmp(oci_spec->linux->resources->devices[1]->type, "bind"), 0); ++ ASSERT_EQ(oci_spec->linux->resources->devices[1]->major, 10); ++ ASSERT_EQ(oci_spec->linux->resources->devices[1]->minor, 9); ++ ASSERT_EQ(strcmp(oci_spec->linux->resources->devices[1]->access, "rwm"), 0); ++ ++ ASSERT_EQ(spec_add_linux_resources_device(nullptr, true, "bind", 10, 9, "rwm"), -1); ++ ++ free_oci_runtime_spec(oci_spec); ++} ++ ++TEST_F(SpecsUnitTest, test_spec_remove_mount) ++{ ++ oci_runtime_spec *oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec)); ++ ASSERT_NE(oci_spec, nullptr); ++ oci_spec->mounts_len = 2; ++ oci_spec->mounts = (defs_mount **)util_common_calloc_s(sizeof(defs_mount *) * oci_spec->mounts_len); ++ ASSERT_NE(oci_spec->mounts, nullptr); ++ ++ oci_spec->mounts[0] = (defs_mount *)util_common_calloc_s(sizeof(defs_mount)); ++ ASSERT_NE(oci_spec->mounts[0], nullptr); ++ oci_spec->mounts[1] = (defs_mount *)util_common_calloc_s(sizeof(defs_mount)); ++ ASSERT_NE(oci_spec->mounts[1], nullptr); ++ oci_spec->mounts[0]->destination = util_strdup_s("/mount/path/0"); ++ oci_spec->mounts[1]->destination = util_strdup_s("/mount/path/1"); ++ spec_remove_mount(oci_spec, oci_spec->mounts[0]->destination); ++ ASSERT_EQ(oci_spec->mounts_len, 1); ++ ASSERT_EQ(strcmp(oci_spec->mounts[0]->destination, "/mount/path/1"), 0); ++ ++ free_oci_runtime_spec(oci_spec); ++} ++ ++TEST_F(SpecsUnitTest, test_spec_add_mount) ++{ ++ defs_mount *mnt = (defs_mount *)util_common_calloc_s(sizeof(defs_mount)); ++ ASSERT_NE(mnt, nullptr); ++ oci_runtime_spec *oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec)); ++ ASSERT_NE(oci_spec, nullptr); ++ oci_spec->mounts_len = 1; ++ oci_spec->mounts = (defs_mount **)util_common_calloc_s(sizeof(defs_mount *) * oci_spec->mounts_len ); ++ ASSERT_NE(oci_spec->mounts, nullptr); ++ ++ oci_spec->mounts[0] = (defs_mount *)util_common_calloc_s(sizeof(defs_mount)); ++ ASSERT_NE(oci_spec->mounts[0], nullptr); ++ oci_spec->mounts[0]->destination = util_strdup_s("/mount/path/0"); ++ ASSERT_EQ(spec_add_mount(oci_spec, mnt), 0); ++ ASSERT_EQ(oci_spec->mounts_len, 2); ++ ASSERT_EQ(oci_spec->mounts[1], mnt); ++ ++ ASSERT_EQ(spec_add_mount(nullptr, mnt), -1); ++ ASSERT_EQ(spec_add_mount(oci_spec, nullptr), -1); ++ ++ free_oci_runtime_spec(oci_spec); ++} ++ ++#define TEST_SPEC_ADD_HOOKS_ITEM_DEF(hooktype) \ ++ void test_spec_add_##hooktype##_hook(void) \ ++ { \ ++ defs_hook *hook = (defs_hook *)util_common_calloc_s(sizeof(defs_hook)); \ ++ ASSERT_NE(hook, nullptr); \ ++ oci_runtime_spec *oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec)); \ ++ ASSERT_NE(oci_spec, nullptr); \ ++ oci_spec->hooks = (oci_runtime_spec_hooks *)util_common_calloc_s(sizeof(oci_runtime_spec_hooks)); \ ++ ASSERT_NE(oci_spec->hooks, nullptr); \ ++ oci_spec->hooks->hooktype##_len = 1; \ ++ oci_spec->hooks->hooktype = (defs_hook **)util_common_calloc_s(sizeof(defs_hook *) * oci_spec->hooks->hooktype##_len); \ ++ ASSERT_NE(oci_spec->hooks->hooktype, nullptr); \ ++ \ ++ oci_spec->hooks->hooktype[0] = (defs_hook *)util_common_calloc_s(sizeof(defs_hook)); \ ++ ASSERT_NE(oci_spec->hooks->hooktype[0], nullptr); \ ++ ASSERT_EQ(spec_add_##hooktype##_hook(oci_spec, hook), 0); \ ++ ASSERT_EQ(oci_spec->hooks->hooktype##_len, 2); \ ++ ASSERT_EQ(oci_spec->hooks->hooktype[1], hook); \ ++ \ ++ ASSERT_EQ(spec_add_##hooktype##_hook(nullptr, hook), -1); \ ++ ASSERT_EQ(spec_add_##hooktype##_hook(oci_spec, nullptr), -1); \ ++ \ ++ free_oci_runtime_spec(oci_spec); \ ++ } ++ ++TEST_SPEC_ADD_HOOKS_ITEM_DEF(prestart) ++TEST_SPEC_ADD_HOOKS_ITEM_DEF(poststart) ++TEST_SPEC_ADD_HOOKS_ITEM_DEF(poststop) ++ ++TEST_F(SpecsUnitTest, test_spec_add_hook) ++{ ++ test_spec_add_prestart_hook(); ++ test_spec_add_poststart_hook(); ++ test_spec_add_poststop_hook(); ++} ++ ++#endif /* ENABLE_CDI */ ++ + /********************************* UT for merge caps *******************************************/ + struct capabilities_lens { + size_t bounding_len; +diff --git a/test/specs/specs_extend/CMakeLists.txt b/test/specs/specs_extend/CMakeLists.txt +index 2fd37e1c..bd4d2dd6 100644 +--- a/test/specs/specs_extend/CMakeLists.txt ++++ b/test/specs/specs_extend/CMakeLists.txt +@@ -44,6 +44,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/image_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/cdi_operate_api_mock.cc + specs_extend_ut.cc) + + target_include_directories(${EXE} PUBLIC +-- +2.25.1 + diff --git a/0077-remove-extra-s-in-CreateContainerLogSymlink.patch b/0077-remove-extra-s-in-CreateContainerLogSymlink.patch new file mode 100644 index 0000000..571f47c --- /dev/null +++ b/0077-remove-extra-s-in-CreateContainerLogSymlink.patch @@ -0,0 +1,26 @@ +From cade2ae3b53848bbedb3f89ff45333f3871e878a Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Thu, 25 Apr 2024 12:46:10 +0000 +Subject: [PATCH 77/78] remove extra %s in CreateContainerLogSymlink + +Signed-off-by: jikai <jikai11@huawei.com> +--- + src/daemon/common/cri/cri_helpers.cc | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/daemon/common/cri/cri_helpers.cc b/src/daemon/common/cri/cri_helpers.cc +index 5189ac00..d7ec9f36 100644 +--- a/src/daemon/common/cri/cri_helpers.cc ++++ b/src/daemon/common/cri/cri_helpers.cc +@@ -552,7 +552,7 @@ void CreateContainerLogSymlink(const std::string &containerID, Errors &error) + if (symlink(realPath.c_str(), path.c_str()) != 0) { + SYSERROR("failed to create symbolic link %s to the container log file %s for container %s", path.c_str(), realPath.c_str(), + containerID.c_str()); +- error.Errorf("failed to create symbolic link %s to the container log file %s for container %s: %s", path.c_str(), ++ error.Errorf("failed to create symbolic link %s to the container log file %s for container %s", path.c_str(), + realPath.c_str(), containerID.c_str()); + } + } +-- +2.25.1 + diff --git a/0078-allow-env-variable-has-an-empty-value.patch b/0078-allow-env-variable-has-an-empty-value.patch new file mode 100644 index 0000000..213a996 --- /dev/null +++ b/0078-allow-env-variable-has-an-empty-value.patch @@ -0,0 +1,69 @@ +From f0212d54afc695b2039f09456b10c47f8edaf2de Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Sat, 27 Apr 2024 14:32:19 +0800 +Subject: [PATCH 78/78] allow env variable has an empty value + +Signed-off-by: jikai <jikai11@huawei.com> +--- + src/daemon/modules/spec/specs_extend.c | 10 ++++++---- + src/utils/cutils/utils_string.c | 3 +++ + 2 files changed, 9 insertions(+), 4 deletions(-) + +diff --git a/src/daemon/modules/spec/specs_extend.c b/src/daemon/modules/spec/specs_extend.c +index 199cba54..8cad2cbe 100644 +--- a/src/daemon/modules/spec/specs_extend.c ++++ b/src/daemon/modules/spec/specs_extend.c +@@ -195,6 +195,7 @@ static int generate_env_map_from_file(FILE *fp, json_map_string_string *env_map) + char *pline = NULL; + size_t length = 0; + char *saveptr = NULL; ++ char empty_str[1] = {'\0'}; + + while (getline(&pline, &length, fp) != -1) { + util_trim_newline(pline); +@@ -204,7 +205,9 @@ static int generate_env_map_from_file(FILE *fp, json_map_string_string *env_map) + } + key = strtok_r(pline, "=", &saveptr); + value = strtok_r(NULL, "=", &saveptr); +- if (key != NULL && value != NULL) { ++ // value of an env varible is allowed to be empty ++ value = value ? value : empty_str; ++ if (key != NULL) { + key = util_trim_space(key); + value = util_trim_space(value); + if ((size_t)(MAX_BUFFER_SIZE - 1) - strlen(key) < strlen(value)) { +@@ -291,15 +294,14 @@ static int check_env_need_append(const oci_runtime_spec *oci_spec, const char *e + { + size_t i = 0; + char *key = NULL; +- char *value = NULL; + char *saveptr = NULL; + + for (i = 0; i < oci_spec->process->env_len; i++) { + char *tmp_env = NULL; + tmp_env = util_strdup_s(oci_spec->process->env[i]); + key = strtok_r(tmp_env, "=", &saveptr); +- value = strtok_r(NULL, "=", &saveptr); +- if (key == NULL || value == NULL) { ++ // value of an env varible is allowed to be empty ++ if (key == NULL) { + ERROR("Bad env format"); + free(tmp_env); + tmp_env = NULL; +diff --git a/src/utils/cutils/utils_string.c b/src/utils/cutils/utils_string.c +index 64afb570..11a65f19 100644 +--- a/src/utils/cutils/utils_string.c ++++ b/src/utils/cutils/utils_string.c +@@ -534,6 +534,9 @@ static char *util_left_trim_space(char *str) + { + char *begin = str; + char *tmp = str; ++ if (strlen(str) == 0) { ++ return str; ++ } + while (isspace(*begin)) { + begin++; + } +-- +2.25.1 + diff --git a/0079-Fix-Failed-to-execute-image-pull-on-name-tag-digest-.patch b/0079-Fix-Failed-to-execute-image-pull-on-name-tag-digest-.patch new file mode 100644 index 0000000..6c9fc35 --- /dev/null +++ b/0079-Fix-Failed-to-execute-image-pull-on-name-tag-digest-.patch @@ -0,0 +1,72 @@ +From 1d90c5a855a267bb156d53b6a43bc451a993efd5 Mon Sep 17 00:00:00 2001 +From: huj13k4n9 <huj13k4n9@qq.com> +Date: Mon, 29 Apr 2024 12:52:17 +0800 +Subject: [PATCH 79/85] Fix 'Failed to execute image pull' on 'name:tag@digest' + type image name + +--- + src/daemon/modules/image/oci/utils_images.c | 6 ++++++ + src/utils/cutils/utils_verify.c | 3 ++- + src/utils/cutils/utils_verify.h | 3 ++- + 3 files changed, 10 insertions(+), 2 deletions(-) + +diff --git a/src/daemon/modules/image/oci/utils_images.c b/src/daemon/modules/image/oci/utils_images.c +index d94388bd..b4e2a1a8 100644 +--- a/src/daemon/modules/image/oci/utils_images.c ++++ b/src/daemon/modules/image/oci/utils_images.c +@@ -207,6 +207,7 @@ int oci_split_image_name(const char *image_name, char **host, char **name, char + char *tag_digest_pos = NULL; + char *name_pos = NULL; + char *tmp_image_name = NULL; ++ char *name_end_pos = NULL; + + if (!util_valid_image_name(image_name)) { + ERROR("Invalid full image name %s", image_name); +@@ -234,6 +235,11 @@ int oci_split_image_name(const char *image_name, char **host, char **name, char + *name_pos = '\0'; + name_pos++; + if (name != NULL) { ++ // Need to check if image name contains tag ++ name_end_pos = strchr(name_pos, ':'); ++ if (name_end_pos != NULL) { ++ *name_end_pos = '\0'; ++ } + *name = util_strdup_s(name_pos); + } + if (host != NULL) { +diff --git a/src/utils/cutils/utils_verify.c b/src/utils/cutils/utils_verify.c +index cd636fff..474e28f0 100644 +--- a/src/utils/cutils/utils_verify.c ++++ b/src/utils/cutils/utils_verify.c +@@ -319,6 +319,7 @@ bool util_valid_image_name(const char *name) + } + } + ++ // In name check phase, image name with both tag and digest is also allowed + if (util_reg_match(__NamePattern, copy)) { + goto cleanup; + } +@@ -767,4 +768,4 @@ bool util_valid_search_name(const char *name) + + return true; + } +-#endif +\ No newline at end of file ++#endif +diff --git a/src/utils/cutils/utils_verify.h b/src/utils/cutils/utils_verify.h +index bafd2a82..fc59f6c0 100644 +--- a/src/utils/cutils/utils_verify.h ++++ b/src/utils/cutils/utils_verify.h +@@ -32,7 +32,8 @@ extern "C" { + #define __NamePattern \ + "^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])" \ + "((\\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9]))+)?(:[0-9]+)?/)?[a-z0-9]" \ +- "+((([._]|__|[-]*)[a-z0-9]+)+)?((/[a-z0-9]+((([._]|__|[-]*)[a-z0-9]+)+)?)+)?$" ++ "+((([._]|__|[-]*)[a-z0-9]+)+)?((/[a-z0-9]+((([._]|__|[-]*)[a-z0-9]+)+)?)+)?" \ ++ "(:([A-Za-z_0-9][A-Za-z_0-9.-]{0,127}))?$" + + #define __DIGESTPattern "@[a-z0-9]+:[a-z0-9]{32,}" + +-- +2.34.1 + diff --git a/0080-bugfix-for-hostname-env-set-only-once.patch b/0080-bugfix-for-hostname-env-set-only-once.patch new file mode 100644 index 0000000..191a790 --- /dev/null +++ b/0080-bugfix-for-hostname-env-set-only-once.patch @@ -0,0 +1,148 @@ +From 8ff32819d84f59085c4c541b00f9671db55d0fd1 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Mon, 29 Apr 2024 09:14:53 +0800 +Subject: [PATCH 80/85] bugfix for hostname env: set only once + +Signed-off-by: jikai <jikai11@huawei.com> +--- + src/daemon/modules/spec/specs.c | 11 +++++- + src/daemon/modules/spec/specs_extend.c | 52 +++++++++++++++++--------- + src/daemon/modules/spec/specs_extend.h | 2 + + 3 files changed, 46 insertions(+), 19 deletions(-) + +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index 77ca70f9..65a860d4 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -1863,14 +1863,21 @@ static int merge_process_conf(oci_runtime_spec *oci_spec, const host_config *hos + goto out; + } + +- /* environment variables */ ++ /* 1. merge env from container_spec: --env or --env-file */ + ret = merge_env(oci_spec, (const char **)container_spec->env, container_spec->env_len); + if (ret != 0) { + ERROR("Failed to merge environment variables"); + goto out; + } + +- /* env target file */ ++ /* 2. merge default env hostname, only if hostname not set before */ ++ ret = merge_hostname_env(oci_spec); ++ if (ret != 0) { ++ ERROR("Failed to merge hostname env"); ++ goto out; ++ } ++ ++ /* 3. persist env from --env-target-file, only if the env not set before, system container only */ + ret = merge_env_target_file(oci_spec, host_spec->env_target_file); + if (ret != 0) { + ERROR("Failed to merge env target file"); +diff --git a/src/daemon/modules/spec/specs_extend.c b/src/daemon/modules/spec/specs_extend.c +index 8cad2cbe..4c154281 100644 +--- a/src/daemon/modules/spec/specs_extend.c ++++ b/src/daemon/modules/spec/specs_extend.c +@@ -420,34 +420,23 @@ out: + int merge_env(oci_runtime_spec *oci_spec, const char **env, size_t env_len) + { + int ret = 0; +- int nret = 0; + size_t new_size = 0; + size_t old_size = 0; + size_t i; + char **temp = NULL; +- // 10 is lenght of "HOSTNAME=" and '\0' +- char host_name_env[MAX_HOST_NAME_LEN + 10] = { 0 }; +- +- nret = snprintf(host_name_env, sizeof(host_name_env), "HOSTNAME=%s", oci_spec->hostname); +- if (nret < 0 || (size_t)nret >= sizeof(host_name_env)) { +- ret = -1; +- ERROR("Sprint failed"); +- goto out; +- } + + ret = make_sure_oci_spec_process(oci_spec); + if (ret < 0) { + goto out; + } + +- if (env_len > LIST_ENV_SIZE_MAX - oci_spec->process->env_len - 1) { ++ if (env_len > LIST_ENV_SIZE_MAX - oci_spec->process->env_len) { + ERROR("The length of envionment variables is too long, the limit is %lld", LIST_ENV_SIZE_MAX); + isulad_set_error_message("The length of envionment variables is too long, the limit is %d", LIST_ENV_SIZE_MAX); + ret = -1; + goto out; + } +- // add 1 for hostname env +- new_size = (oci_spec->process->env_len + env_len + 1) * sizeof(char *); ++ new_size = (oci_spec->process->env_len + env_len) * sizeof(char *); + old_size = oci_spec->process->env_len * sizeof(char *); + ret = util_mem_realloc((void **)&temp, new_size, oci_spec->process->env, old_size); + if (ret != 0) { +@@ -458,10 +447,6 @@ int merge_env(oci_runtime_spec *oci_spec, const char **env, size_t env_len) + + oci_spec->process->env = temp; + +- // append hostname env into default oci spec env list +- oci_spec->process->env[oci_spec->process->env_len] = util_strdup_s(host_name_env); +- oci_spec->process->env_len++; +- + for (i = 0; i < env_len && env != NULL; i++) { + oci_spec->process->env[oci_spec->process->env_len] = util_strdup_s(env[i]); + oci_spec->process->env_len++; +@@ -470,6 +455,39 @@ out: + return ret; + } + ++int merge_hostname_env(oci_runtime_spec *oci_spec) ++{ ++ int nret = 0; ++ bool is_append = true; ++ // 10 is lenght of "HOSTNAME=" and '\0' ++ char host_name_env[MAX_HOST_NAME_LEN + 10] = { 0 }; ++ const char *envs[1] = {host_name_env}; ++ ++ if (make_sure_oci_spec_process(oci_spec) < 0) { ++ return -1; ++ } ++ ++ if (check_env_need_append(oci_spec, "HOSTNAME", &is_append) < 0) { ++ return -1; ++ } ++ ++ if (!is_append) { ++ return 0; ++ } ++ ++ nret = snprintf(host_name_env, sizeof(host_name_env), "HOSTNAME=%s", oci_spec->hostname); ++ if (nret < 0 || (size_t)nret >= sizeof(host_name_env)) { ++ ERROR("Sprint failed"); ++ return -1; ++ } ++ ++ if (merge_env(oci_spec, (const char **)envs, 1) < 0) { ++ return -1; ++ } ++ ++ return 0; ++} ++ + char *oci_container_get_env(const oci_runtime_spec *oci_spec, const char *key) + { + const defs_process *op = NULL; +diff --git a/src/daemon/modules/spec/specs_extend.h b/src/daemon/modules/spec/specs_extend.h +index d70f5bec..15ec6b2f 100644 +--- a/src/daemon/modules/spec/specs_extend.h ++++ b/src/daemon/modules/spec/specs_extend.h +@@ -50,6 +50,8 @@ int make_userns_remap(oci_runtime_spec *container, const char *user_remap); + + int merge_env(oci_runtime_spec *oci_spec, const char **env, size_t env_len); + ++int merge_hostname_env(oci_runtime_spec *oci_spec); ++ + int merge_env_target_file(oci_runtime_spec *oci_spec, const char *env_target_file); + + char *oci_container_get_env(const oci_runtime_spec *oci_spec, const char *key); +-- +2.34.1 + diff --git a/0081-set-the-sandbox-status-to-not-ready-under-abnormal-c.patch b/0081-set-the-sandbox-status-to-not-ready-under-abnormal-c.patch new file mode 100644 index 0000000..91a57f2 --- /dev/null +++ b/0081-set-the-sandbox-status-to-not-ready-under-abnormal-c.patch @@ -0,0 +1,86 @@ +From 934d289aa535bbb87bfe484c4de34275b968fb87 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Wed, 8 May 2024 11:40:40 +0800 +Subject: [PATCH 81/85] set the sandbox status to not ready under abnormal + circumstances + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/sandbox/sandbox.cc | 34 +++++++++++++++++++++++++--------- + src/daemon/sandbox/sandbox.h | 1 + + 2 files changed, 26 insertions(+), 9 deletions(-) + +diff --git a/src/daemon/sandbox/sandbox.cc b/src/daemon/sandbox/sandbox.cc +index bae5b8db..279bf628 100644 +--- a/src/daemon/sandbox/sandbox.cc ++++ b/src/daemon/sandbox/sandbox.cc +@@ -371,6 +371,8 @@ void Sandbox::DoUpdateStatus(std::unique_ptr<ControllerSandboxStatus> status, Er + m_state.exitedAt = status->exitedAt; + if (status->state == std::string(SANDBOX_READY_STATE_STR)) { + m_state.status = SANDBOX_STATUS_RUNNING; ++ } else { ++ m_state.status = SANDBOX_STATUS_STOPPED; + } + } + +@@ -459,6 +461,24 @@ auto Sandbox::Save(Errors &error) -> bool + return true; + } + ++bool Sandbox::DoStatusUpdateAndWaitInLoad(const std::string &sandboxID, Errors &error) ++{ ++ if (!UpdateStatus(error)) { ++ ERROR("Failed to update status of Sandbox, id='%s'", sandboxID.c_str()); ++ return false; ++ } ++ ++ // Regardless of whether the sandbox is ready, ++ // Wait() is required to call to monitor whether the kuasar sandbox is ready or exits. ++ // TODO: distinguish the meaning of Wait() return value in different states of sandbox ++ if (!m_controller->Wait(shared_from_this(), sandboxID, error)) { ++ ERROR("Failed to restore wait callback"); ++ return false; ++ } ++ ++ return true; ++} ++ + auto Sandbox::Load(Errors &error) -> bool + { + if (!LoadState(error)) { +@@ -478,15 +498,11 @@ auto Sandbox::Load(Errors &error) -> bool + + LoadNetworkSetting(); + +- if (!UpdateStatus(error)) { +- ERROR("Failed to update status of Sandbox, id='%s'", m_id.c_str()); +- return false; +- } +- +- // TODO: distinguish the meaning of Wait() return value in different states of sandbox +- if (!m_controller->Wait(shared_from_this(), m_id, error)) { +- ERROR("Failed to restore wait callback"); +- return false; ++ // When the sandbox status acquisition fails or wait fails, the sandbox status is set to not ready, ++ // and the user decides whether to delete the sandbox. ++ if (!DoStatusUpdateAndWaitInLoad(m_id, error)) { ++ WriteGuard<RWMutex> lock(m_stateMutex); ++ m_state.status = SANDBOX_STATUS_STOPPED; + } + + return true; +diff --git a/src/daemon/sandbox/sandbox.h b/src/daemon/sandbox/sandbox.h +index 20a8e338..42fbee2a 100644 +--- a/src/daemon/sandbox/sandbox.h ++++ b/src/daemon/sandbox/sandbox.h +@@ -156,6 +156,7 @@ private: + auto SetupSandboxFiles(Errors &error) -> bool; + void DoUpdateStatus(std::unique_ptr<ControllerSandboxStatus> status, Errors &error); + void DoUpdateExitedStatus(const ControllerExitInfo &exitInfo); ++ bool DoStatusUpdateAndWaitInLoad(const std::string &sandboxID, Errors &error); + + auto GetMetadataJsonPath() -> std::string; + auto GetStatePath() -> std::string; +-- +2.34.1 + diff --git a/0082-fix-shim-controller-set-incorrect-sandbox-status-sta.patch b/0082-fix-shim-controller-set-incorrect-sandbox-status-sta.patch new file mode 100644 index 0000000..13df472 --- /dev/null +++ b/0082-fix-shim-controller-set-incorrect-sandbox-status-sta.patch @@ -0,0 +1,60 @@ +From 1d51e3e9f14199854cc2d586651c5809345aee18 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Wed, 8 May 2024 14:48:47 +0800 +Subject: [PATCH 82/85] fix shim controller set incorrect sandbox status state + +Signed-off-by: jikai <jikai11@huawei.com> +--- + src/daemon/sandbox/controller/controller.h | 3 +++ + src/daemon/sandbox/controller/shim/shim_controller.cc | 6 ++++-- + src/daemon/sandbox/sandbox.cc | 3 --- + 3 files changed, 7 insertions(+), 5 deletions(-) + +diff --git a/src/daemon/sandbox/controller/controller.h b/src/daemon/sandbox/controller/controller.h +index f479a0ac..9ad45855 100644 +--- a/src/daemon/sandbox/controller/controller.h ++++ b/src/daemon/sandbox/controller/controller.h +@@ -27,6 +27,9 @@ + + namespace sandbox { + ++#define SANDBOX_READY_STATE_STR "SANDBOX_READY" ++#define SANDBOX_NOTREADY_STATE_STR "SANDBOX_NOTREADY" ++ + struct ControllerMountInfo { + std::string source; + std::string destination; +diff --git a/src/daemon/sandbox/controller/shim/shim_controller.cc b/src/daemon/sandbox/controller/shim/shim_controller.cc +index 4da637c7..ce09c076 100644 +--- a/src/daemon/sandbox/controller/shim/shim_controller.cc ++++ b/src/daemon/sandbox/controller/shim/shim_controller.cc +@@ -446,8 +446,10 @@ void ShimController::InspectResponseToSandboxStatus(container_inspect *inspect, + sandboxStatus.id = inspect->id; + if (inspect->state != nullptr) { + sandboxStatus.pid = inspect->state->pid; +- if (inspect->state->status != nullptr) { +- sandboxStatus.state = std::string(inspect->state->status); ++ if (inspect->state->running) { ++ sandboxStatus.state = std::string(SANDBOX_READY_STATE_STR); ++ } else { ++ sandboxStatus.state = std::string(SANDBOX_NOTREADY_STATE_STR); + } + } + +diff --git a/src/daemon/sandbox/sandbox.cc b/src/daemon/sandbox/sandbox.cc +index 279bf628..d44abb99 100644 +--- a/src/daemon/sandbox/sandbox.cc ++++ b/src/daemon/sandbox/sandbox.cc +@@ -39,9 +39,6 @@ + #include "utils_timestamp.h" + #include "mailbox.h" + +-#define SANDBOX_READY_STATE_STR "SANDBOX_READY" +-#define SANDBOX_NOTREADY_STATE_STR "SANDBOX_NOTREADY" +- + namespace sandbox { + + const std::string SHM_MOUNT_POINT = "/dev/shm"; +-- +2.34.1 + diff --git a/0083-fix-bug-for-invalid-env-write.patch b/0083-fix-bug-for-invalid-env-write.patch new file mode 100644 index 0000000..527fe0f --- /dev/null +++ b/0083-fix-bug-for-invalid-env-write.patch @@ -0,0 +1,158 @@ +From fb48f036fece9d64c4cfc19c52091afad5f42fd9 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Sat, 11 May 2024 03:46:02 +0000 +Subject: [PATCH 83/85] fix bug for invalid env write + +Signed-off-by: jikai <jikai11@huawei.com> +--- + src/daemon/modules/spec/specs_extend.c | 57 +++++++++----------------- + src/utils/cutils/utils_verify.c | 25 +++++++++++ + src/utils/cutils/utils_verify.h | 2 + + 3 files changed, 46 insertions(+), 38 deletions(-) + +diff --git a/src/daemon/modules/spec/specs_extend.c b/src/daemon/modules/spec/specs_extend.c +index 4c154281..f4208405 100644 +--- a/src/daemon/modules/spec/specs_extend.c ++++ b/src/daemon/modules/spec/specs_extend.c +@@ -190,41 +190,33 @@ int make_userns_remap(oci_runtime_spec *container, const char *user_remap) + static int generate_env_map_from_file(FILE *fp, json_map_string_string *env_map) + { + int ret = 0; +- char *key = NULL; +- char *value = NULL; +- char *pline = NULL; ++ __isula_auto_free char *pline = NULL; + size_t length = 0; +- char *saveptr = NULL; +- char empty_str[1] = {'\0'}; + + while (getline(&pline, &length, fp) != -1) { ++ __isula_auto_free char *key = NULL; ++ __isula_auto_free char *value = NULL; + util_trim_newline(pline); + pline = util_trim_space(pline); + if (pline == NULL || pline[0] == '#') { + continue; + } +- key = strtok_r(pline, "=", &saveptr); +- value = strtok_r(NULL, "=", &saveptr); +- // value of an env varible is allowed to be empty +- value = value ? value : empty_str; +- if (key != NULL) { +- key = util_trim_space(key); +- value = util_trim_space(value); +- if ((size_t)(MAX_BUFFER_SIZE - 1) - strlen(key) < strlen(value)) { +- ERROR("env length exceed %d bytes", MAX_BUFFER_SIZE); +- ret = -1; +- goto out; +- } +- ret = append_json_map_string_string(env_map, key, value); +- if (ret < 0) { +- ERROR("append env to map failed"); +- goto out; +- } ++ if (util_valid_split_env(pline, &key, &value) < 0) { ++ // ignore invalid env ++ continue; ++ } ++ if ((size_t)(MAX_BUFFER_SIZE - 1) - strlen(key) < strlen(value)) { ++ ERROR("env length exceed %d bytes", MAX_BUFFER_SIZE); ++ return -1; ++ } ++ ret = append_json_map_string_string(env_map, key, value); ++ if (ret < 0) { ++ ERROR("append env to map failed"); ++ return -1; + } + } +-out: +- free(pline); +- return ret; ++ ++ return 0; + } + + static json_map_string_string *parse_env_target_file(const char *env_path) +@@ -293,28 +285,17 @@ static int do_append_env(char ***env, size_t *env_len, const char *key, const ch + static int check_env_need_append(const oci_runtime_spec *oci_spec, const char *env_key, bool *is_append) + { + size_t i = 0; +- char *key = NULL; +- char *saveptr = NULL; + + for (i = 0; i < oci_spec->process->env_len; i++) { +- char *tmp_env = NULL; +- tmp_env = util_strdup_s(oci_spec->process->env[i]); +- key = strtok_r(tmp_env, "=", &saveptr); +- // value of an env varible is allowed to be empty +- if (key == NULL) { ++ __isula_auto_free char *key = NULL; ++ if (util_valid_split_env(oci_spec->process->env[i], &key, NULL) < 0) { + ERROR("Bad env format"); +- free(tmp_env); +- tmp_env = NULL; + return -1; + } + if (strcmp(key, env_key) == 0) { + *is_append = false; +- free(tmp_env); +- tmp_env = NULL; + return 0; + } +- free(tmp_env); +- tmp_env = NULL; + } + return 0; + } +diff --git a/src/utils/cutils/utils_verify.c b/src/utils/cutils/utils_verify.c +index 474e28f0..6f1da12c 100644 +--- a/src/utils/cutils/utils_verify.c ++++ b/src/utils/cutils/utils_verify.c +@@ -651,6 +651,31 @@ bool util_valid_device_cgroup_rule(const char *value) + return util_reg_match(patten, value) == 0; + } + ++int util_valid_split_env(const char *env, char **key, char **value) ++{ ++ __isula_auto_array_t char **arr = NULL; ++ ++ arr = util_string_split_n(env, '=', 2); ++ if (arr == NULL) { ++ ERROR("Failed to split env string"); ++ return -1; ++ } ++ ++ if (strlen(arr[0]) == 0) { ++ ERROR("Invalid environment variable: %s", env); ++ return -1; ++ } ++ ++ if (key != NULL) { ++ *key = util_strdup_s(arr[0]); ++ } ++ if (value != NULL) { ++ *value = util_strdup_s(util_array_len((const char **)arr) > 1 ? arr[1] : ""); ++ } ++ ++ return 0; ++} ++ + int util_valid_env(const char *env, char **dst) + { + int ret = 0; +diff --git a/src/utils/cutils/utils_verify.h b/src/utils/cutils/utils_verify.h +index fc59f6c0..58b22b85 100644 +--- a/src/utils/cutils/utils_verify.h ++++ b/src/utils/cutils/utils_verify.h +@@ -119,6 +119,8 @@ bool util_valid_positive_interger(const char *value); + + bool util_valid_device_cgroup_rule(const char *value); + ++int util_valid_split_env(const char *env, char **key, char **value); ++ + int util_valid_env(const char *env, char **dst); + + bool util_valid_sysctl(const char *sysctl_key); +-- +2.34.1 + diff --git a/0084-trim-key-value-for-env.patch b/0084-trim-key-value-for-env.patch new file mode 100644 index 0000000..628f77d --- /dev/null +++ b/0084-trim-key-value-for-env.patch @@ -0,0 +1,26 @@ +From de9ed770a254c8f67ac228f56fe461e1c834609c Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Sat, 11 May 2024 07:51:35 +0000 +Subject: [PATCH 84/85] trim key/value for env + +Signed-off-by: jikai <jikai11@huawei.com> +--- + src/daemon/modules/spec/specs_extend.c | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/src/daemon/modules/spec/specs_extend.c b/src/daemon/modules/spec/specs_extend.c +index f4208405..926aaf3c 100644 +--- a/src/daemon/modules/spec/specs_extend.c ++++ b/src/daemon/modules/spec/specs_extend.c +@@ -205,6 +205,8 @@ static int generate_env_map_from_file(FILE *fp, json_map_string_string *env_map) + // ignore invalid env + continue; + } ++ key = util_trim_space(key); ++ value = util_trim_space(value); + if ((size_t)(MAX_BUFFER_SIZE - 1) - strlen(key) < strlen(value)) { + ERROR("env length exceed %d bytes", MAX_BUFFER_SIZE); + return -1; +-- +2.34.1 + diff --git a/0085-cdi-allow-env-variable-has-an-empty-value.patch b/0085-cdi-allow-env-variable-has-an-empty-value.patch new file mode 100644 index 0000000..a694d75 --- /dev/null +++ b/0085-cdi-allow-env-variable-has-an-empty-value.patch @@ -0,0 +1,135 @@ +From 9208d73274da0bd18c0d77cdf59ead3dc8e06021 Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Fri, 10 May 2024 18:12:49 +0800 +Subject: [PATCH 85/85] cdi:allow env variable has an empty value + +Signed-off-by: liuxu <liuxu156@huawei.com> +--- + src/daemon/modules/spec/specs.c | 28 ++++++---------------------- + test/specs/specs/specs_ut.cc | 16 ++++++++-------- + 2 files changed, 14 insertions(+), 30 deletions(-) + +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index 65a860d4..e779c22e 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -2607,17 +2607,11 @@ int spec_module_init(void) + static int add_env(defs_process *dp, const char *env, const char *key) + { + size_t i; +- char *oci_key = NULL; +- char *oci_value = NULL; +- char *saveptr = NULL; +- __isula_auto_free char *tmp_env = NULL; + + for (i = 0; i < dp->env_len; i++) { +- tmp_env = util_strdup_s(dp->env[i]); +- oci_key = strtok_r(tmp_env, "=", &saveptr); +- oci_value = strtok_r(NULL, "=", &saveptr); +- if (oci_key == NULL || oci_value == NULL) { +- ERROR("Bad env format"); ++ __isula_auto_free char *oci_key = NULL; ++ if (util_valid_split_env(dp->env[i], &oci_key, NULL) < 0) { ++ ERROR("Bad env format, %s", dp->env[i]); + return -1; + } + if (strcmp(key, oci_key) == 0) { +@@ -2625,8 +2619,6 @@ static int add_env(defs_process *dp, const char *env, const char *key) + dp->env[i] = util_strdup_s(env); + return 0; + } +- free(tmp_env); +- tmp_env = NULL; + } + if (util_mem_realloc((void **)&dp->env, (dp->env_len + 1) * sizeof(char *), + (void *)dp->env, dp->env_len * sizeof(char *)) != 0) { +@@ -2641,10 +2633,6 @@ static int add_env(defs_process *dp, const char *env, const char *key) + int defs_process_add_multiple_env(defs_process *dp, const char **envs, size_t env_len) + { + size_t i; +- char *key = NULL; +- char *value = NULL; +- char *saveptr = NULL; +- __isula_auto_free char *tmp_env = NULL; + + if (envs == NULL || env_len == 0) { + DEBUG("empty envs"); +@@ -2656,18 +2644,14 @@ int defs_process_add_multiple_env(defs_process *dp, const char **envs, size_t en + } + + for (i = 0; i < env_len; i++) { +- tmp_env = util_strdup_s(envs[i]); +- key = strtok_r(tmp_env, "=", &saveptr); +- value = strtok_r(NULL, "=", &saveptr); +- if (key == NULL || value == NULL) { +- ERROR("Bad env format: %s", tmp_env); ++ __isula_auto_free char *key = NULL; ++ if (util_valid_split_env(envs[i], &key, NULL) < 0) { ++ ERROR("Bad env format: %s", envs[i]); + return -1; + } + if (add_env(dp, envs[i], key) != 0) { + return -1; + } +- free(tmp_env); +- tmp_env = NULL; + } + + return 0; +diff --git a/test/specs/specs/specs_ut.cc b/test/specs/specs/specs_ut.cc +index 47836e5b..3f108f0f 100644 +--- a/test/specs/specs/specs_ut.cc ++++ b/test/specs/specs/specs_ut.cc +@@ -593,20 +593,20 @@ TEST_F(SpecsUnitTest, test_defs_process_add_multiple_env) + ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1); + free(envs[0]); + envs[0] = util_strdup_s("key0="); +- ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1); ++ ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), 0); + free(envs[0]); + envs[0] = util_strdup_s("key0xxxx"); +- ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1); ++ ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), 0); + + free(dp->env[0]); + dp->env[0] = util_strdup_s("=value0"); + ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1); + free(dp->env[0]); + dp->env[0] = util_strdup_s("key0="); +- ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1); ++ ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), 0); + free(dp->env[0]); + dp->env[0] = util_strdup_s("key0xxxx"); +- ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1); ++ ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), 0); + + free_defs_process(dp); + free(envs[0]); +@@ -644,20 +644,20 @@ TEST_F(SpecsUnitTest, test_spec_add_multiple_process_env) + ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1); + free(envs[0]); + envs[0] = util_strdup_s("key0="); +- ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1); ++ ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), 0); + free(envs[0]); + envs[0] = util_strdup_s("key0xxxx"); +- ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1); ++ ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), 0); + + free(oci_spec->process->env[0]); + oci_spec->process->env[0] = util_strdup_s("=value0"); + ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1); + free(oci_spec->process->env[0]); + oci_spec->process->env[0] = util_strdup_s("key0="); +- ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1); ++ ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), 0); + free(oci_spec->process->env[0]); + oci_spec->process->env[0] = util_strdup_s("key0xxxx"); +- ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1); ++ ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), 0); + + free_oci_runtime_spec(oci_spec); + free(envs[0]); +-- +2.34.1 + diff --git a/0086-cdi-test-case-and-gateway.patch b/0086-cdi-test-case-and-gateway.patch new file mode 100644 index 0000000..42cef23 --- /dev/null +++ b/0086-cdi-test-case-and-gateway.patch @@ -0,0 +1,359 @@ +From 3fb6de60ecf10278a676a41f8d0c334d1f90d303 Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Wed, 17 Apr 2024 18:02:48 +0800 +Subject: [PATCH 086/108] cdi:test case and gateway + +Signed-off-by: liuxu <liuxu156@huawei.com> +--- + CI/make-and-install.sh | 2 +- + CI/test_cases/container_cases/cdi_test.sh | 205 ++++++++++++++++++ + .../criconfigs/container-config-cdi.json | 50 +++++ + .../test_data/cdi_devices.json | 53 +++++ + 4 files changed, 309 insertions(+), 1 deletion(-) + create mode 100755 CI/test_cases/container_cases/cdi_test.sh + create mode 100644 CI/test_cases/container_cases/criconfigs/container-config-cdi.json + create mode 100644 CI/test_cases/container_cases/test_data/cdi_devices.json + +diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh +index 9d4c5533..61281965 100755 +--- a/CI/make-and-install.sh ++++ b/CI/make-and-install.sh +@@ -105,7 +105,7 @@ cd build + if [[ ${enable_gcov} -ne 0 ]]; then + cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_CRI_API_V1=ON -DENABLE_CDI=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON -DENABLE_GRPC_REMOTE_CONNECT=ON .. + else +- cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON -DENABLE_CRI_API_V1=ON -DENABLE_GRPC_REMOTE_CONNECT=ON .. ++ cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON -DENABLE_CRI_API_V1=ON -DENABLE_CDI=ON -DENABLE_GRPC_REMOTE_CONNECT=ON .. + fi + make -j $(nproc) + make install +diff --git a/CI/test_cases/container_cases/cdi_test.sh b/CI/test_cases/container_cases/cdi_test.sh +new file mode 100755 +index 00000000..dd7b1177 +--- /dev/null ++++ b/CI/test_cases/container_cases/cdi_test.sh +@@ -0,0 +1,205 @@ ++#!/bin/bash ++# ++# attributes: isulad cdi ++# concurrent: NA ++# spend time: 41 ++ ++####################################################################### ++##- Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++# - iSulad licensed under the Mulan PSL v2. ++# - You can use this software according to the terms and conditions of the Mulan PSL v2. ++# - You may obtain a copy of Mulan PSL v2 at: ++# - http://license.coscl.org.cn/MulanPSL2 ++# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++# - PURPOSE. ++# - See the Mulan PSL v2 for more details. ++##- @Description:CI ++##- @Author: liuxu ++##- @Create: 2024-04-16 ++####################################################################### ++ ++source ../helpers.sh ++curr_path=$(dirname $(readlink -f "$0")) ++data_path=$(realpath $curr_path/criconfigs) ++pause_img_path=$(realpath $curr_path/test_data) ++cdi_static_dir="/etc/cdi" ++ ++function do_pre() ++{ ++ cp /etc/isulad/daemon.json /etc/isulad/daemon.bak ++ sed -i "s#\"pod-sandbox-image\": \"\"#\"pod-sandbox-image\": \"mirrorgooglecontainers/pause-amd64:3.0\"#g" /etc/isulad/daemon.json ++ sed -i "/\"cni-conf-dir\": \".*\"/a\ \ \ \ \"enable-cri-v1\": true," /etc/isulad/daemon.json ++ sed -i "/\"cni-conf-dir\": \".*\"/a\ \ \ \ \"enable-cdi\": true," /etc/isulad/daemon.json ++ ++ check_valgrind_log ++ start_isulad_without_valgrind ++ ++ isula load -i ${pause_img_path}/pause.tar ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to load pause" && return ${FAILURE} ++ ++ isula pull busybox ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull busybox" && return ${FAILURE} ++ ++ crictl images | grep "mirrorgooglecontainers/pause-amd64" ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Failed to find mirrorgooglecontainers/pause-amd64 image" && return ${FAILURE} ++ ++ return 0 ++} ++ ++function do_post() ++{ ++ cp -f /etc/isulad/daemon.bak /etc/isulad/daemon.json ++ check_valgrind_log ++ start_isulad_without_valgrind ++} ++ ++function verify_injected_vendor0() { ++ # check env ++ output=$(crictl exec --sync "$1" sh -c 'echo $VENDOR0') ++ [[ "$output" != "injected" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - env check failed" && return ${FAILURE} ++ ++ # check hooks ++ cat /tmp/cdi_hook_test.log | grep "prestart" ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - hook check failed" && return ${FAILURE} ++ ++ # check mounts ++ output=$(crictl exec --sync "$1" sh -c 'stat -c %a /tmp/cdi_mounts_test') ++ [[ "$output" != "755" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - mount check failed" && return ${FAILURE} ++ ++ return 0 ++} ++ ++function verify_injected_loop8() { ++ # check env ++ output=$(crictl exec --sync "$1" sh -c 'echo $LOOP8') ++ [[ "$output" != "CDI8" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - env check failed" && return ${FAILURE} ++ ++ # check device nodes ++ output=$(crictl exec --sync "$1" sh -c 'stat -c %a /dev/loop8') ++ [[ "$output" != "640" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - device nodes check failed" && return ${FAILURE} ++ output=$(crictl exec --sync "$1" sh -c 'stat -c %t.%T /dev/loop8') ++ [[ "$output" != "7.8" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - device nodes check failed" && return ${FAILURE} ++ output=$(crictl exec --sync "$1" sh -c 'stat -c %t.%T /dev/loop8c') ++ [[ "$output" != "7.b" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - device nodes check failed" && return ${FAILURE} ++ ++ # check mounts ++ output=$(crictl exec --sync "$1" sh -c 'stat -c %a /tmp/cdi_mounts_test_loop8') ++ [[ "$output" != "755" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - mount check failed" && return ${FAILURE} ++ ++ return 0 ++} ++ ++function verify_injected_loop9() { ++ # check env ++ output=$(crictl exec --sync "$1" sh -c 'echo $LOOP9') ++ [[ "$output" != "present" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - env check failed" && return ${FAILURE} ++ ++ # check device nodes ++ output=$(crictl exec --sync "$1" sh -c 'stat -c %a /dev/loop9') ++ [[ "$output" != "644" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - device nodes check failed" && return ${FAILURE} ++ output=$(crictl exec --sync "$1" sh -c 'stat -c %t.%T /dev/loop9') ++ [[ "$output" != "7.9" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - device nodes check failed" && return ${FAILURE} ++ ++ return 0 ++} ++ ++function check_full_cdi() ++{ ++ verify_injected_vendor0 $1 ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - verify_injected_vendor0 failed" && return ${FAILURE} ++ ++ verify_injected_loop8 $1 ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - verify_injected_loop8 failed" && return ${FAILURE} ++ ++ verify_injected_loop9 $1 ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - verify_injected_loop9 failed" && return ${FAILURE} ++ ++ return 0 ++} ++ ++function do_test_help() ++{ ++ msg_info "cdi test starting..." ++ ++ isula rm -f `isula ps -a -q` ++ ++ sid=`crictl runp ${data_path}/$1` ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Failed to run sandbox" && return ${FAILURE} ++ ++ cid=`crictl create $sid ${data_path}/$2 ${data_path}/$1` ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - create container failed" && return ${FAILURE} ++ ++ crictl start $cid ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start container failed" && return ${FAILURE} ++ ++ crictl stats ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stats container failed" && return ${FAILURE} ++ ++ check_full_cdi $cid ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check cdi failed" && return ${FAILURE} ++ ++ crictl stop $cid ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop container failed" && return ${FAILURE} ++ ++ crictl rm $cid ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - rm container failed" && return ${FAILURE} ++ ++ crictl stopp $sid ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop sandbox failed" && return ${FAILURE} ++ ++ crictl rmp $sid ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - rm sandbox failed" && return ${FAILURE} ++ ++ return 0 ++} ++ ++function do_test_full_cdi() ++{ ++ if [ ! -d "$cdi_static_dir" ]; then ++ mkdir -p "$cdi_static_dir" ++ fi ++ cp -f ${pause_img_path}/cdi_devices.json ${cdi_static_dir}/cdi_devices.json ++ ++ mkdir -p /tmp/cdi_mounts_test ++ cat > /tmp/cdi_mounts_test_loop8 << EOF ++origin data ++EOF ++ chmod 755 /tmp/cdi_mounts_test_loop8 ++ mkdir -p /tmp/cdi_mounts_test_loop9 ++ ++ mknod /dev/loop8 b 7 8 ++ mknod /dev/loop9 b 7 9 ++ mknod /dev/loop8c c 7 11 ++ ++ cat > /tmp/cdi_printargs.sh << EOF ++#!/bin/bash ++echo "\$(date +'%Y-%m-%d %H:%M:%S') Input parameter: \$1 \$2" >> /tmp/cdi_hook_test.log ++EOF ++ chmod 755 /tmp/cdi_printargs.sh ++ ++ do_test_help "sandbox-config.json" "container-config-cdi.json" || ((ans++)) ++ ++ rm -f /tmp/cdi_printargs.sh ++ rm -f /tmp/cdi_hook_test.log ++ rm -f /dev/loop8 ++ rm -f /dev/loop9 ++ rm -f /dev/loop8c ++ ++ rm -f ${cdi_static_dir}/cdi_devices.json ++ rm -f /tmp/cdi_printargs ++ rmdir /tmp/cdi_mounts_test ++ rm -f /tmp/cdi_mounts_test_loop8 ++ rmdir /tmp/cdi_mounts_test_loop9 ++ rm -f /tmp/cdi_printargs.sh ++ ++ return 0 ++} ++ ++declare -i ans=0 ++ ++do_pre || ((ans++)) ++do_test_full_cdi || ((ans++)) ++do_post ++ ++show_result ${ans} "${curr_path}/${0}" +diff --git a/CI/test_cases/container_cases/criconfigs/container-config-cdi.json b/CI/test_cases/container_cases/criconfigs/container-config-cdi.json +new file mode 100644 +index 00000000..b9805c8a +--- /dev/null ++++ b/CI/test_cases/container_cases/criconfigs/container-config-cdi.json +@@ -0,0 +1,50 @@ ++{ ++ "metadata": { ++ "name": "haozi" ++ }, ++ "image":{ ++ "image": "busybox:latest" ++ }, ++ "command": [ ++ "/bin/sh", ++ "-c", ++ "i=0; while true; do echo \"$i: $(date)\"; i=$((i+1)); sleep 10; done" ++ ], ++ "labels": { ++ "filter_label_key": "filter_label_val" ++ }, ++ "annotations": { ++ "extension.network.kubernetes.io/cni/instancename": "pod_instance_name", ++ "cdi.k8s.io/test": "vendor0.com/device=loop8,vendor0.com/device=loop9" ++ }, ++ "CDI_Devices":[ ++ { "Name": "vendor0.com/device=loop8" }, ++ { "Name": "vendor0.com/device=loop9" } ++ ], ++ "mounts" : [ ++ { ++ "container_path": "/tmp/contpath", ++ "host_path": "/tmp/hostpath", ++ "readonly": true, ++ "selinux_relabel": true, ++ "propagation": 0 ++ }, ++ { ++ "container_path": "/tmp/contpath2", ++ "host_path": "/tmp/hostpath2", ++ "readonly": false, ++ "selinux_relabel": false, ++ "propagation": 0 ++ } ++ ], ++ "linux": { ++ "security_context": { ++ "namespace_options": { ++ "host_network": true, ++ "host_pid": false, ++ "host_ipc": false ++ } ++ } ++ }, ++ "log_path": "cri_test.log" ++} +diff --git a/CI/test_cases/container_cases/test_data/cdi_devices.json b/CI/test_cases/container_cases/test_data/cdi_devices.json +new file mode 100644 +index 00000000..f7fe65d4 +--- /dev/null ++++ b/CI/test_cases/container_cases/test_data/cdi_devices.json +@@ -0,0 +1,53 @@ ++{ ++ "cdiVersion": "0.6.0", ++ "kind": "vendor0.com/device", ++ "annotations":{ ++ "cdi_annotation":"cdi_annotation_value" ++ }, ++ "devices": [ ++ { ++ "name": "loop8", ++ "annotations":{ ++ "loop8_annotation":"loop8_annotation_value" ++ }, ++ "containerEdits": { ++ "env": [ ++ "LOOP8=CDI8", ++ "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/dev" ++ ], ++ "deviceNodes": [ ++ {"path": "/dev/loop8", "type": "b", "fileMode": 416}, ++ {"path": "/dev/loop8c", "type": "c"} ++ ], ++ "mounts": [ ++ {"hostPath": "/tmp/cdi_mounts_test_loop8", "containerPath": "/tmp/cdi_mounts_test_loop8", "options": ["ro","nosuid","nodev","bind"]}, ++ {"hostPath": "tmpfs", "containerPath": "/tmp/data", "type": "tmpfs", "options": ["nosuid","strictatime","mode=755","size=65536k"]} ++ ] ++ } ++ },{ ++ "name": "loop9", ++ "containerEdits": { ++ "env":["LOOP9=present"], ++ "deviceNodes": [{ ++ "path": "/dev/loop9", "type": "b", "major": 7, "minor": 9, "fileMode": 420 ++ } ++ ] ++ } ++ } ++ ], ++ "containerEdits": { ++ "env": [ ++ "VENDOR0=injected", ++ "BAR=BARVALUE1" ++ ], ++ "hooks": [ ++ {"hookName": "prestart", "path": "/tmp/cdi_printargs.sh", "args":["prestart0", "prestart1"], "env":["prestartenv=value"]}, ++ {"hookName": "prestart", "path": "/tmp/cdi_printargs.sh", "args":["prestart0", "prestart1"], "env":["prestartenv=value"]}, ++ {"hookName": "poststart", "path": "/tmp/cdi_printargs.sh", "args":["poststart0", "poststart1"], "env":["poststartenv=value"]}, ++ {"hookName": "poststop", "path": "/tmp/cdi_printargs.sh", "args":["poststop0", "poststop1"], "env":["poststopenv=value"]} ++ ], ++ "mounts": [ ++ {"hostPath": "/tmp/cdi_mounts_test", "containerPath": "/tmp/cdi_mounts_test", "options": ["ro","nosuid","nodev","bind"]} ++ ] ++ } ++} +\ No newline at end of file +-- +2.25.1 + diff --git a/0087-code-improve.patch b/0087-code-improve.patch new file mode 100644 index 0000000..66f9f05 --- /dev/null +++ b/0087-code-improve.patch @@ -0,0 +1,1690 @@ +From e17bb2f5995d261790c453d81a453d25024450d1 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Mon, 13 May 2024 17:06:07 +1400 +Subject: [PATCH 087/108] code improve + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/client/connect/protocol_type.c | 2 +- + src/cmd/isula/base/create.c | 3 +- + src/cmd/isula/inspect_format.c | 4 +- + src/cmd/isulad-shim/common.c | 2 +- + src/cmd/isulad-shim/main.c | 5 +- + src/cmd/isulad-shim/process.c | 18 +++---- + src/daemon/common/cgroup/cgroup.c | 9 ++-- + src/daemon/common/cgroup/cgroup.h | 9 ++-- + src/daemon/common/cgroup/cgroup_common.h | 9 ++-- + src/daemon/common/cgroup/cgroup_v1.c | 10 ++-- + src/daemon/common/cgroup/cgroup_v2.c | 9 ++-- + src/daemon/common/cri/cri_helpers.cc | 4 +- + src/daemon/common/cri/v1/v1_cri_helpers.cc | 11 ++-- + src/daemon/common/sysinfo.c | 4 +- + src/daemon/config/isulad_config.c | 2 +- + .../cri/v1/cri_v1_runtime_runtime_service.cc | 5 +- + .../v1/v1_cri_container_manager_service.cc | 3 +- + .../v1/v1_cri_pod_sandbox_manager_service.cc | 8 +-- + .../v1/v1_cri_pod_sandbox_manager_service.h | 6 ++- + .../cri/v1/v1_cri_runtime_service_impl.cc | 3 +- + .../cri/v1/v1_cri_runtime_service_impl.h | 3 +- + .../executor/container_cb/execution_create.c | 2 +- + src/daemon/mailbox/mailbox.c | 3 +- + src/daemon/mailbox/mailbox.h | 4 +- + src/daemon/mailbox/mailbox_message.c | 9 ++-- + src/daemon/mailbox/mailbox_message.h | 4 +- + src/daemon/modules/api/container_api.h | 3 +- + src/daemon/modules/api/network_api.h | 2 +- + .../modules/container/supervisor/supervisor.c | 3 +- + .../device/cdi/behavior/cdi_container_edits.c | 46 ++++++++-------- + .../modules/device/cdi/behavior/cdi_device.c | 16 +++--- + .../modules/device/cdi/behavior/cdi_spec.c | 10 ++-- + .../modules/device/cdi/behavior/cdi_spec.h | 2 +- + .../device/cdi/behavior/cdi_spec_dirs.c | 6 +-- + .../device/cdi/behavior/cdi_spec_dirs.h | 2 +- + .../modules/device/cdi/behavior/cdi_version.c | 2 +- + .../device/cdi/behavior/parser/cdi_parser.c | 6 +-- + src/daemon/modules/device/cdi/cdi_cache.c | 22 ++++---- + src/daemon/modules/device/cdi/cdi_cache.h | 6 +-- + src/daemon/modules/device/cdi_operate.c | 8 +-- + src/daemon/modules/image/oci/oci_image.c | 3 +- + src/daemon/modules/image/oci/progress.h | 2 +- + .../modules/image/oci/registry/registry.c | 3 +- + .../cni_operator/libcni/invoke/libcni_exec.h | 2 +- + src/daemon/modules/network/cri/adaptor_cri.c | 2 +- + src/daemon/modules/network/cri/adaptor_cri.h | 4 +- + .../modules/runtime/engines/lcr/lcr_rt_ops.c | 6 ++- + .../modules/runtime/isula/isula_rt_ops.c | 9 ++-- + .../modules/service/service_container.c | 5 +- + src/daemon/modules/spec/specs.c | 52 ++++++++++--------- + src/daemon/modules/spec/specs_mount.c | 4 +- + src/utils/cutils/blocking_queue.c | 3 +- + src/utils/cutils/network_namespace.h | 4 +- + src/utils/cutils/utils_array.c | 12 ++--- + src/utils/cutils/utils_port.h | 2 +- + src/utils/cutils/utils_verify.c | 4 +- + src/utils/progress/show.h | 8 +-- + src/utils/tar/isulad_tar.c | 2 +- + test/cgroup/cpu/cgroup_cpu_ut.cc | 2 +- + 59 files changed, 223 insertions(+), 191 deletions(-) + +diff --git a/src/client/connect/protocol_type.c b/src/client/connect/protocol_type.c +index 4d530aea..57dcf753 100644 +--- a/src/client/connect/protocol_type.c ++++ b/src/client/connect/protocol_type.c +@@ -72,7 +72,7 @@ int isula_filters_last_parse_args(size_t last_n, struct isula_filters **flt) + goto cleanup; + } + +- ret = snprintf(value, VALUE_SIZE, "%ld", last_n); ++ ret = snprintf(value, VALUE_SIZE, "%lu", last_n); + if (ret < 0 || ret >= VALUE_SIZE) { + ret = -1; + ERROR("Sprintf lastest n containers args failed"); +diff --git a/src/cmd/isula/base/create.c b/src/cmd/isula/base/create.c +index b8e3667e..543b8fd6 100644 +--- a/src/cmd/isula/base/create.c ++++ b/src/cmd/isula/base/create.c +@@ -932,7 +932,8 @@ inline static int request_pack_host_device_cgroup_rules(const struct client_argu + + inline static int request_pack_host_blockio(const struct client_arguments *args, isula_host_config_t *hostconfig) + { +- return (request_pack_host_weight_devices(args, hostconfig) != 0 || request_pack_host_device_read_bps(args, hostconfig) != 0 || ++ return (request_pack_host_weight_devices(args, hostconfig) != 0 || ++ request_pack_host_device_read_bps(args, hostconfig) != 0 || + request_pack_host_device_write_bps(args, hostconfig) != 0 || + request_pack_host_device_read_iops(args, hostconfig) != 0 || + request_pack_host_device_write_iops(args, hostconfig) != 0); +diff --git a/src/cmd/isula/inspect_format.c b/src/cmd/isula/inspect_format.c +index 36e8a31c..1fd67924 100644 +--- a/src/cmd/isula/inspect_format.c ++++ b/src/cmd/isula/inspect_format.c +@@ -26,12 +26,12 @@ + + #define PRINTF_TAB_LEN 4 + #define TOP_LEVEL_OBJ 0x10 +-#define IS_TOP_LEVEL_OBJ(value) ((value)&TOP_LEVEL_OBJ) ++#define IS_TOP_LEVEL_OBJ(value) ((value) & TOP_LEVEL_OBJ) + + #define LAST_ELEMENT_BIT 0x0F + #define NOT_LAST_ELEMENT 0x00 + #define LAST_ELEMENT 0x01 +-#define IS_LAST_ELEMENT(value) (LAST_ELEMENT == ((value)&LAST_ELEMENT_BIT)) ++#define IS_LAST_ELEMENT(value) (LAST_ELEMENT == ((value) & LAST_ELEMENT_BIT)) + + #define YAJL_TYPEOF(json) ((json)->type) + +diff --git a/src/cmd/isulad-shim/common.c b/src/cmd/isulad-shim/common.c +index a288c170..799da3c2 100644 +--- a/src/cmd/isulad-shim/common.c ++++ b/src/cmd/isulad-shim/common.c +@@ -40,7 +40,7 @@ + static __thread char *g_shim_errmsg = NULL; + + // currently, log_to_stderr is only modified in the main process +-// and there is no need to set a thread-local variable. ++// and there is no need to set a thread-local variable. + // if it can be modified by multiple threads in the future, + // this variable needs to be set as a thread-local variable. + static bool log_to_stderr = false; +diff --git a/src/cmd/isulad-shim/main.c b/src/cmd/isulad-shim/main.c +index 54876d44..9061e568 100644 +--- a/src/cmd/isulad-shim/main.c ++++ b/src/cmd/isulad-shim/main.c +@@ -103,7 +103,7 @@ int main(int argc, char **argv) + engine_log_path = getenv(SHIIM_LOG_PATH_ENV); + if (engine_log_path == NULL) { + dprintf(STDERR_FILENO, "empty SHIIM_LOG_PATH_ENV"); +- _exit(EXIT_FAILURE); ++ _exit(EXIT_FAILURE); + } + + log_level = getenv(SHIIM_LOG_LEVEL_ENV); +@@ -149,7 +149,8 @@ int main(int argc, char **argv) + + // If isulad-shim is a child process of the isulad process, + // print the log to stderr so that isulad can obtain the exit information of isulad-shim. +- set_log_to_stderr((p->state->exec) && (p->state->isulad_stdin != NULL || p->state->isulad_stdout != NULL || p->state->isulad_stderr != NULL)); ++ set_log_to_stderr((p->state->exec) && (p->state->isulad_stdin != NULL || p->state->isulad_stdout != NULL || ++ p->state->isulad_stderr != NULL)); + + /* + * Open exit pipe +diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c +index 6b5f8f7f..18fae03f 100644 +--- a/src/cmd/isulad-shim/process.c ++++ b/src/cmd/isulad-shim/process.c +@@ -199,7 +199,7 @@ static int add_attach_terminal_fifos(const char *in, const char *out, const char + struct isula_linked_list *node = NULL; + + bool invalid = (in != NULL && !fifo_exists(in)) || (out != NULL && !fifo_exists(out)) || (err != NULL && +- !fifo_exists(err)); ++ !fifo_exists(err)); + if (invalid) { + ERROR("File %s or %s or %s does not refer to a FIFO", in, out, err); + return -1; +@@ -371,7 +371,7 @@ static int stdout_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t + (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF); + + r_count = isula_file_read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF); +- if (r_count <= 0 ) { ++ if (r_count <= 0) { + isula_epoll_remove_handler(descr, fd); + // fd cannot be closed here, which will cause the container process to exit abnormally + // due to terminal fd receiving the sighup signal. +@@ -417,7 +417,7 @@ static int stderr_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t + (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF); + + r_count = isula_file_read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF); +- if (r_count <= 0 ) { ++ if (r_count <= 0) { + isula_epoll_remove_handler(descr, fd); + // fd cannot be closed here, which will cause the container process to exit abnormally + // due to terminal fd receiving the sighup signal. +@@ -563,7 +563,7 @@ static int attach_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t + // limit the number of attach connections to MAX_ATTACH_NUM + if (isula_linked_list_len(p->attach_fifos) >= MAX_ATTACH_NUM) { + ERROR("The number of attach connections exceeds the limit:%d, and this connection is rejected.", +- MAX_ATTACH_NUM); ++ MAX_ATTACH_NUM); + goto out; + } + +@@ -934,8 +934,8 @@ static void *io_epoll_loop(void *data) + + (void)sem_post(&p->sem_mainloop); + +- // th frist epoll_loop will exit in the following scenarios: +- // 1. Receive sync fd event ++ // th frist epoll_loop will exit in the following scenarios: ++ // 1. Receive sync fd event + // 2. stdin fd receive EPOLLHUP event + // 3. stdin fd read failed + ret = isula_epoll_loop(&descr, -1); +@@ -945,8 +945,8 @@ static void *io_epoll_loop(void *data) + error_exit(EXIT_FAILURE); + } + +- // use a timeout epoll loop to ensure complete data reception +- // th second epoll_loop will exit in the following scenarios: ++ // use a timeout epoll loop to ensure complete data reception ++ // th second epoll_loop will exit in the following scenarios: + // 1. both stdout fd and stderr fd failed to read + // 2. no event received within 100 milliseconds + ret = isula_epoll_loop(&descr, 100); +@@ -1408,7 +1408,7 @@ static void exec_runtime_process(process_t *p, int exec_fd) + // the standard streams of the child process are set to /dev/null to prevent incorrect information acquisition. + if (isula_null_stdfds() != 0) { + (void)dprintf(exec_fd, "failed to set std console to /dev/null"); +- exit(EXIT_FAILURE); ++ exit(EXIT_FAILURE); + } + } + +diff --git a/src/daemon/common/cgroup/cgroup.c b/src/daemon/common/cgroup/cgroup.c +index 007dbb70..71bf9801 100644 +--- a/src/daemon/common/cgroup/cgroup.c ++++ b/src/daemon/common/cgroup/cgroup.c +@@ -82,9 +82,9 @@ int common_get_cgroup_version(void) + } + + int common_get_cgroup_info(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, +- cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, +- cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, +- cgroup_files_info_t *filesinfo, bool quiet) ++ cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, ++ cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, ++ cgroup_files_info_t *filesinfo, bool quiet) + { + if (g_cgroup_ops.get_cgroup_info == NULL) { + ERROR("Unimplemented get_cgroup_info ops"); +@@ -197,7 +197,8 @@ char *common_convert_cgroup_path(const char *cgroup_path) + return util_strdup_s(result); + } + +-cgroup_oom_handler_info_t *common_get_cgroup_oom_handler(int fd, const char *name, const char *cgroup_path, const char *exit_fifo) ++cgroup_oom_handler_info_t *common_get_cgroup_oom_handler(int fd, const char *name, const char *cgroup_path, ++ const char *exit_fifo) + { + if (g_cgroup_ops.get_cgroup_oom_handler == NULL) { + ERROR("Unimplmented get_cgroup_oom_handler op"); +diff --git a/src/daemon/common/cgroup/cgroup.h b/src/daemon/common/cgroup/cgroup.h +index 8c76d99d..0bbb70a0 100644 +--- a/src/daemon/common/cgroup/cgroup.h ++++ b/src/daemon/common/cgroup/cgroup.h +@@ -31,9 +31,9 @@ int cgroup_ops_init(void); + + int common_get_cgroup_version(void); + int common_get_cgroup_info(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, +- cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, +- cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, +- cgroup_files_info_t *filesinfo, bool quiet); ++ cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, ++ cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, ++ cgroup_files_info_t *filesinfo, bool quiet); + int common_get_cgroup_metrics(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics); + int common_get_cgroup_mnt_and_root_path(const char *subsystem, char **mountpoint, char **root); + +@@ -43,7 +43,8 @@ char *common_get_own_cgroup_path(const char *subsystem); + + char *common_convert_cgroup_path(const char *cgroup_path); + +-cgroup_oom_handler_info_t *common_get_cgroup_oom_handler(int fd, const char *name, const char *cgroup_path, const char *exit_fifo); ++cgroup_oom_handler_info_t *common_get_cgroup_oom_handler(int fd, const char *name, const char *cgroup_path, ++ const char *exit_fifo); + void common_free_cgroup_oom_handler_info(cgroup_oom_handler_info_t *info); + + #ifdef __cplusplus +diff --git a/src/daemon/common/cgroup/cgroup_common.h b/src/daemon/common/cgroup/cgroup_common.h +index e3912bf0..46a7de50 100644 +--- a/src/daemon/common/cgroup/cgroup_common.h ++++ b/src/daemon/common/cgroup/cgroup_common.h +@@ -130,9 +130,9 @@ typedef struct _cgroup_oom_handler_info_t { + typedef struct { + int (*get_cgroup_version)(void); + int (*get_cgroup_info)(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, +- cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, +- cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, +- cgroup_files_info_t *filesinfo, bool quiet); ++ cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, ++ cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, ++ cgroup_files_info_t *filesinfo, bool quiet); + int (*get_cgroup_metrics)(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics); + + int (*get_cgroup_mnt_and_root_path)(const char *subsystem, char **mountpoint, char **root); +@@ -140,7 +140,8 @@ typedef struct { + char *(*get_init_cgroup_path)(const char *subsystem); + char *(*get_own_cgroup_path)(const char *subsystem); + +- cgroup_oom_handler_info_t *(*get_cgroup_oom_handler)(int fd, const char *name, const char *cgroup_path, const char *exit_fifo); ++ cgroup_oom_handler_info_t *(*get_cgroup_oom_handler)(int fd, const char *name, const char *cgroup_path, ++ const char *exit_fifo); + } cgroup_ops; + + #ifdef __cplusplus +diff --git a/src/daemon/common/cgroup/cgroup_v1.c b/src/daemon/common/cgroup/cgroup_v1.c +index 41f3110a..45b1d096 100644 +--- a/src/daemon/common/cgroup/cgroup_v1.c ++++ b/src/daemon/common/cgroup/cgroup_v1.c +@@ -772,7 +772,8 @@ static void get_cgroup_v1_blkio_info(const cgroup_layer_t *layers, const bool qu + blkioinfo->blkio_write_iops_device = check_cgroup_v1_file_exists(mountpoint, BLKIO_WRITE_IOPS, quiet); + } + +-static void get_cgroup_v1_hugetlb_info(const cgroup_layer_t *layers, const bool quiet, cgroup_hugetlb_info_t *hugetlbinfo) ++static void get_cgroup_v1_hugetlb_info(const cgroup_layer_t *layers, const bool quiet, ++ cgroup_hugetlb_info_t *hugetlbinfo) + { + int nret; + char *mountpoint = NULL; +@@ -1038,8 +1039,8 @@ static char *common_get_cgroup_path(const char *path, const char *subsystem) + + for (i = 0; i < util_array_len((const char **)nlist); i++) { + const char *prefix = "name="; +- bool find_sub = (strcmp(nlist[i], subsystem) == 0 || (strncmp(nlist[i], prefix, strlen(prefix)) == 0 +- && strcmp(nlist[i]+strlen(prefix), subsystem) == 0)); ++ bool find_sub = (strcmp(nlist[i], subsystem) == 0 || (strncmp(nlist[i], prefix, strlen(prefix)) == 0 && ++ strcmp(nlist[i] + strlen(prefix), subsystem) == 0)); + if (find_sub) { + res = util_strdup_s(plist[i]); + break; +@@ -1135,7 +1136,8 @@ static char *get_memory_cgroup_path_v1(const char *cgroup_path) + return util_path_join(fpath, converted_cgroup_path); + } + +-static cgroup_oom_handler_info_t *get_cgroup_oom_handler_v1(int fd, const char *name, const char *cgroup_path, const char *exit_fifo) ++static cgroup_oom_handler_info_t *get_cgroup_oom_handler_v1(int fd, const char *name, const char *cgroup_path, ++ const char *exit_fifo) + { + __isula_auto_free char *memory_cgroup_path = NULL; + __isula_auto_free char *memory_cgroup_oom_control_path = NULL; +diff --git a/src/daemon/common/cgroup/cgroup_v2.c b/src/daemon/common/cgroup/cgroup_v2.c +index a36258f0..76754dc1 100644 +--- a/src/daemon/common/cgroup/cgroup_v2.c ++++ b/src/daemon/common/cgroup/cgroup_v2.c +@@ -387,7 +387,7 @@ static int get_cgroup_info_v2(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpu + if (ret != 0) { + return ret; + } +- ++ + get_cgroup_v2_pids_info(quiet, pidsinfo); + get_cgroup_v2_files_info(quiet, filesinfo); + +@@ -433,7 +433,7 @@ static bool oom_cb_cgroup_v2(int fd, void *cbdata) + return CGROUP_OOM_HANDLE_CLOSE; + } + +- if (((struct inotify_event *)events)->mask & ( IN_DELETE | IN_DELETE_SELF)) { ++ if (((struct inotify_event *)events)->mask & (IN_DELETE | IN_DELETE_SELF)) { + return CGROUP_OOM_HANDLE_CLOSE; + } + +@@ -450,7 +450,7 @@ static bool oom_cb_cgroup_v2(int fd, void *cbdata) + int count; + const char *oom_str = "oom "; + const char *oom_kill_str = "oom_kill "; +- const int oom_len = strlen(oom_str), oom_kill_len = strlen(oom_kill_str); ++ const size_t oom_len = strlen(oom_str), oom_kill_len = strlen(oom_kill_str); + + if (read >= oom_kill_len + 2 && memcmp(line, oom_kill_str, oom_kill_len) == 0) { + len = oom_kill_len; +@@ -492,7 +492,8 @@ static char *get_real_cgroup_path_v2(const char *cgroup_path) + return util_path_join(CGROUP_MOUNTPOINT, converted_cgroup_path); + } + +-cgroup_oom_handler_info_t *get_cgroup_oom_handler_v2(int fd, const char *name, const char *cgroup_path, const char *exit_fifo) ++cgroup_oom_handler_info_t *get_cgroup_oom_handler_v2(int fd, const char *name, const char *cgroup_path, ++ const char *exit_fifo) + { + __isula_auto_free char *real_cgroup_path = NULL; + if (name == NULL || cgroup_path == NULL || exit_fifo == NULL) { +diff --git a/src/daemon/common/cri/cri_helpers.cc b/src/daemon/common/cri/cri_helpers.cc +index d7ec9f36..68d569cc 100644 +--- a/src/daemon/common/cri/cri_helpers.cc ++++ b/src/daemon/common/cri/cri_helpers.cc +@@ -550,8 +550,8 @@ void CreateContainerLogSymlink(const std::string &containerID, Errors &error) + WARN("Deleted previously existing symlink file: %s", path.c_str()); + } + if (symlink(realPath.c_str(), path.c_str()) != 0) { +- SYSERROR("failed to create symbolic link %s to the container log file %s for container %s", path.c_str(), realPath.c_str(), +- containerID.c_str()); ++ SYSERROR("failed to create symbolic link %s to the container log file %s for container %s", path.c_str(), ++ realPath.c_str(), containerID.c_str()); + error.Errorf("failed to create symbolic link %s to the container log file %s for container %s", path.c_str(), + realPath.c_str(), containerID.c_str()); + } +diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.cc b/src/daemon/common/cri/v1/v1_cri_helpers.cc +index 1f797ad7..478dd105 100644 +--- a/src/daemon/common/cri/v1/v1_cri_helpers.cc ++++ b/src/daemon/common/cri/v1/v1_cri_helpers.cc +@@ -636,7 +636,8 @@ void ContainerStatusToGRPC(container_inspect *inspect, + ConvertResourcesToStatus(inspect, contStatus); + } + +-std::unique_ptr<runtime::v1::ContainerStatus> GetContainerStatus(service_executor_t *m_cb, const std::string &containerID, Errors &error) ++std::unique_ptr<runtime::v1::ContainerStatus> GetContainerStatus(service_executor_t *m_cb, ++ const std::string &containerID, Errors &error) + { + if (m_cb == nullptr) { + error.SetError("Invalid input arguments: empty service executor"); +@@ -694,7 +695,7 @@ static int InsertCDIDevices(std::unordered_set<std::string> &fromCRI, const std: + } + return 0; + } +- ++ + void GenerateCDIRequestedDevices(const runtime::v1::ContainerConfig &config, host_config *hostconfig, Errors &err) + { + std::unordered_set<std::string> fromCRI; +@@ -703,13 +704,13 @@ void GenerateCDIRequestedDevices(const runtime::v1::ContainerConfig &config, hos + __isula_auto_string_array_t string_array *devices = nullptr; + json_map_string_string *annotations = nullptr; + __isula_auto_free char *error = nullptr; +- ++ + if (hostconfig == nullptr) { + ERROR("Invalid input arguments"); + err.Errorf("Invalid input arguments"); + return; + } +- ++ + if (config.cdi_devices().empty() && config.annotations().empty()) { + return; + } +@@ -746,7 +747,7 @@ void GenerateCDIRequestedDevices(const runtime::v1::ContainerConfig &config, hos + requested->items = nullptr; + hostconfig->cdi_requested_devices_len = requested->len; + requested->len = 0; +- ++ + free_out: + free_json_map_string_string(annotations); + } +diff --git a/src/daemon/common/sysinfo.c b/src/daemon/common/sysinfo.c +index e369c3e3..ed1c01a3 100644 +--- a/src/daemon/common/sysinfo.c ++++ b/src/daemon/common/sysinfo.c +@@ -384,8 +384,8 @@ sysinfo_t *get_sys_info(bool quiet) + sysinfo->ncpus_conf = get_nprocs_conf(); + + ret = common_get_cgroup_info(&sysinfo->cgmeminfo, &sysinfo->cgcpuinfo, &sysinfo->hugetlbinfo, +- &sysinfo->blkioinfo, &sysinfo->cpusetinfo, &sysinfo->pidsinfo, +- &sysinfo->filesinfo, quiet); ++ &sysinfo->blkioinfo, &sysinfo->cpusetinfo, &sysinfo->pidsinfo, ++ &sysinfo->filesinfo, quiet); + if (ret != 0) { + goto out; + } +diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c +index 617db7a2..80689bce 100644 +--- a/src/daemon/config/isulad_config.c ++++ b/src/daemon/config/isulad_config.c +@@ -1839,7 +1839,7 @@ int merge_json_confs_into_global(struct service_arguments *args) + args->json_confs->cdi_spec_dirs_len = tmp_json_confs->cdi_spec_dirs_len; + tmp_json_confs->cdi_spec_dirs_len = 0; + #endif /* ENABLE_CDI */ +- ++ + out: + free(err); + free_isulad_daemon_configs(tmp_json_confs); +diff --git a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc +index fb5aad3c..f8df8e0a 100644 +--- a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc ++++ b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc +@@ -90,7 +90,8 @@ void RuntimeV1RuntimeServiceImpl::Init(std::string &podSandboxImage, + m_enablePodEvents = enablePodEvents; + } + +- m_rService = std::unique_ptr<CRIV1::CRIRuntimeService>(new CRIRuntimeServiceImpl(podSandboxImage, cb, networkPlugin, m_enablePodEvents)); ++ m_rService = std::unique_ptr<CRIV1::CRIRuntimeService>(new CRIRuntimeServiceImpl(podSandboxImage, cb, networkPlugin, ++ m_enablePodEvents)); + } + + void RuntimeV1RuntimeServiceImpl::Wait() +@@ -134,7 +135,7 @@ auto RuntimeV1RuntimeServiceImpl::GenerateCRIContainerEvent(const char *containe + m_rService->PodSandboxStatus(sandboxID, statusReply, error); + if (!error.Empty()) { + WARN("Object: CRI, Type: Failed to status pod:%s due to %s", sandboxID.c_str(), +- error.GetMessage().c_str()); ++ error.GetMessage().c_str()); + } else { + *(response->mutable_pod_sandbox_status()) = *(statusReply->mutable_status()); + for (auto &containerStatus : statusReply->containers_statuses()) { +diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +index 1097c32c..1cee68ec 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +@@ -1015,7 +1015,8 @@ cleanup: + return contStats; + } + +-std::unique_ptr<runtime::v1::ContainerStatus> ContainerManagerService::ContainerStatus(const std::string &containerID, Errors &error) ++std::unique_ptr<runtime::v1::ContainerStatus> ContainerManagerService::ContainerStatus(const std::string &containerID, ++ Errors &error) + { + return CRIHelpersV1::GetContainerStatus(m_cb, containerID, error); + } +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +index 4291d8a0..fa726e2c 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +@@ -819,7 +819,8 @@ void PodSandboxManagerService::SetSandboxStatusNetwork(std::shared_ptr<sandbox:: + + void PodSandboxManagerService::GetContainerStatuses(const std::string &podSandboxID, + std::vector<std::unique_ptr<runtime::v1::ContainerStatus>> &containerStatuses, +- std::vector<std::string> &errors) { ++ std::vector<std::string> &errors) ++{ + auto list_response_wrapper = GetContainerListResponse(podSandboxID, errors); + if (list_response_wrapper == nullptr) { + return; +@@ -837,7 +838,8 @@ void PodSandboxManagerService::GetContainerStatuses(const std::string &podSandbo + } + } + +-std::unique_ptr<runtime::v1::PodSandboxStatus> PodSandboxManagerService::GetPodSandboxStatus(const std::string &podSandboxID, Errors &error) ++std::unique_ptr<runtime::v1::PodSandboxStatus> PodSandboxManagerService::GetPodSandboxStatus( ++ const std::string &podSandboxID, Errors &error) + { + std::unique_ptr<runtime::v1::PodSandboxStatus> podStatus(new (std::nothrow) runtime::v1::PodSandboxStatus); + if (podStatus == nullptr) { +@@ -876,7 +878,7 @@ void PodSandboxManagerService::PodSandboxStatus(const std::string &podSandboxID, + return; + } + +- ++ + auto podStatus = GetPodSandboxStatus(podSandboxID, error); + if (error.NotEmpty()) { + ERROR("Failed to get pod sandbox status: %s", error.GetCMessage()); +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h +index 3872c4c9..d5c0cf91 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h +@@ -129,8 +129,10 @@ private: + std::vector<std::string> &podSandboxIDs, Errors &error); + void ApplySandboxLinuxOptions(const runtime::v1::LinuxPodSandboxConfig &lc, host_config *hc, + container_config *custom_config, Errors &error); +- auto GetPodSandboxStatus(const std::string &podSandboxID, Errors &error) -> std::unique_ptr<runtime::v1::PodSandboxStatus>; +- void GetContainerStatuses(const std::string &podSandboxID, std::vector<std::unique_ptr<runtime::v1::ContainerStatus>> &containerStatuses, ++ auto GetPodSandboxStatus(const std::string &podSandboxID, ++ Errors &error) -> std::unique_ptr<runtime::v1::PodSandboxStatus>; ++ void GetContainerStatuses(const std::string &podSandboxID, ++ std::vector<std::unique_ptr<runtime::v1::ContainerStatus>> &containerStatuses, + std::vector<std::string> &errors); + + private: +diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc +index 7b40e29d..56c89c1e 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc +@@ -125,7 +125,8 @@ void CRIRuntimeServiceImpl::RemovePodSandbox(const std::string &podSandboxID, Er + m_podSandboxManager->RemovePodSandbox(podSandboxID, error); + } + +-void CRIRuntimeServiceImpl::PodSandboxStatus(const std::string &podSandboxID, runtime::v1::PodSandboxStatusResponse *reply, Errors &error) ++void CRIRuntimeServiceImpl::PodSandboxStatus(const std::string &podSandboxID, ++ runtime::v1::PodSandboxStatusResponse *reply, Errors &error) + { + m_podSandboxManager->PodSandboxStatus(podSandboxID, reply, error); + } +diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h +index 6ae59bfa..3d93c7bb 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h ++++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h +@@ -73,7 +73,8 @@ public: + + void RemovePodSandbox(const std::string &podSandboxID, Errors &error) override; + +- void PodSandboxStatus(const std::string &podSandboxID, runtime::v1::PodSandboxStatusResponse *reply, Errors &error) override; ++ void PodSandboxStatus(const std::string &podSandboxID, runtime::v1::PodSandboxStatusResponse *reply, ++ Errors &error) override; + + void ListPodSandbox(const runtime::v1::PodSandboxFilter &filter, + std::vector<std::unique_ptr<runtime::v1::PodSandbox>> &pods, Errors &error) override; +diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c +index 785b4e27..8e930c8a 100644 +--- a/src/daemon/executor/container_cb/execution_create.c ++++ b/src/daemon/executor/container_cb/execution_create.c +@@ -551,7 +551,7 @@ static int merge_config_for_syscontainer(const container_create_request *request + goto out; + } + if (request->rootfs != NULL && (append_json_map_string_string(container_spec->annotations, "external.rootfs", "true") +- || append_json_map_string_string(oci_spec->annotations, "external.rootfs", "true"))) { ++ || append_json_map_string_string(oci_spec->annotations, "external.rootfs", "true"))) { + ERROR("Realloc annotations failed"); + ret = -1; + goto out; +diff --git a/src/daemon/mailbox/mailbox.c b/src/daemon/mailbox/mailbox.c +index 732b91b9..39f003f3 100644 +--- a/src/daemon/mailbox/mailbox.c ++++ b/src/daemon/mailbox/mailbox.c +@@ -23,7 +23,8 @@ + + mailbox_topic_handler_t mailbox_topic_handlers[MAILBOX_TOPIC_MAX] = { 0 }; + +-static bool mailbox_topic_valid(mailbox_topic topic) { ++static bool mailbox_topic_valid(mailbox_topic topic) ++{ + return topic > MAILBOX_TOPIC_INVALID && topic < MAILBOX_TOPIC_MAX; + } + +diff --git a/src/daemon/mailbox/mailbox.h b/src/daemon/mailbox/mailbox.h +index 1dc2e934..aef63514 100644 +--- a/src/daemon/mailbox/mailbox.h ++++ b/src/daemon/mailbox/mailbox.h +@@ -64,7 +64,7 @@ typedef struct { + cri_container_message_type type; + } cri_container_message_t; + +-int mailbox_register_topic_handler(mailbox_topic topic, message_generator_t handle, void *context, ++int mailbox_register_topic_handler(mailbox_topic topic, message_generator_t generator, void *context, + message_release_t release, bool async); + + void mailbox_unregister_topic_handler(mailbox_topic topic); +@@ -73,7 +73,7 @@ void mailbox_publish(mailbox_topic topic, void *data); + + message_subscriber *mailbox_subscribe(mailbox_topic topic); + +-void mailbox_unsubscribe(mailbox_topic, message_subscriber *sub); ++void mailbox_unsubscribe(mailbox_topic topic, message_subscriber *sub); + + #ifdef __cplusplus + } +diff --git a/src/daemon/mailbox/mailbox_message.c b/src/daemon/mailbox/mailbox_message.c +index b16a1bdd..316b7428 100644 +--- a/src/daemon/mailbox/mailbox_message.c ++++ b/src/daemon/mailbox/mailbox_message.c +@@ -20,7 +20,8 @@ + #include "utils.h" + + // Once the create succeeds, the ownership is transferred to the mailbox_message. +-mailbox_message *mailbox_message_create(void *data, void (*destroy)(void *)) { ++mailbox_message *mailbox_message_create(void *data, void (*destroy)(void *)) ++{ + __isula_auto_free mailbox_message *msg = NULL; + msg = util_common_calloc_s(sizeof(mailbox_message)); + if (msg == NULL) { +@@ -40,7 +41,8 @@ mailbox_message *mailbox_message_create(void *data, void (*destroy)(void *)) { + return isula_transfer_ptr(msg); + } + +-int mailbox_message_ref(mailbox_message *dest) { ++int mailbox_message_ref(mailbox_message *dest) ++{ + __isula_auto_pm_unlock pthread_mutex_t *lock = NULL; + if (dest == NULL) { + ERROR("Invalid mailbox_message"); +@@ -63,7 +65,8 @@ int mailbox_message_ref(mailbox_message *dest) { + return 0; + } + +-void mailbox_message_unref(mailbox_message *dest) { ++void mailbox_message_unref(mailbox_message *dest) ++{ + __isula_auto_pm_unlock pthread_mutex_t *lock = NULL; + if (dest == NULL) { + return; +diff --git a/src/daemon/mailbox/mailbox_message.h b/src/daemon/mailbox/mailbox_message.h +index 39e40b70..af323069 100644 +--- a/src/daemon/mailbox/mailbox_message.h ++++ b/src/daemon/mailbox/mailbox_message.h +@@ -34,9 +34,9 @@ typedef struct mailbox_message { + + mailbox_message *mailbox_message_create(void *ptr, void (*destroy)(void *)); + +-int mailbox_message_ref(mailbox_message *p); ++int mailbox_message_ref(mailbox_message *dest); + +-void mailbox_message_unref(mailbox_message *p); ++void mailbox_message_unref(mailbox_message *dest); + + // define auto free function callback for mailbox_message + define_auto_cleanup_callback(mailbox_message_unref, mailbox_message); +diff --git a/src/daemon/modules/api/container_api.h b/src/daemon/modules/api/container_api.h +index 55c59980..a6ec8e22 100644 +--- a/src/daemon/modules/api/container_api.h ++++ b/src/daemon/modules/api/container_api.h +@@ -271,7 +271,8 @@ bool container_is_valid_state_string(const char *state); + + void container_update_health_monitor(const char *container_id); + +-extern int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const pid_ppid_info_t *pid_info, const container_t *cont); ++extern int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const pid_ppid_info_t *pid_info, ++ const container_t *cont); + + extern char *container_exit_fifo_create(const char *cont_state_path); + +diff --git a/src/daemon/modules/api/network_api.h b/src/daemon/modules/api/network_api.h +index 4c1e3480..fd8607b3 100644 +--- a/src/daemon/modules/api/network_api.h ++++ b/src/daemon/modules/api/network_api.h +@@ -77,7 +77,7 @@ void free_network_api_result_list(network_api_result_list *ptr); + + void free_attach_net_conf(struct attach_net_conf *ptr); + +-void free_network_api_conf(network_api_conf *ptr); ++void free_network_api_conf(network_api_conf *conf); + + bool network_module_init(const char *network_plugin, const char *cache_dir, const char *conf_dir, const char* bin_path); + +diff --git a/src/daemon/modules/container/supervisor/supervisor.c b/src/daemon/modules/container/supervisor/supervisor.c +index 83d46268..39d9fdb8 100644 +--- a/src/daemon/modules/container/supervisor/supervisor.c ++++ b/src/daemon/modules/container/supervisor/supervisor.c +@@ -307,7 +307,8 @@ static int oom_handle_cb(int fd, uint32_t events, void *cbdata, struct epoll_des + } + + /* supervisor add exit monitor */ +-int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const pid_ppid_info_t *pid_info, const container_t *cont) ++int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const pid_ppid_info_t *pid_info, ++ const container_t *cont) + { + int ret = 0; + struct supervisor_handler_data *data = NULL; +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c +index 816b9c2d..d146f7e1 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c +@@ -27,8 +27,8 @@ + #include "utils_array.h" + #include "specs_api.h" + +-/* +- * The OCI being used by the iSulad not supportes ++/* ++ * The OCI being used by the iSulad not supportes + * createRuntime/createContainer/startContainer currently. + */ + // PRESTART_HOOK is the name of the OCI "prestart" hook. +@@ -105,8 +105,8 @@ static int fill_device_node_info(cdi_device_node *d) + dev_type = NULL; + } else { + if (strcmp(d->type, dev_type) != 0) { +- ERROR("CDI device (%s, %s), host type mismatch (%s, %s)", +- d->path, d->host_path, d->type, dev_type); ++ ERROR("CDI device (%s, %s), host type mismatch (%s, %s)", ++ d->path, d->host_path, d->type, dev_type); + return -1; + } + } +@@ -168,7 +168,7 @@ static cdi_hook *clone_cdi_hook(cdi_hook *h) + hook->env_len = h->env_len; + } + hook->timeout = h->timeout; +- ++ + return hook; + + error_out: +@@ -280,7 +280,7 @@ static defs_device *cdi_device_node_to_oci(cdi_device_node *d) + oci_device->file_mode = d->file_mode; + oci_device->uid = d->uid; + oci_device->gid = d->gid; +- ++ + return oci_device; + } + +@@ -331,7 +331,7 @@ static int apply_cdi_device_nodes(cdi_container_edits *e, oci_runtime_spec *spec + access = "rwm"; + } + if (spec_add_linux_resources_device(spec, true, dev->type, +- dev->major, dev->minor, access)) { ++ dev->major, dev->minor, access)) { + dev = NULL; + goto error_out; + } +@@ -391,9 +391,9 @@ static int apply_cdi_mounts(cdi_container_edits *e, oci_runtime_spec *spec) + return -1; + } + } +- ++ + qsort(spec->mounts, spec->mounts_len, +- sizeof(defs_mount *), (int (*)(const void *, const void *))defs_mount_cmp); ++ sizeof(defs_mount *), (int (*)(const void *, const void *))defs_mount_cmp); + return 0; + } + +@@ -411,8 +411,8 @@ static int apply_cdi_hooks(cdi_container_edits *e, oci_runtime_spec *spec) + } else if (strcmp(e->hooks[i]->hook_name, POSTSTOP_HOOK)) { + ret = spec_add_poststop_hook(spec, oci_hook); + } else { +- /* +- * The OCI being used by the iSulad not supportes ++ /* ++ * The OCI being used by the iSulad not supportes + * createRuntime/createContainer/startContainer currently. + */ + ERROR("Unknown hook name %s", e->hooks[i]->hook_name); +@@ -503,19 +503,19 @@ int cdi_container_edits_validate(cdi_container_edits *e) + static int append_##item(cdi_container_edits *e, cdi_container_edits *o, clone_common_array_item_cb cb) \ + { \ + common_array e_array = { \ +- .items = (void **)e->item, \ +- .len = e->item##_len, \ +- .cap = e->item##_len, \ +- .free_item_cb = NULL, \ +- .clone_item_cb = cb \ +- }; \ ++ .items = (void **)e->item, \ ++ .len = e->item##_len, \ ++ .cap = e->item##_len, \ ++ .free_item_cb = NULL, \ ++ .clone_item_cb = cb \ ++ }; \ + common_array o_array = { \ +- .items = (void **)o->item, \ +- .len = o->item##_len, \ +- .cap = o->item##_len, \ +- .free_item_cb = NULL, \ +- .clone_item_cb = cb \ +- }; \ ++ .items = (void **)o->item, \ ++ .len = o->item##_len, \ ++ .cap = o->item##_len, \ ++ .free_item_cb = NULL, \ ++ .clone_item_cb = cb \ ++ }; \ + if (util_merge_common_array(&e_array, &o_array) != 0) { \ + ERROR("Out of memory"); \ + return -1; \ +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_device.c b/src/daemon/modules/device/cdi/behavior/cdi_device.c +index aec3d7c0..9104416c 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_device.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_device.c +@@ -28,21 +28,21 @@ void free_cdi_cache_device(struct cdi_cache_device *d) + if (d == NULL) { + return; + } +- +- /* ++ ++ /* + * free_cdi_cache_device should not be recursively free raw_device. + * Otherwise, the function conflicts with the raw_spec free raw_device +- * when cdi_cache_spec free raw_spec, triggering double free. ++ * when cdi_cache_spec free raw_spec, triggering double free. + */ + d->raw_device = NULL; +- +- /* ++ ++ /* + * free_cdi_cache_device should not be recursively free cache_spec. + * Otherwise, the function conflicts with the cache free specs, +- * triggering double free. ++ * triggering double free. + */ + d->cache_spec = NULL; +- ++ + free(d); + } + +@@ -86,7 +86,7 @@ char *cdi_device_get_qualified_name(const struct cdi_cache_device *d) + return NULL; + } + return cdi_parser_qualified_name(cdi_spec_get_vendor(d->cache_spec), +- cdi_spec_get_class(d->cache_spec), d->raw_device->name); ++ cdi_spec_get_class(d->cache_spec), d->raw_device->name); + } + + cdi_container_edits *cdi_device_get_edits(const struct cdi_cache_device *d) +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec.c b/src/daemon/modules/device/cdi/behavior/cdi_spec.c +index 8783debc..ec563230 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_spec.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec.c +@@ -33,7 +33,7 @@ void free_cdi_cache_spec(struct cdi_cache_spec *s) + if (s == NULL) { + return; + } +- ++ + free_cdi_spec(s->raw_spec); + s->raw_spec = NULL; + free(s->vendor); +@@ -44,7 +44,7 @@ void free_cdi_cache_spec(struct cdi_cache_spec *s) + s->path = NULL; + map_free(s->devices); + s->devices = NULL; +- ++ + free(s); + } + +@@ -58,7 +58,7 @@ struct cdi_cache_spec *cdi_spec_read_spec(const char *path, int priority) + ERROR("Failed to get clean path %s", path); + return NULL; + } +- ++ + raw_spec = cdi_spec_parse_file(cleanpath, NULL, &err); + if (raw_spec == NULL) { + ERROR("Failed to read CDI Spec %s: %s", cleanpath, err); +@@ -106,7 +106,7 @@ struct cdi_cache_spec *cdi_spec_new_spec(cdi_spec *raw, const char *path, int pr + ERROR("Invalid CDI Spec"); + goto error_out; + } +- ++ + return spec; + + error_out: +@@ -183,7 +183,7 @@ static int cdi_spec_init(struct cdi_cache_spec *s) + cdi_device *d = NULL; + size_t i; + bool version_result = true; +- ++ + if (!cdi_is_valid_version(s->raw_spec->cdi_version)) { + ERROR("Failed to validate cdi spec version: %s", s->raw_spec->cdi_version); + return -1; +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec.h b/src/daemon/modules/device/cdi/behavior/cdi_spec.h +index ca7b2cfa..d20826c6 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_spec.h ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec.h +@@ -37,7 +37,7 @@ struct cdi_cache_spec { + }; + + #define CDI_DEFAULT_SPEC_EXT ".json" +- ++ + void free_cdi_cache_spec(struct cdi_cache_spec *s); + + struct cdi_cache_spec *cdi_spec_read_spec(const char *path, int priority); +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c +index cafb52b8..75cd7abf 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c +@@ -27,13 +27,13 @@ + + #define DEFAULT_SPEC_DIRS_LEN 2 + static char *default_spec_dirs_items[DEFAULT_SPEC_DIRS_LEN] = {CDI_DEFAULT_STATIC_DIR, CDI_DEFAULT_DYNAMIC_DIR}; +- ++ + string_array g_default_spec_dirs = { + .items = default_spec_dirs_items, + .len = DEFAULT_SPEC_DIRS_LEN, + .cap = DEFAULT_SPEC_DIRS_LEN, + }; +- ++ + struct scan_spec_dir_cb_args { + struct cdi_scan_fn_maps *scan_fn_maps; + cdi_scan_spec_func scan_fn; +@@ -64,7 +64,7 @@ static bool scan_spec_dir_cb(const char *dir, const struct dirent *pdirent, void + DEBUG("Skip dir %s", file_path); + return true; + } +- ++ + if (!util_has_suffix(file_path, ".json")) { + DEBUG("Skip file %s", file_path); + return true; +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h +index b17a0cd8..fdb0f2f9 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h +@@ -27,7 +27,7 @@ extern "C" { + #define CDI_DEFAULT_DYNAMIC_DIR "/var/run/cdi" + + extern string_array g_default_spec_dirs; +- ++ + struct cdi_scan_fn_maps { + map_t *specs; + map_t *devices; +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_version.c b/src/daemon/modules/device/cdi/behavior/cdi_version.c +index 550f3107..4cdc116a 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_version.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_version.c +@@ -177,7 +177,7 @@ const char *cdi_minimum_required_version(cdi_spec *spec) + bool cdi_is_valid_version(const char *spec_version) + { + int i; +- ++ + for (i = 0; i < VALID_SPEC_VERSIONS_LEN; i++) { + if (strcmp(g_valid_spec_versions[i].version, spec_version) == 0) { + return true; +diff --git a/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c +index 8824d29c..603ce28d 100644 +--- a/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c ++++ b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c +@@ -47,7 +47,7 @@ char *cdi_parser_qualified_name(const char *vendor, const char *class, const cha + } + + nret = snprintf(device_name, sizeof(device_name), "%s/%s=%s", +- vendor, class, name); ++ vendor, class, name); + if (nret < 0 || (size_t)nret >= sizeof(device_name)) { + ERROR("Device name is too long"); + return NULL; +@@ -118,7 +118,7 @@ int cdi_parser_parse_device(const char *device, char **vendor, char **class, cha + { + __isula_auto_array_t char **parts = NULL; + +- if (vendor == NULL || class == NULL || name == NULL || ++ if (vendor == NULL || class == NULL || name == NULL || + device == NULL || device[0] == '/') { + ERROR("Invalid argument"); + return -1; +@@ -214,7 +214,7 @@ int cdi_parser_validate_class_name(const char *class) + int cdi_parser_validate_device_name(const char *name) + { + size_t i; +- ++ + if (name == NULL) { + ERROR("Invalid (empty) device name"); + return -1; +diff --git a/src/daemon/modules/device/cdi/cdi_cache.c b/src/daemon/modules/device/cdi/cdi_cache.c +index e637f7cd..e9a9b804 100644 +--- a/src/daemon/modules/device/cdi/cdi_cache.c ++++ b/src/daemon/modules/device/cdi/cdi_cache.c +@@ -179,7 +179,7 @@ static int cdi_refresh(struct cdi_cache *c) + { + bool refreshed; + int ret = 0; +- ++ + if (c == NULL) { + ERROR("Invalid arguments"); + return -1; +@@ -206,10 +206,10 @@ static void map_cdi_cache_specs_kvfree(void *key, void *value) + static void map_cdi_cache_device_kvfree(void *key, void *value) + { + free(key); +- /* ++ /* + * map_cdi_cache_device_kvfree should not be recursively free cdi_cache_device. + * Otherwise, the function conflicts with the cdi_cache_specs free devices, +- * triggering double free. ++ * triggering double free. + */ + (void)value; + } +@@ -249,8 +249,8 @@ static bool resolve_conflict(struct cdi_scan_fn_maps *scan_fn_maps, const char * + return true; + } + +-static void refresh_scan_spec_func(struct cdi_scan_fn_maps *scan_fn_maps, const char *path, +- int priority, struct cdi_cache_spec *spec) ++static void refresh_scan_spec_func(struct cdi_scan_fn_maps *scan_fn_maps, const char *path, ++ int priority, struct cdi_cache_spec *spec) + { + map_t *specs = scan_fn_maps->specs; + map_t *devices = scan_fn_maps->devices; +@@ -375,7 +375,7 @@ static int refresh(struct cdi_cache *c) + util_swap_ptr((void **)&c->devices, (void **)&devices); + + ret = c->refresh_error_flag ? -1 : 0; +- ++ + free_out: + map_itor_free(itor); + map_free(specs); +@@ -443,7 +443,7 @@ static int cdi_inject_devices(struct cdi_cache *c, oci_runtime_spec *oci_spec, s + + (void)refresh_if_required(c, false, &ret); + +- for(i = 0; i < devices->len; i++) { ++ for (i = 0; i < devices->len; i++) { + device = devices->items[i]; + d = map_search(c->devices, (void *)device); + if (d == NULL) { +@@ -524,7 +524,7 @@ static int init_tracked(struct cdi_watch *w, string_array *dirs) + ERROR("Out of memory"); + return -1; + } +- for(i = 0; i < dirs->len; i++) { ++ for (i = 0; i < dirs->len; i++) { + if (!map_replace(w->tracked, (void *)dirs->items[i], (void *)&tmp_value)) { + ERROR("Failed to insert tracked by dir %s", dirs->items[i]); + goto error_out; +@@ -624,7 +624,7 @@ static int process_cdi_events(int watcher_fd, struct cdi_cache *c) + } + + (void)pthread_mutex_lock(&c->mutex); +- ++ + while (events_index < events_length) { + cdi_event = (struct inotify_event *)(&buffer[events_index]); + ssize_t event_size = (ssize_t)(cdi_event->len) + (ssize_t)offsetof(struct inotify_event, name); +@@ -633,8 +633,8 @@ static int process_cdi_events(int watcher_fd, struct cdi_cache *c) + } + events_index += event_size; + +- /* +- * file: ++ /* ++ * file: + * Rename: mask == IN_MOVED_TO | IN_MOVED_FROM + * Remove: mask == IN_MOVED_FROM || mask == IN_DELETE + * Write: mask == IN_MODIFY +diff --git a/src/daemon/modules/device/cdi/cdi_cache.h b/src/daemon/modules/device/cdi/cdi_cache.h +index 638e954e..9f6793c8 100644 +--- a/src/daemon/modules/device/cdi/cdi_cache.h ++++ b/src/daemon/modules/device/cdi/cdi_cache.h +@@ -34,7 +34,7 @@ struct cdi_cache_ops { + // injecting CDI devices into an OCI Spec. + // Resolver + int (*inject_devices)(struct cdi_cache *c, oci_runtime_spec *spec, string_array *devices); +- ++ + // refreshing the cache of CDI Specs and devices. + // Refresher + int (*configure)(struct cdi_cache *c, string_array *spec_dirs); +@@ -55,12 +55,12 @@ struct cdi_cache { + // This map holding the reference to cdi device, the devices will not released when the map is freed. + map_t *devices; // MAP_STR_PTR devices[cdi_device.name] = cdi_cache_device* + bool refresh_error_flag; +- bool auto_refresh; ++ bool auto_refresh; + struct cdi_watch *watch; + }; + + void free_cdi_cache(struct cdi_cache *c); +- ++ + struct cdi_cache *cdi_new_cache(string_array *spec_dirs); + struct cdi_cache_ops *cdi_get_cache_ops(void); + +diff --git a/src/daemon/modules/device/cdi_operate.c b/src/daemon/modules/device/cdi_operate.c +index f99bb7e4..2cc2a17d 100644 +--- a/src/daemon/modules/device/cdi_operate.c ++++ b/src/daemon/modules/device/cdi_operate.c +@@ -29,7 +29,7 @@ int cdi_operate_registry_init(char **specs_dirs, size_t specs_dirs_len) + .len = specs_dirs_len, + .cap = specs_dirs_len, + }; +- ++ + return cdi_registry_init(&spec_dirs_array); + } + +@@ -40,7 +40,7 @@ int cdi_operate_refresh(void) + ERROR("Failed to get registry"); + return -1; + } +- ++ + return registry->ops->refresh(registry->cdi_cache); + } + +@@ -52,13 +52,13 @@ int cdi_operate_inject_devices(oci_runtime_spec *spec, string_array *devices) + ERROR("Invalid params"); + return -1; + } +- ++ + registry = cdi_get_registry(); + if (registry == NULL || registry->ops == NULL || registry->ops->inject_devices == NULL) { + ERROR("Failed to get registry"); + return -1; + } +- ++ + return registry->ops->inject_devices(registry->cdi_cache, spec, devices); + } + +diff --git a/src/daemon/modules/image/oci/oci_image.c b/src/daemon/modules/image/oci/oci_image.c +index ce1c8a6b..e9f16024 100644 +--- a/src/daemon/modules/image/oci/oci_image.c ++++ b/src/daemon/modules/image/oci/oci_image.c +@@ -295,7 +295,8 @@ static bool remove_image_tmpdir_cb(const char *path_name, const struct dirent *s + return true; + } + +- if (!util_has_prefix(sub_dir->d_name, LOAD_TMPDIR_PREFIX) && !util_has_prefix(sub_dir->d_name, REGISTRY_TMPDIR_PREFIX)) { ++ if (!util_has_prefix(sub_dir->d_name, LOAD_TMPDIR_PREFIX) && ++ !util_has_prefix(sub_dir->d_name, REGISTRY_TMPDIR_PREFIX)) { + // only remove directory that image module created + return true; + } +diff --git a/src/daemon/modules/image/oci/progress.h b/src/daemon/modules/image/oci/progress.h +index dcc8e144..673019a4 100644 +--- a/src/daemon/modules/image/oci/progress.h ++++ b/src/daemon/modules/image/oci/progress.h +@@ -35,7 +35,7 @@ typedef struct progress { + + bool progress_status_map_udpate(progress_status_map *progress_status_map, char *key, int64_t current, int64_t total); + +-progress_status_map *progress_status_map_new(); ++progress_status_map *progress_status_map_new(void); + + size_t progress_status_map_size(progress_status_map *progress_status_map); + +diff --git a/src/daemon/modules/image/oci/registry/registry.c b/src/daemon/modules/image/oci/registry/registry.c +index 66fa0076..2e99255a 100644 +--- a/src/daemon/modules/image/oci/registry/registry.c ++++ b/src/daemon/modules/image/oci/registry/registry.c +@@ -293,7 +293,8 @@ static bool is_manifest_schemav1(char *media_type) + return false; + } + +- if (strcmp(media_type, DOCKER_MANIFEST_SCHEMA1_JSON) == 0 || strcmp(media_type, DOCKER_MANIFEST_SCHEMA1_PRETTYJWS) == 0 || ++ if (strcmp(media_type, DOCKER_MANIFEST_SCHEMA1_JSON) == 0 || ++ strcmp(media_type, DOCKER_MANIFEST_SCHEMA1_PRETTYJWS) == 0 || + strcmp(media_type, MEDIA_TYPE_APPLICATION_JSON) == 0) { + return true; + } +diff --git a/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.h b/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.h +index 48d8d8b6..10c14f05 100644 +--- a/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.h ++++ b/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.h +@@ -36,7 +36,7 @@ struct cni_args { + void free_cni_args(struct cni_args *cargs); + + int exec_plugin_with_result(const char *plugin_path, const char *cni_net_conf_json, const struct cni_args *cniargs, +- struct cni_opt_result **ret); ++ struct cni_opt_result **result); + + int exec_plugin_without_result(const char *plugin_path, const char *cni_net_conf_json, const struct cni_args *cniargs); + +diff --git a/src/daemon/modules/network/cri/adaptor_cri.c b/src/daemon/modules/network/cri/adaptor_cri.c +index 2d03dd98..55826d52 100644 +--- a/src/daemon/modules/network/cri/adaptor_cri.c ++++ b/src/daemon/modules/network/cri/adaptor_cri.c +@@ -71,7 +71,7 @@ int adaptor_cni_update_confs() + size_t tmp_net_list_len = 0; + size_t i; + char message[MAX_BUFFER_SIZE] = { 0 }; +- int pos = 0; ++ size_t pos = 0; + + work = map_new(MAP_STR_INT, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); + if (work == NULL) { +diff --git a/src/daemon/modules/network/cri/adaptor_cri.h b/src/daemon/modules/network/cri/adaptor_cri.h +index 3d4fe82d..bc157626 100644 +--- a/src/daemon/modules/network/cri/adaptor_cri.h ++++ b/src/daemon/modules/network/cri/adaptor_cri.h +@@ -23,9 +23,9 @@ extern "C" { + + int adaptor_cni_init_confs(const char *conf_dir, const char **bin_paths, const size_t bin_paths_len); + +-int adaptor_cni_update_confs(); ++int adaptor_cni_update_confs(void); + +-bool adaptor_cni_check_inited(); ++bool adaptor_cni_check_inited(void); + + int adaptor_cni_setup(const network_api_conf *conf, network_api_result_list *result); + +diff --git a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c +index a89d0375..64a8adbc 100644 +--- a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c ++++ b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c +@@ -166,7 +166,8 @@ int rt_lcr_start(const char *name, const char *runtime, const rt_start_params_t + } + isulad_set_error_message("Start container error: %s", + (tmpmsg != NULL && strcmp(tmpmsg, DEF_SUCCESS_STR) != 0) ? tmpmsg : DEF_ERR_RUNTIME_STR); +- ERROR("Start container error: %s", (tmpmsg != NULL && strcmp(tmpmsg, DEF_SUCCESS_STR) != 0) ? tmpmsg : DEF_ERR_RUNTIME_STR); ++ ERROR("Start container error: %s", (tmpmsg != NULL && ++ strcmp(tmpmsg, DEF_SUCCESS_STR) != 0) ? tmpmsg : DEF_ERR_RUNTIME_STR); + ret = -1; + goto out; + } +@@ -264,7 +265,8 @@ int rt_lcr_rm(const char *name, const char *runtime, const rt_rm_params_t *param + if (engine_ops == NULL || engine_ops->engine_delete_op == NULL) { + // if engine_ops is NULL, container root path may have been corrupted, try to remove by daemon + // If user runs container with lcr but remove lcr runtime after, there might be resources remaining +- ERROR("Failed to get engine delete operations, container %s root path may have been corrupted, try to remove by daemon", name); ++ ERROR("Failed to get engine delete operations, container %s root path may have been corrupted, try to remove by daemon", ++ name); + if (remove_container_rootpath(name, params->rootpath) == 0) { + ret = 0; + goto out; +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 854752ea..62cff3cf 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -623,7 +623,8 @@ out: + return ret; + } + +-static void transform_stats_info_from_runtime(shim_client_runtime_stats *stats, struct runtime_container_resources_stats_info *info) ++static void transform_stats_info_from_runtime(shim_client_runtime_stats *stats, ++ struct runtime_container_resources_stats_info *info) + { + size_t i; + if (stats == NULL || stats->data == NULL) { +@@ -887,7 +888,7 @@ static int shim_create(shim_create_args *args) + + if (get_engine_routine_log_info(&engine_log_path, &log_level) != 0) { + ERROR("failed to get engine log path"); +- return -1; ++ return -1; + } + + nret = snprintf(fpid, sizeof(fpid), "%s/shim-pid", args->workdir); +@@ -938,7 +939,7 @@ static int shim_create(shim_create_args *args) + //prevent the child process from having the same standard streams as the parent process + if (isula_null_stdfds() != 0) { + (void)dprintf(exec_err_pipe[1], "failed to set std console to /dev/null"); +- exit(EXIT_FAILURE); ++ exit(EXIT_FAILURE); + } + + if (args->fg) { +@@ -1054,7 +1055,7 @@ out: + close(shim_stdout_pipe[0]); + if (ret != 0) { + show_shim_errlog(shim_stderr_pipe[0]); +- // Since users are more concerned about runtime error information, ++ // Since users are more concerned about runtime error information, + // the runtime log will overwrite the shim log if it exists. + show_runtime_errlog(args->workdir); + if (args->timeout != NULL) { +diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c +index fe8ee72f..0b95cdad 100644 +--- a/src/daemon/modules/service/service_container.c ++++ b/src/daemon/modules/service/service_container.c +@@ -691,7 +691,8 @@ out: + epoll_loop_close(&descr); + } + +-static int do_oci_spec_update(const char *id, oci_runtime_spec *oci_spec, container_config *container_spec, host_config *hostconfig) ++static int do_oci_spec_update(const char *id, oci_runtime_spec *oci_spec, container_config *container_spec, ++ host_config *hostconfig) + { + int ret; + +@@ -2013,7 +2014,7 @@ static defs_process *make_exec_process_spec(const container_config *container_sp + #ifdef ENABLE_CDI + // extend step: merge env from oci_spec which comes from injected devices + ret = defs_process_add_multiple_env(spec, (const char **)oci_spec->process->env, +- oci_spec->process->env_len); ++ oci_spec->process->env_len); + if (ret != 0) { + ERROR("Failed to dup oci env for exec process spec"); + goto err_out; +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index e779c22e..122f9992 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -2483,7 +2483,8 @@ out: + return ret; + } + +-int update_oci_ulimit(oci_runtime_spec *oci_spec, const host_config *hostconfig) { ++int update_oci_ulimit(oci_runtime_spec *oci_spec, const host_config *hostconfig) ++{ + if (oci_spec == NULL || hostconfig == NULL) { + ERROR("Invalid arguments"); + return -1; +@@ -2660,7 +2661,7 @@ int defs_process_add_multiple_env(defs_process *dp, const char **envs, size_t en + int spec_add_multiple_process_env(oci_runtime_spec *oci_spec, const char **envs, size_t env_len) + { + int ret = 0; +- ++ + if (envs == NULL || env_len == 0) { + DEBUG("empty envs"); + return 0; +@@ -2669,26 +2670,26 @@ int spec_add_multiple_process_env(oci_runtime_spec *oci_spec, const char **envs, + ERROR("Invalid params"); + return -1; + } +- ++ + ret = make_sure_oci_spec_process(oci_spec); + if (ret < 0) { + ERROR("Out of memory"); + return -1; + } +- ++ + ret = defs_process_add_multiple_env(oci_spec->process, envs, env_len); + if (ret < 0) { + ERROR("Failed to add envs"); + } +- ++ + return ret; + } +- ++ + int spec_add_device(oci_runtime_spec *oci_spec, defs_device *device) + { + int ret = 0; + size_t i; +- ++ + if (device == NULL) { + return -1; + } +@@ -2696,7 +2697,7 @@ int spec_add_device(oci_runtime_spec *oci_spec, defs_device *device) + if (ret < 0) { + return -1; + } +- ++ + for (i = 0; i < oci_spec->linux->devices_len; i++) { + if (strcmp(oci_spec->linux->devices[i]->path, device->path) == 0) { + free_defs_device(oci_spec->linux->devices[i]); +@@ -2712,21 +2713,21 @@ int spec_add_device(oci_runtime_spec *oci_spec, defs_device *device) + } + oci_spec->linux->devices[oci_spec->linux->devices_len] = device; + oci_spec->linux->devices_len++; +- ++ + return 0; + } +- ++ + int spec_add_linux_resources_device(oci_runtime_spec *oci_spec, bool allow, const char *dev_type, + int64_t major, int64_t minor, const char *access) + { + int ret = 0; + defs_device_cgroup *device = NULL; +- ++ + ret = make_sure_oci_spec_linux_resources(oci_spec); + if (ret < 0) { + return -1; + } +- ++ + device = util_common_calloc_s(sizeof(*device)); + if (device == NULL) { + ERROR("Out of memory"); +@@ -2738,7 +2739,8 @@ int spec_add_linux_resources_device(oci_runtime_spec *oci_spec, bool allow, cons + device->major = major; + device->minor = minor; + +- if (util_mem_realloc((void **)&oci_spec->linux->resources->devices, (oci_spec->linux->resources->devices_len + 1) * sizeof(char *), ++ if (util_mem_realloc((void **)&oci_spec->linux->resources->devices, ++ (oci_spec->linux->resources->devices_len + 1) * sizeof(char *), + (void *)oci_spec->linux->resources->devices, oci_spec->linux->resources->devices_len * sizeof(char *)) != 0) { + ERROR("Out of memory"); + free_defs_device_cgroup(device); +@@ -2746,35 +2748,35 @@ int spec_add_linux_resources_device(oci_runtime_spec *oci_spec, bool allow, cons + } + oci_spec->linux->resources->devices[oci_spec->linux->resources->devices_len] = device; + oci_spec->linux->resources->devices_len++; +- ++ + return 0; + } +- ++ + void spec_remove_mount(oci_runtime_spec *oci_spec, const char *dest) + { + size_t i; +- ++ + if (oci_spec == NULL || oci_spec->mounts == NULL || dest == NULL) { + return; + } +- ++ + for (i = 0; i < oci_spec->mounts_len; i++) { + if (strcmp(oci_spec->mounts[i]->destination, dest) == 0) { + free_defs_mount(oci_spec->mounts[i]); + (void)memcpy((void **)&oci_spec->mounts[i], (void **)&oci_spec->mounts[i + 1], +- (oci_spec->mounts_len - i - 1) * sizeof(void *)); ++ (oci_spec->mounts_len - i - 1) * sizeof(void *)); + oci_spec->mounts_len--; + return; + } + } + } +- ++ + int spec_add_mount(oci_runtime_spec *oci_spec, defs_mount *mnt) + { + if (oci_spec == NULL || mnt == NULL) { + return -1; + } +- ++ + if (util_mem_realloc((void **)&oci_spec->mounts, (oci_spec->mounts_len + 1) * sizeof(char *), + (void *)oci_spec->mounts, oci_spec->mounts_len * sizeof(char *)) != 0) { + ERROR("Out of memory"); +@@ -2782,10 +2784,10 @@ int spec_add_mount(oci_runtime_spec *oci_spec, defs_mount *mnt) + } + oci_spec->mounts[oci_spec->mounts_len] = mnt; + oci_spec->mounts_len++; +- ++ + return 0; + } +- ++ + #define SPEC_ADD_HOOKS_ITEM_DEF(hooktype) \ + int spec_add_##hooktype##_hook(oci_runtime_spec *oci_spec, defs_hook *hooktype##_hook) \ + { \ +@@ -2806,9 +2808,9 @@ int spec_add_mount(oci_runtime_spec *oci_spec, defs_mount *mnt) + oci_spec->hooks->hooktype##_len++; \ + return 0; \ + } +- +-/* +-* The OCI being used by the iSulad not supportes ++ ++/* ++* The OCI being used by the iSulad not supportes + * createRuntime/createContainer/startContainer currently. + */ + SPEC_ADD_HOOKS_ITEM_DEF(prestart) +diff --git a/src/daemon/modules/spec/specs_mount.c b/src/daemon/modules/spec/specs_mount.c +index 12bd261b..2e065e3d 100644 +--- a/src/daemon/modules/spec/specs_mount.c ++++ b/src/daemon/modules/spec/specs_mount.c +@@ -2871,7 +2871,7 @@ static inline int set_host_ipc_shm_path(container_config_v2_common_config *v2_sp + * 1. The user defined /dev/shm in mounts, which takes the first priority + * 2. If sharable is set in ipc mode (or by default ipc_mode is null), the container provides shm path, + * in the case of sandbox API is used, the sandbox module has already provided shm path +- * 3. Use the connected container's shm path if ipc_mode is set to container:<cid>, ++ * 3. Use the connected container's shm path if ipc_mode is set to container:<cid>, + * if connected containerd is a sandbox, use the sandbox's shm path + * 4. Use /dev/shm if ipc_mode is set to host + */ +@@ -3613,7 +3613,7 @@ int inject_CDI_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hos + int ret = 0; + string_array devices_array = { 0 }; + __isula_auto_free char *error = NULL; +- ++ + if (oci_spec == NULL || hostconfig == NULL) { + ERROR("Invalid params"); + return -1; +diff --git a/src/utils/cutils/blocking_queue.c b/src/utils/cutils/blocking_queue.c +index 9bdb2ca3..02059690 100644 +--- a/src/utils/cutils/blocking_queue.c ++++ b/src/utils/cutils/blocking_queue.c +@@ -97,7 +97,8 @@ int blocking_queue_push(blocking_queue *queue, void *data) + return 0; + } + +-int blocking_queue_pop(blocking_queue *queue, void **data) { ++int blocking_queue_pop(blocking_queue *queue, void **data) ++{ + if (queue == NULL || data == NULL) { + ERROR("Invalid NULL arguments"); + return -1; +diff --git a/src/utils/cutils/network_namespace.h b/src/utils/cutils/network_namespace.h +index 6ac7b28b..14410736 100644 +--- a/src/utils/cutils/network_namespace.h ++++ b/src/utils/cutils/network_namespace.h +@@ -22,9 +22,9 @@ + extern "C" { + #endif + +-int prepare_network_namespace(const char *netns_path, const bool post_prepare_network, const int pid); ++int prepare_network_namespace(const char *netns_path, const bool post_setup_network, const int pid); + +-int remove_network_namespace(const char *netns); ++int remove_network_namespace(const char *netns_path); + + int create_network_namespace_file(const char *netns_path); + +diff --git a/src/utils/cutils/utils_array.c b/src/utils/cutils/utils_array.c +index 72294005..6c7444f0 100644 +--- a/src/utils/cutils/utils_array.c ++++ b/src/utils/cutils/utils_array.c +@@ -90,17 +90,17 @@ char **util_copy_array_by_len(char **array, size_t len) + { + char **new_array = NULL; + size_t i; +- ++ + if (array == NULL || len == 0) { + return NULL; + } +- ++ + new_array = util_smart_calloc_s(sizeof(char *), len); + if (new_array == NULL) { + ERROR("Out of memory"); + return NULL; + } +- ++ + for (i = 0; i < len; i++) { + new_array[i] = util_strdup_s(array[i]); + } +@@ -262,12 +262,12 @@ string_array *util_copy_string_array(string_array *sarr) + { + string_array *ptr = NULL; + size_t i; +- ++ + if (sarr == NULL) { + ERROR("Invalid string array"); + return NULL; + } +- ++ + ptr = util_string_array_new(sarr->cap); + if (ptr == NULL) { + ERROR("Out of memory"); +@@ -277,7 +277,7 @@ string_array *util_copy_string_array(string_array *sarr) + ptr->items[i] = util_strdup_s(sarr->items[i]); + ptr->len += 1; + } +- ++ + return ptr; + } + +diff --git a/src/utils/cutils/utils_port.h b/src/utils/cutils/utils_port.h +index dbbf2a5a..ae1676f2 100644 +--- a/src/utils/cutils/utils_port.h ++++ b/src/utils/cutils/utils_port.h +@@ -66,7 +66,7 @@ void util_free_network_port(struct network_port *ptr); + + bool util_valid_proto(const char *proto); + +-int util_get_random_port(); ++int util_get_random_port(void); + + static inline bool is_valid_port(const int port) + { +diff --git a/src/utils/cutils/utils_verify.c b/src/utils/cutils/utils_verify.c +index 6f1da12c..0d7c17f4 100644 +--- a/src/utils/cutils/utils_verify.c ++++ b/src/utils/cutils/utils_verify.c +@@ -540,8 +540,8 @@ bool util_valid_propagation_mode(const char *mode) + if (mode == NULL) { + return false; + } +- return strcmp(mode, "private") == 0 || strcmp(mode, "rprivate") == 0 || strcmp(mode, "slave") == 0 || strcmp(mode, "rslave") == 0 || +- strcmp(mode, "shared") == 0 || strcmp(mode, "rshared") == 0; ++ return strcmp(mode, "private") == 0 || strcmp(mode, "rprivate") == 0 || strcmp(mode, "slave") == 0 || ++ strcmp(mode, "rslave") == 0 || strcmp(mode, "shared") == 0 || strcmp(mode, "rshared") == 0; + } + + bool util_valid_mount_mode(const char *mode) +diff --git a/src/utils/progress/show.h b/src/utils/progress/show.h +index c1f71d86..1942db8e 100644 +--- a/src/utils/progress/show.h ++++ b/src/utils/progress/show.h +@@ -21,11 +21,11 @@ extern "C" { + #endif + + void move_to_row(int row); +-void move_cursor_up(int lines); ++void move_cursor_up(int rows); + void clear_row(int row); +-void clear_lines_below(); +-int get_current_row(); +-int get_terminal_width(); ++void clear_lines_below(void); ++int get_current_row(void); ++int get_terminal_width(void); + + #ifdef __cplusplus + } +diff --git a/src/utils/tar/isulad_tar.c b/src/utils/tar/isulad_tar.c +index 13343922..29008420 100644 +--- a/src/utils/tar/isulad_tar.c ++++ b/src/utils/tar/isulad_tar.c +@@ -445,7 +445,7 @@ static int tar_resource_rebase(const char *path, const char *rebase, const char + ERROR("Can not split path: %s", path); + goto cleanup; + } +- ++ + if (realpath(srcdir, cleanpath) == NULL) { + ERROR("Failed to get real path for %s", srcdir); + goto cleanup; +diff --git a/test/cgroup/cpu/cgroup_cpu_ut.cc b/test/cgroup/cpu/cgroup_cpu_ut.cc +index 6e6e04f4..16eec69b 100644 +--- a/test/cgroup/cpu/cgroup_cpu_ut.cc ++++ b/test/cgroup/cpu/cgroup_cpu_ut.cc +@@ -80,7 +80,7 @@ TEST(CgroupCpuUnitTest, test_sysinfo_cgroup_controller_cpurt_mnt_path) + { + MOCK_SET(util_common_calloc_s, nullptr); + ASSERT_EQ(get_sys_info(true), nullptr); +- ++ + int ret = cgroup_ops_init(); + ASSERT_EQ(ret, 0); + +-- +2.25.1 + diff --git a/0088-testcase-close-cdi-testcase.patch b/0088-testcase-close-cdi-testcase.patch new file mode 100644 index 0000000..702342f --- /dev/null +++ b/0088-testcase-close-cdi-testcase.patch @@ -0,0 +1,29 @@ +From eba353bef72bf62cd47f1b03a9fbd4c621ad479e Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Thu, 16 May 2024 18:00:01 +0800 +Subject: [PATCH 088/108] testcase:close cdi testcase + +Signed-off-by: liuxu <liuxu156@huawei.com> +--- + CI/test_cases/container_cases/cdi_test.sh | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/CI/test_cases/container_cases/cdi_test.sh b/CI/test_cases/container_cases/cdi_test.sh +index dd7b1177..f9fd4567 100755 +--- a/CI/test_cases/container_cases/cdi_test.sh ++++ b/CI/test_cases/container_cases/cdi_test.sh +@@ -198,8 +198,8 @@ EOF + + declare -i ans=0 + +-do_pre || ((ans++)) +-do_test_full_cdi || ((ans++)) +-do_post ++# do_pre || ((ans++)) ++# do_test_full_cdi || ((ans++)) ++# do_post + + show_result ${ans} "${curr_path}/${0}" +-- +2.25.1 + diff --git a/0089-docs-update-cni-doc.patch b/0089-docs-update-cni-doc.patch new file mode 100644 index 0000000..32d1fad --- /dev/null +++ b/0089-docs-update-cni-doc.patch @@ -0,0 +1,68 @@ +From 7fc8578097b9f8254962dc4fb277492b3251e5cb Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Thu, 16 May 2024 17:56:08 +0800 +Subject: [PATCH 089/108] docs:update cni doc + +Signed-off-by: liuxu <liuxu156@huawei.com> +--- + README.md | 2 +- + README_zh.md | 2 +- + docs/design/README.md | 2 ++ + docs/design/README_zh.md | 2 ++ + 4 files changed, 6 insertions(+), 2 deletions(-) + +diff --git a/README.md b/README.md +index 694ddbc2..7ddc62ab 100644 +--- a/README.md ++++ b/README.md +@@ -228,7 +228,7 @@ Using [ptcr](https://gitee.com/openeuler/ptcr) as a performance test tool , it s + The standard specification versions that `iSulad` is compatible with are as follows: + + - Compatible with OCI 1.0.0. +-- Compatible with CNI 0.3.0 and above. ++- Compatible with CNI 0.3.0 - 1.0.0, iSulad supports CNI 1.0.0 from 2.1.4 version. + - Compatible with lcr 2.1.x and above. + + ## Kubernetes Support +diff --git a/README_zh.md b/README_zh.md +index 5db28f3a..45ac66ac 100755 +--- a/README_zh.md ++++ b/README_zh.md +@@ -224,7 +224,7 @@ $ sudo isula rm test + `iSulad` 能够兼容的标准规范版本如下： + + - 兼容 1.0.0 版本的OCI +-- 兼容 0.3.0 版本以上的CNI ++- 兼容 0.3.0-1.0.0 版本的CNI，iSulad从2.1.4版本后支持 CNI 1.0.0版本 + - 兼容 2.1.x 版本以上的lcr + + ## Kubernetes Support +diff --git a/docs/design/README.md b/docs/design/README.md +index d2a3702d..c171cb20 100644 +--- a/docs/design/README.md ++++ b/docs/design/README.md +@@ -43,6 +43,8 @@ This section contains some design documents for users who want to learn more abo + + - You can see how the cni operator modules are designed in [cni_operator_design](./detailed/Network/cni_operator_design.md). + ++- You can see how the cni operator modules update to CNI v1.0.0 in [cni_1.0.0_change](./detailed/Network/cni_1.0.0_change.md)。 ++ + - You can see how the CRI adapter modules are designed in [CRI_adapter_design](./detailed/Network/CRI_adapter_design.md). + + - You can see how the native network adapter modules are designed in [native_network_adapter_design](./detailed/Network/native_network_adapter_design.md). +diff --git a/docs/design/README_zh.md b/docs/design/README_zh.md +index c6172b6f..0f4cf13e 100644 +--- a/docs/design/README_zh.md ++++ b/docs/design/README_zh.md +@@ -49,6 +49,8 @@ + + - 查看 cni operator 模块的设计文档： [cni_operator_design](./detailed/Network/cni_operator_design_zh.md) 。 + ++- 查看 cni operator 模块升级到CNI v1.0.0的设计文档： [cni_1.0.0_change](./detailed/Network/cni_1.0.0_change.md) 。 ++ + - 查看 CRI adapter 模块的设计文档： [CRI_adapter_design](./detailed/Network/CRI_adapter_design_zh.md) 。 + + - 查看 native network adapter 模块的设计文档： [native_network_adapter_design](./detailed/Network/native_network_adapter_design_zh.md) 。 +-- +2.25.1 + diff --git a/0090-modify-the-user-error-log-to-be-the-same-as-before.patch b/0090-modify-the-user-error-log-to-be-the-same-as-before.patch new file mode 100644 index 0000000..582f7e4 --- /dev/null +++ b/0090-modify-the-user-error-log-to-be-the-same-as-before.patch @@ -0,0 +1,100 @@ +From 3b0f34c7cd55686cf18f65efbdc0be8a84f13e3e Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Mon, 20 May 2024 17:54:04 +1400 +Subject: [PATCH 090/108] modify the user error log to be the same as before + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/common/id_name_manager.c | 4 +-- + .../executor/container_cb/execution_create.c | 26 +++++++++++-------- + .../container_cb/execution_information.c | 4 ++- + 3 files changed, 20 insertions(+), 14 deletions(-) + +diff --git a/src/daemon/common/id_name_manager.c b/src/daemon/common/id_name_manager.c +index 263a584d..f64094b9 100644 +--- a/src/daemon/common/id_name_manager.c ++++ b/src/daemon/common/id_name_manager.c +@@ -242,7 +242,7 @@ static bool try_add_name(const char *name) + } + + if (!util_valid_container_name(name)) { +- ERROR("Failed to add invalid name: %s", name); ++ ERROR("Invalid container name (%s), only [a-zA-Z0-9][a-zA-Z0-9_.-]+$ are allowed.", name); + return false; + } + +@@ -262,7 +262,7 @@ static bool try_remove_name(const char *name) + } + + if (!util_valid_container_name(name)) { +- ERROR("Failed to remove invalid name: %s", name); ++ ERROR("Invalid container name (%s), only [a-zA-Z0-9][a-zA-Z0-9_.-]+$ are allowed.", name); + return false; + } + +diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c +index 785b4e27..041089dd 100644 +--- a/src/daemon/executor/container_cb/execution_create.c ++++ b/src/daemon/executor/container_cb/execution_create.c +@@ -761,8 +761,17 @@ static int maintain_container_id(const container_create_request *request, char * + #endif + + if (!nret) { +- ERROR("Failed to add entry to id name manager with new id and name"); +- isulad_set_error_message("Failed to add entry to id name manager with new id and name"); ++ __isula_auto_free char *used_id = NULL; ++ used_id = container_name_index_get(name); ++ if(used_id != NULL) { ++ ERROR("Name %s is in use by container %s", name, used_id); ++ isulad_set_error_message("Conflict. The name \"%s\" is already in use by container %s. " ++ "You have to remove (or rename) that container to be able to reuse that name.", ++ name, used_id); ++ } else { ++ ERROR("Failed to add entry to id name manager with new id and name"); ++ isulad_set_error_message("Failed to add entry to id name manager with new id and name"); ++ } + ret = -1; + goto out; + } +@@ -775,19 +784,14 @@ static int maintain_container_id(const container_create_request *request, char * + goto out; + } + +- char *used_id = NULL; +- used_id = container_name_index_get(name); +- ERROR("Name %s is in use by container %s", name, used_id); +- isulad_set_error_message("Conflict. The name \"%s\" is already in use by container %s. " +- "You have to remove (or rename) that container to be able to reuse that name.", +- name, used_id); +- free(used_id); +- used_id = NULL; +- ret = -1; + if (!skip_id_name_manage && !id_name_manager_remove_entry(id, name)) { + WARN("Failed to remove %s and %s from id name manager", id, name); + } + ++ ERROR("Failed to add %s to container name index", name); ++ isulad_set_error_message("Failed to add %s to container name index", name); ++ ret = -1; ++ + out: + *out_id = id; + *out_name = name; +diff --git a/src/daemon/executor/container_cb/execution_information.c b/src/daemon/executor/container_cb/execution_information.c +index c02cc830..58924257 100644 +--- a/src/daemon/executor/container_cb/execution_information.c ++++ b/src/daemon/executor/container_cb/execution_information.c +@@ -1149,7 +1149,9 @@ static int container_rename(container_t *cont, const char *new_name) + + if (!id_name_manager_rename(new_name, old_name)) { + ERROR("Failed to rename %s to %s in id-name manager", old_name, new_name); +- isulad_set_error_message("Failed to rename %s to %s in id-name manager", old_name, new_name); ++ isulad_set_error_message("Conflict. The name \"%s\" is already in use by container %s. " ++ "You have to remove (or rename) that container to be able to reuse that name.", ++ new_name, new_name); + ret = -1; + goto out; + } +-- +2.25.1 + diff --git a/0091-add-enable-cri-v1-in-k8s-integration.patch b/0091-add-enable-cri-v1-in-k8s-integration.patch new file mode 100644 index 0000000..8c36478 --- /dev/null +++ b/0091-add-enable-cri-v1-in-k8s-integration.patch @@ -0,0 +1,54 @@ +From 1f69ffe589f7225a1db83377e276ddbab963bd16 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Tue, 21 May 2024 01:13:08 +0000 +Subject: [PATCH 091/108] add enable cri v1 in k8s integration + +Signed-off-by: jikai <jikai11@huawei.com> +--- + docs/manual/k8s_integration.md | 9 +++++++++ + docs/manual/k8s_integration_zh.md | 9 +++++++++ + 2 files changed, 18 insertions(+) + +diff --git a/docs/manual/k8s_integration.md b/docs/manual/k8s_integration.md +index 8fcd0a54..14de0ef4 100644 +--- a/docs/manual/k8s_integration.md ++++ b/docs/manual/k8s_integration.md +@@ -20,6 +20,15 @@ + + if `hosts` is not configured, the default endpoint is `unix:///var/run/isulad.sock`. + ++ `iSulad` supports both `CRI V1alpha2` and `CRI V1`, and uses `CRI V1alph2` by default. ++ If `CRI V1` is required, it can be configured in `/etc/isulad/daemon.json` to enable `CRI V1`: ++ ++ ```json ++ "enable-cri-v1": true, ++ ``` ++ ++ If `iSulad` is compiled from source codes, `-D ENABLE_CRI_API_V1=ON` option is required in cmake. ++ + 2. Restart `isulad`: + + ```bash +diff --git a/docs/manual/k8s_integration_zh.md b/docs/manual/k8s_integration_zh.md +index 6dda1e4d..26ba6cc4 100644 +--- a/docs/manual/k8s_integration_zh.md ++++ b/docs/manual/k8s_integration_zh.md +@@ -20,6 +20,15 @@ + + 如果`hosts`没有配置，默认的`endpoint`为``unix:///var/run/isulad.sock`` + ++ `iSulad`同时支持`CRI V1alpha2`和`CRI V1`两种`CRI`接口，默认使用`CRI V1alph2`，若需使用`CRI V1`， ++ 需要在`/etc/isulad/daemon.json`对`iSulad`进行相关配置，配置方式为： ++ ++ ```json ++ "enable-cri-v1": true, ++ ``` ++ ++ 若使用源码编译`iSulad`，还需在编译时增加cmake编译选项`-D ENABLE_CRI_API_V1=ON`。 ++ + 2. 重启`isulad` + + ```bash +-- +2.25.1 + diff --git a/0092-isolate-oom-monitor-codes.patch b/0092-isolate-oom-monitor-codes.patch new file mode 100644 index 0000000..d7d61de --- /dev/null +++ b/0092-isolate-oom-monitor-codes.patch @@ -0,0 +1,317 @@ +From d97656a8b99f4fa95a9c15abfbac777a94b84d55 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Mon, 20 May 2024 08:48:00 +0000 +Subject: [PATCH 092/108] isolate oom monitor codes + +Signed-off-by: jikai <jikai11@huawei.com> +--- + cmake/options.cmake | 7 +++++++ + src/daemon/common/cgroup/cgroup.c | 2 ++ + src/daemon/common/cgroup/cgroup.h | 2 ++ + src/daemon/common/cgroup/cgroup_common.h | 4 ++++ + src/daemon/common/cgroup/cgroup_v1.c | 8 ++++++++ + src/daemon/common/cgroup/cgroup_v2.c | 8 ++++++++ + src/daemon/common/cri/v1/v1_cri_helpers.cc | 2 ++ + src/daemon/modules/container/container_state.c | 2 ++ + .../modules/container/supervisor/supervisor.c | 14 ++++++++++++++ + 9 files changed, 49 insertions(+) + +diff --git a/cmake/options.cmake b/cmake/options.cmake +index a15b8194..5b17f631 100644 +--- a/cmake/options.cmake ++++ b/cmake/options.cmake +@@ -58,6 +58,13 @@ if (ENABLE_SANDBOXER STREQUAL "ON") + message("${Green}-- Enable sandbox API${ColourReset}") + endif() + ++option(ENABLE_OOM_MONITOR "Enable oom monitor" ON) ++IF (ENABLE_OOM_MONITOR STREQUAL "ON") ++ add_definitions(-DENABLE_OOM_MONITOR) ++ set(ENABLE_OOM_MONITOR 1) ++ message("${Green}-- Enable oom monitor${ColourReset}") ++endif() ++ + option(ENABLE_SYSTEMD_NOTIFY "Enable systemd notify" ON) + if (ENABLE_SYSTEMD_NOTIFY STREQUAL "ON") + add_definitions(-DSYSTEMD_NOTIFY) +diff --git a/src/daemon/common/cgroup/cgroup.c b/src/daemon/common/cgroup/cgroup.c +index 71bf9801..77fafdae 100644 +--- a/src/daemon/common/cgroup/cgroup.c ++++ b/src/daemon/common/cgroup/cgroup.c +@@ -197,6 +197,7 @@ char *common_convert_cgroup_path(const char *cgroup_path) + return util_strdup_s(result); + } + ++#ifdef ENABLE_OOM_MONITOR + cgroup_oom_handler_info_t *common_get_cgroup_oom_handler(int fd, const char *name, const char *cgroup_path, + const char *exit_fifo) + { +@@ -225,3 +226,4 @@ void common_free_cgroup_oom_handler_info(cgroup_oom_handler_info_t *info) + free(info->cgroup_memory_event_path); + free(info); + } ++#endif +diff --git a/src/daemon/common/cgroup/cgroup.h b/src/daemon/common/cgroup/cgroup.h +index 0bbb70a0..1ebbfa98 100644 +--- a/src/daemon/common/cgroup/cgroup.h ++++ b/src/daemon/common/cgroup/cgroup.h +@@ -43,9 +43,11 @@ char *common_get_own_cgroup_path(const char *subsystem); + + char *common_convert_cgroup_path(const char *cgroup_path); + ++#ifdef ENABLE_OOM_MONITOR + cgroup_oom_handler_info_t *common_get_cgroup_oom_handler(int fd, const char *name, const char *cgroup_path, + const char *exit_fifo); + void common_free_cgroup_oom_handler_info(cgroup_oom_handler_info_t *info); ++#endif + + #ifdef __cplusplus + } +diff --git a/src/daemon/common/cgroup/cgroup_common.h b/src/daemon/common/cgroup/cgroup_common.h +index 46a7de50..01fc669c 100644 +--- a/src/daemon/common/cgroup/cgroup_common.h ++++ b/src/daemon/common/cgroup/cgroup_common.h +@@ -116,6 +116,7 @@ typedef struct { + cgroup_pids_metrics_t cgpids_metrics; + } cgroup_metrics_t; + ++#ifdef ENABLE_OOM_MONITOR + #define CGROUP_OOM_HANDLE_CONTINUE false + #define CGROUP_OOM_HANDLE_CLOSE true + +@@ -126,6 +127,7 @@ typedef struct _cgroup_oom_handler_info_t { + char *cgroup_memory_event_path; + bool (*oom_event_handler)(int, void *); + } cgroup_oom_handler_info_t; ++#endif + + typedef struct { + int (*get_cgroup_version)(void); +@@ -140,8 +142,10 @@ typedef struct { + char *(*get_init_cgroup_path)(const char *subsystem); + char *(*get_own_cgroup_path)(const char *subsystem); + ++#ifdef ENABLE_OOM_MONITOR + cgroup_oom_handler_info_t *(*get_cgroup_oom_handler)(int fd, const char *name, const char *cgroup_path, + const char *exit_fifo); ++#endif + } cgroup_ops; + + #ifdef __cplusplus +diff --git a/src/daemon/common/cgroup/cgroup_v1.c b/src/daemon/common/cgroup/cgroup_v1.c +index 45b1d096..018336ea 100644 +--- a/src/daemon/common/cgroup/cgroup_v1.c ++++ b/src/daemon/common/cgroup/cgroup_v1.c +@@ -20,12 +20,16 @@ + + #include <stdio.h> + #include <stdlib.h> ++#ifdef ENABLE_OOM_MONITOR + #include <sys/eventfd.h> ++#endif + + #include "utils.h" + #include "sysinfo.h" + #include "err_msg.h" ++#ifdef ENABLE_OOM_MONITOR + #include "events_sender_api.h" ++#endif + + #define CGROUP_HUGETLB_LIMIT "hugetlb.%s.limit_in_bytes" + #define CGROUP_MOUNT_PATH_PREFIX "/sys/fs/cgroup/" +@@ -1052,6 +1056,7 @@ static char *common_get_cgroup_path(const char *path, const char *subsystem) + return res; + } + ++#ifdef ENABLE_OOM_MONITOR + static bool oom_cb_cgroup_v1(int fd, void *cbdata) + { + cgroup_oom_handler_info_t *info = (cgroup_oom_handler_info_t *)cbdata; +@@ -1205,6 +1210,7 @@ cleanup: + common_free_cgroup_oom_handler_info(info); + return NULL; + } ++#endif + + char *get_init_cgroup_path_v1(const char *subsystem) + { +@@ -1232,6 +1238,8 @@ int cgroup_v1_ops_init(cgroup_ops *ops) + ops->get_cgroup_mnt_and_root_path = get_cgroup_mnt_and_root_path_v1; + ops->get_init_cgroup_path = get_init_cgroup_path_v1; + ops->get_own_cgroup_path = get_own_cgroup_v1; ++#ifdef ENABLE_OOM_MONITOR + ops->get_cgroup_oom_handler = get_cgroup_oom_handler_v1; ++#endif + return 0; + } +\ No newline at end of file +diff --git a/src/daemon/common/cgroup/cgroup_v2.c b/src/daemon/common/cgroup/cgroup_v2.c +index 76754dc1..ce72e6c4 100644 +--- a/src/daemon/common/cgroup/cgroup_v2.c ++++ b/src/daemon/common/cgroup/cgroup_v2.c +@@ -17,14 +17,18 @@ + #include <stdio.h> + #include <stdlib.h> + #include <sys/stat.h> ++#ifdef ENABLE_OOM_MONITOR + #include <sys/inotify.h> ++#endif + + #include <isula_libutils/auto_cleanup.h> + + #include "utils.h" + #include "path.h" + #include "sysinfo.h" ++#ifdef ENABLE_OOM_MONITOR + #include "events_sender_api.h" ++#endif + + // Cgroup V2 Item Definition + #define CGROUP2_CPU_WEIGHT "cpu.weight" +@@ -416,6 +420,7 @@ static int get_cgroup_mnt_and_root_v2(const char *subsystem, char **mountpoint, + return 0; + } + ++#ifdef ENABLE_OOM_MONITOR + static bool oom_cb_cgroup_v2(int fd, void *cbdata) + { + const size_t events_size = sizeof(struct inotify_event) + NAME_MAX + 1; +@@ -547,6 +552,7 @@ cleanup: + common_free_cgroup_oom_handler_info(info); + return NULL; + } ++#endif + + int get_cgroup_version_v2() + { +@@ -562,6 +568,8 @@ int cgroup_v2_ops_init(cgroup_ops *ops) + ops->get_cgroup_info = get_cgroup_info_v2; + ops->get_cgroup_metrics = get_cgroup_metrics_v2; + ops->get_cgroup_mnt_and_root_path = get_cgroup_mnt_and_root_v2; ++#ifdef ENABLE_OOM_MONITOR + ops->get_cgroup_oom_handler = get_cgroup_oom_handler_v2; ++#endif + return 0; + } +\ No newline at end of file +diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.cc b/src/daemon/common/cri/v1/v1_cri_helpers.cc +index 478dd105..31b6b137 100644 +--- a/src/daemon/common/cri/v1/v1_cri_helpers.cc ++++ b/src/daemon/common/cri/v1/v1_cri_helpers.cc +@@ -517,9 +517,11 @@ void UpdateBaseStatusFromInspect( + } else { // Case 3 + state = runtime::v1::CONTAINER_CREATED; + } ++#ifdef ENABLE_OOM_MONITOR + if (inspect->state->oom_killed == true) { + reason = "OOMKilled"; + } ++#endif + if (inspect->state->error != nullptr) { + message = inspect->state->error; + } +diff --git a/src/daemon/modules/container/container_state.c b/src/daemon/modules/container/container_state.c +index 452a2b26..f8ad0537 100644 +--- a/src/daemon/modules/container/container_state.c ++++ b/src/daemon/modules/container/container_state.c +@@ -587,7 +587,9 @@ container_inspect_state *container_state_to_inspect_state(container_state_t *s) + state->running = s->state->running; + state->paused = s->state->paused; + state->restarting = s->state->restarting; ++#ifdef ENABLE_OOM_MONITOR + state->oom_killed = s->state->oom_killed; ++#endif + state->pid = s->state->pid; + + state->exit_code = s->state->exit_code; +diff --git a/src/daemon/modules/container/supervisor/supervisor.c b/src/daemon/modules/container/supervisor/supervisor.c +index 39d9fdb8..294783eb 100644 +--- a/src/daemon/modules/container/supervisor/supervisor.c ++++ b/src/daemon/modules/container/supervisor/supervisor.c +@@ -42,8 +42,10 @@ + #ifdef ENABLE_CRI_API_V1 + #include "sandbox_ops.h" + #endif ++#ifdef ENABLE_OOM_MONITOR + #include "cgroup.h" + #include "specs_api.h" ++#endif + + pthread_mutex_t g_supervisor_lock = PTHREAD_MUTEX_INITIALIZER; + struct epoll_descr g_supervisor_descr; +@@ -286,6 +288,7 @@ static int supervisor_exit_cb(int fd, uint32_t events, void *cbdata, struct epol + return EPOLL_LOOP_HANDLE_CONTINUE; + } + ++#ifdef ENABLE_OOM_MONITOR + static int oom_handle_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr) + { + cgroup_oom_handler_info_t *oom_handler_info = (cgroup_oom_handler_info_t *)cbdata; +@@ -305,6 +308,7 @@ static int oom_handle_cb(int fd, uint32_t events, void *cbdata, struct epoll_des + + return EPOLL_LOOP_HANDLE_CONTINUE; + } ++#endif + + /* supervisor add exit monitor */ + int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const pid_ppid_info_t *pid_info, +@@ -312,8 +316,10 @@ int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const p + { + int ret = 0; + struct supervisor_handler_data *data = NULL; ++#ifdef ENABLE_OOM_MONITOR + cgroup_oom_handler_info_t *oom_handler_info = NULL; + __isula_auto_free char *cgroup_path = NULL; ++#endif + + if (fd < 0) { + ERROR("Invalid exit fifo fd"); +@@ -326,12 +332,14 @@ int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const p + return -1; + } + ++#ifdef ENABLE_OOM_MONITOR + cgroup_path = merge_container_cgroups_path(cont->common_config->id, cont->hostconfig); + if (cgroup_path == NULL) { + ERROR("Failed to get cgroup path"); + close(fd); + return -1; + } ++#endif + + data = util_common_calloc_s(sizeof(struct supervisor_handler_data)); + if (data == NULL) { +@@ -353,9 +361,12 @@ int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const p + data->pid_info.start_time = pid_info->start_time; + data->pid_info.ppid = pid_info->ppid; + data->pid_info.pstart_time = pid_info->pstart_time; ++#ifdef ENABLE_OOM_MONITOR + oom_handler_info = common_get_cgroup_oom_handler(fd, cont->common_config->id, cgroup_path, exit_fifo); ++#endif + + supervisor_handler_lock(); ++#ifdef ENABLE_OOM_MONITOR + if (oom_handler_info != NULL) { + ret = epoll_loop_add_handler(&g_supervisor_descr, oom_handler_info->oom_event_fd, oom_handle_cb, oom_handler_info); + if (ret != 0) { +@@ -363,6 +374,7 @@ int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const p + goto err; + } + } ++#endif + + ret = epoll_loop_add_handler(&g_supervisor_descr, fd, supervisor_exit_cb, data); + if (ret != 0) { +@@ -374,7 +386,9 @@ int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const p + + err: + supervisor_handler_data_free(data); ++#ifdef ENABLE_OOM_MONITOR + common_free_cgroup_oom_handler_info(oom_handler_info); ++#endif + out: + supervisor_handler_unlock(); + return ret; +-- +2.25.1 + diff --git a/0093-change-fork-process-exit-mode.patch b/0093-change-fork-process-exit-mode.patch new file mode 100644 index 0000000..2806227 --- /dev/null +++ b/0093-change-fork-process-exit-mode.patch @@ -0,0 +1,64 @@ +From 0ff5a421e31096fbd10cf00c45a3849297023391 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Thu, 23 May 2024 01:09:41 +1400 +Subject: [PATCH 093/108] change fork process exit mode + +--- + src/utils/tar/util_archive.c | 16 ++++++++-------- + 1 file changed, 8 insertions(+), 8 deletions(-) + +diff --git a/src/utils/tar/util_archive.c b/src/utils/tar/util_archive.c +index 52b51162..204dab83 100644 +--- a/src/utils/tar/util_archive.c ++++ b/src/utils/tar/util_archive.c +@@ -897,9 +897,9 @@ int archive_unpack(const struct io_read_wrapper *content, const char *dstdir, co + + child_out: + if (ret != 0) { +- exit(EXIT_FAILURE); ++ _exit(EXIT_FAILURE); + } +- exit(EXIT_SUCCESS); ++ _exit(EXIT_SUCCESS); + } + close(pipe_stderr[1]); + pipe_stderr[1] = -1; +@@ -1342,9 +1342,9 @@ int archive_chroot_tar(const char *path, const char *file, const char *root_dir, + child_out: + + if (ret != 0) { +- exit(EXIT_FAILURE); ++ _exit(EXIT_FAILURE); + } else { +- exit(EXIT_SUCCESS); ++ _exit(EXIT_SUCCESS); + } + } + close(pipe_for_read[1]); +@@ -1577,9 +1577,9 @@ int archive_chroot_untar_stream(const struct io_read_wrapper *context, const cha + + child_out: + if (ret != 0) { +- exit(EXIT_FAILURE); ++ _exit(EXIT_FAILURE); + } +- exit(EXIT_SUCCESS); ++ _exit(EXIT_SUCCESS); + } + + close(pipe_stderr[1]); +@@ -1727,9 +1727,9 @@ child_out: + free(tar_base_name); + + if (ret != 0) { +- exit(EXIT_FAILURE); ++ _exit(EXIT_FAILURE); + } else { +- exit(EXIT_SUCCESS); ++ _exit(EXIT_SUCCESS); + } + } + +-- +2.25.1 + diff --git a/0094-fix-error-log-for-verify_cpu_realtime.patch b/0094-fix-error-log-for-verify_cpu_realtime.patch new file mode 100644 index 0000000..c62bbe8 --- /dev/null +++ b/0094-fix-error-log-for-verify_cpu_realtime.patch @@ -0,0 +1,26 @@ +From 1671a136d1b7d209c453a8ad2b1bf062a3afbe09 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Tue, 28 May 2024 17:23:45 +1400 +Subject: [PATCH 094/108] fix error log for verify_cpu_realtime + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/modules/spec/verify.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/daemon/modules/spec/verify.c b/src/daemon/modules/spec/verify.c +index 57e16ef9..1ce76c1d 100644 +--- a/src/daemon/modules/spec/verify.c ++++ b/src/daemon/modules/spec/verify.c +@@ -390,7 +390,7 @@ static int verify_cpu_realtime(const sysinfo_t *sysinfo, int64_t realtime_period + + if (realtime_runtime != 0 && !(sysinfo->cgcpuinfo.cpu_rt_runtime)) { + ERROR("Invalid --cpu-rt-runtime: Your kernel does not support cgroup rt runtime"); +- isulad_set_error_message("Invalid --cpu-rt-period: Your kernel does not support cgroup rt runtime"); ++ isulad_set_error_message("Invalid --cpu-rt-runtime: Your kernel does not support cgroup rt runtime"); + ret = -1; + goto out; + } +-- +2.25.1 + diff --git a/0095-bugfix-change-max-network-name-len.patch b/0095-bugfix-change-max-network-name-len.patch new file mode 100644 index 0000000..3a4ffb7 --- /dev/null +++ b/0095-bugfix-change-max-network-name-len.patch @@ -0,0 +1,54 @@ +From e2a7e6bfb0f0e97e5e1543fac7a5e0807fadaba0 Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Thu, 30 May 2024 16:50:56 +0800 +Subject: [PATCH 095/108] bugfix: change max network name len + +Signed-off-by: liuxu <liuxu156@huawei.com> +--- + docs/design/detailed/Network/native_network_adapter_design.md | 2 +- + .../design/detailed/Network/native_network_adapter_design_zh.md | 2 +- + src/utils/cutils/utils.h | 2 +- + 3 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/docs/design/detailed/Network/native_network_adapter_design.md b/docs/design/detailed/Network/native_network_adapter_design.md +index b58989eb..1010bd73 100644 +--- a/docs/design/detailed/Network/native_network_adapter_design.md ++++ b/docs/design/detailed/Network/native_network_adapter_design.md +@@ -204,7 +204,7 @@ Client: + 1. Parse the parameters passed in by the user. + 2. Verify the incoming parameters, including: + - Only one network is allowed to be created at a time, that is, at most one name can be specified. +- - If name is specified, check whether the length of name exceeds MAX_NETWORK_NAME_LEN(128). ++ - If name is specified, check whether the length of name exceeds MAX_NETWORK_NAME_LEN(255). + 3. Send the request to the server + + Server: +diff --git a/docs/design/detailed/Network/native_network_adapter_design_zh.md b/docs/design/detailed/Network/native_network_adapter_design_zh.md +index 30860f3d..62c4b6ef 100644 +--- a/docs/design/detailed/Network/native_network_adapter_design_zh.md ++++ b/docs/design/detailed/Network/native_network_adapter_design_zh.md +@@ -204,7 +204,7 @@ int native_network_add_container_list(const char *network_name, const char *cont + 1. 解析用户传入的参数 + 2. 对传入的参数进行校验，包括 + - 每次只允许创建一个网络， 即最多指定一个name +- - 若指定name，检查name长度是否超过MAX_NETWORK_NAME_LEN(128) ++ - 若指定name，检查name长度是否超过MAX_NETWORK_NAME_LEN(255) + 3. 发送请求到服务端 + + 服务端： +diff --git a/src/utils/cutils/utils.h b/src/utils/cutils/utils.h +index 3671272a..4417a165 100644 +--- a/src/utils/cutils/utils.h ++++ b/src/utils/cutils/utils.h +@@ -99,7 +99,7 @@ int malloc_trim(size_t pad); + #define MAX_IMAGE_REF_LEN 384 + #define MAX_CONTAINER_NAME_LEN 1024 + #define MAX_RUNTIME_NAME_LEN 32 +-#define MAX_NETWORK_NAME_LEN 128 ++#define MAX_NETWORK_NAME_LEN 255 + + #define LOGIN_USERNAME_LEN 255 + #define LOGIN_PASSWORD_LEN 255 +-- +2.25.1 + diff --git a/0096-del-useless-info.patch b/0096-del-useless-info.patch new file mode 100644 index 0000000..ea885dd --- /dev/null +++ b/0096-del-useless-info.patch @@ -0,0 +1,26 @@ +From b36cfa4325f43b3fa1468ba360b3d51f6ef1c3ca Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Thu, 30 May 2024 17:14:48 +0800 +Subject: [PATCH 096/108] del useless info + +Signed-off-by: liuxu <liuxu156@huawei.com> +--- + release_notes | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/release_notes b/release_notes +index 2d564c7e..fdcd3690 100644 +--- a/release_notes ++++ b/release_notes +@@ -72,7 +72,7 @@ + + dev stats: + - 357 files changed, 7886 insertions(+), 2849 deletions(-) +- - contributors: zhongtao, jikai, haozi007, jake, liuxu, xuxuepeng, zhangxiaoyu, sailorvii, chen524, dreamloy, l00804245, yangjiaqi ++ - contributors: zhongtao, jikai, haozi007, jake, liuxu, xuxuepeng, zhangxiaoyu, sailorvii, chen524, dreamloy, yangjiaqi + + 2023-11-07 xuepengxu release 2.1.4 + - !2238 modify the default value of EANBLE_IMAGE_LIBARAY to off * modify the default value of EANBLE_IMAGE_LIBARAY to off +-- +2.25.1 + diff --git a/0097-code-improve.patch b/0097-code-improve.patch new file mode 100644 index 0000000..cb8d41d --- /dev/null +++ b/0097-code-improve.patch @@ -0,0 +1,83 @@ +From 2aa26649f20ae5992ace7bb8cb62a5ee9c3f7a81 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Thu, 30 May 2024 21:30:43 +1400 +Subject: [PATCH 097/108] code improve + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/cmd/isula/volume/prune.c | 2 +- + src/cmd/options/opt_log.c | 2 +- + src/daemon/executor/container_cb/execution_extend.c | 3 ++- + src/daemon/executor/container_cb/execution_information.c | 2 +- + src/daemon/modules/image/oci/registry_type.c | 2 +- + 5 files changed, 6 insertions(+), 5 deletions(-) + +diff --git a/src/cmd/isula/volume/prune.c b/src/cmd/isula/volume/prune.c +index c8d632ed..3b5bfbf1 100644 +--- a/src/cmd/isula/volume/prune.c ++++ b/src/cmd/isula/volume/prune.c +@@ -79,7 +79,7 @@ out: + + int cmd_volume_prune_main(int argc, const char **argv) + { +- int i = 0; ++ size_t i = 0; + struct isula_libutils_log_config lconf = { 0 }; + int exit_code = 1; + command_t cmd; +diff --git a/src/cmd/options/opt_log.c b/src/cmd/options/opt_log.c +index b1abcfaf..8ffb9966 100644 +--- a/src/cmd/options/opt_log.c ++++ b/src/cmd/options/opt_log.c +@@ -67,7 +67,7 @@ static int log_opt_syslog_facility(const char *key, const char *value, char **pa + "authpriv", "ftp", "local0", "local1", "local2", + "local3", "local4", "local5", "local6", "local7" + }; +- int i; ++ size_t i; + size_t f_len = sizeof(facility_values) / sizeof(const char *); + + for (i = 0; i < f_len; i++) { +diff --git a/src/daemon/executor/container_cb/execution_extend.c b/src/daemon/executor/container_cb/execution_extend.c +index 52401633..ed072848 100644 +--- a/src/daemon/executor/container_cb/execution_extend.c ++++ b/src/daemon/executor/container_cb/execution_extend.c +@@ -990,7 +990,8 @@ out: + + static int update_container_unified(const char *id, const host_config *hostconfig, host_config *chostconfig) + { +- int i, cgroup_version; ++ int cgroup_version; ++ size_t i; + + if (hostconfig->unified == NULL || hostconfig->unified->len == 0) { + return 0; +diff --git a/src/daemon/executor/container_cb/execution_information.c b/src/daemon/executor/container_cb/execution_information.c +index 58924257..4aee3aef 100644 +--- a/src/daemon/executor/container_cb/execution_information.c ++++ b/src/daemon/executor/container_cb/execution_information.c +@@ -244,7 +244,7 @@ static int get_proxy_env(char **proxy, const char *type) + } + *col_pos = '\0'; + nret = snprintf(*proxy, proxy_len, "%s:%s%s", tmp_proxy, mask_str, at_pos); +- if (nret < 0 || nret >= proxy_len) { ++ if (nret < 0 || (size_t)nret >= proxy_len) { + ret = -1; + free(*proxy); + *proxy = NULL; +diff --git a/src/daemon/modules/image/oci/registry_type.c b/src/daemon/modules/image/oci/registry_type.c +index 6c9ff747..7a2c25ed 100644 +--- a/src/daemon/modules/image/oci/registry_type.c ++++ b/src/daemon/modules/image/oci/registry_type.c +@@ -59,7 +59,7 @@ void free_layer_blob(layer_blob *layer) + + void free_pull_desc(pull_descriptor *desc) + { +- int i = 0; ++ size_t i = 0; + + if (desc == NULL) { + return; +-- +2.25.1 + diff --git a/0098-cdi-add-debug-info.patch b/0098-cdi-add-debug-info.patch new file mode 100644 index 0000000..aec57f6 --- /dev/null +++ b/0098-cdi-add-debug-info.patch @@ -0,0 +1,33 @@ +From 5a5f4879246783932ab620b2461a7cd832ddc1f0 Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Fri, 31 May 2024 16:38:26 +0800 +Subject: [PATCH 098/108] cdi:add debug info + +Signed-off-by: liuxu <liuxu156@huawei.com> +--- + src/daemon/modules/device/cdi/cdi_cache.c | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/src/daemon/modules/device/cdi/cdi_cache.c b/src/daemon/modules/device/cdi/cdi_cache.c +index e9a9b804..cd7158dd 100644 +--- a/src/daemon/modules/device/cdi/cdi_cache.c ++++ b/src/daemon/modules/device/cdi/cdi_cache.c +@@ -306,6 +306,7 @@ static void refresh_scan_spec_func(struct cdi_scan_fn_maps *scan_fn_maps, const + ERROR("Failed to insert device to devices by name %s", qualified); + goto error_out; + } ++ DEBUG("Add device %s into memory", qualified); + free(qualified); + qualified = NULL; + } +@@ -445,6 +446,7 @@ static int cdi_inject_devices(struct cdi_cache *c, oci_runtime_spec *oci_spec, s + + for (i = 0; i < devices->len; i++) { + device = devices->items[i]; ++ DEBUG("Search cdi devices %s.", device); + d = map_search(c->devices, (void *)device); + if (d == NULL) { + if (util_append_string_array(unresolved, device) != 0) { +-- +2.25.1 + diff --git a/0099-bugfix-cni-network-name-UT.patch b/0099-bugfix-cni-network-name-UT.patch new file mode 100644 index 0000000..4ed04c3 --- /dev/null +++ b/0099-bugfix-cni-network-name-UT.patch @@ -0,0 +1,28 @@ +From 4a98535064319a9df3143d9c4b397f44fbbb56c5 Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Mon, 3 Jun 2024 16:11:06 +0800 +Subject: [PATCH 099/108] bugfix:cni network name UT + +Signed-off-by: liuxu <liuxu156@huawei.com> +--- + test/cutils/utils_network/utils_network_ut.cc | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/test/cutils/utils_network/utils_network_ut.cc b/test/cutils/utils_network/utils_network_ut.cc +index 33eb6eb5..68f6f011 100644 +--- a/test/cutils/utils_network/utils_network_ut.cc ++++ b/test/cutils/utils_network/utils_network_ut.cc +@@ -344,8 +344,8 @@ TEST(utils_network, test_net_contain_ip) + TEST(utils_network, test_validate_network_name) + { + ASSERT_EQ(util_validate_network_name(nullptr), false); +- ASSERT_EQ(util_validate_network_name(std::string(128, 'a').c_str()), true); +- ASSERT_EQ(util_validate_network_name(std::string(129, 'a').c_str()), false); ++ ASSERT_EQ(util_validate_network_name(std::string(255, 'a').c_str()), true); ++ ASSERT_EQ(util_validate_network_name(std::string(256, 'a').c_str()), false); + ASSERT_EQ(util_validate_network_name(std::string(".abce").c_str()), false); + } + +-- +2.25.1 + diff --git a/0100-bugfix-malloc-right-type-size.patch b/0100-bugfix-malloc-right-type-size.patch new file mode 100644 index 0000000..983d55e --- /dev/null +++ b/0100-bugfix-malloc-right-type-size.patch @@ -0,0 +1,40 @@ +From 7bf26415fcba090e281324ba92f7d7e6487b94fc Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Tue, 4 Jun 2024 16:30:19 +0800 +Subject: [PATCH 100/108] bugfix:malloc right type size + +Signed-off-by: liuxu <liuxu156@huawei.com> +--- + src/client/connect/rest/rest_volumes_client.c | 2 +- + src/daemon/modules/service/service_network.c | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/src/client/connect/rest/rest_volumes_client.c b/src/client/connect/rest/rest_volumes_client.c +index 88254e6c..2327c28e 100644 +--- a/src/client/connect/rest/rest_volumes_client.c ++++ b/src/client/connect/rest/rest_volumes_client.c +@@ -251,7 +251,7 @@ static int prune_request_to_rest(const struct isula_prune_volume_request *reques + parser_error err = NULL; + int ret = 0; + +- nrequest = util_common_calloc_s(sizeof(volume_list_volume_request)); ++ nrequest = util_common_calloc_s(sizeof(volume_prune_volume_request)); + if (nrequest == NULL) { + ERROR("Out of memory"); + return -1; +diff --git a/src/daemon/modules/service/service_network.c b/src/daemon/modules/service/service_network.c +index 6754cf1a..fe6dbf1d 100644 +--- a/src/daemon/modules/service/service_network.c ++++ b/src/daemon/modules/service/service_network.c +@@ -74,7 +74,7 @@ static struct attach_net_conf_list *build_attach_networks(const defs_map_string_ + return NULL; + } + +- list = (struct attach_net_conf_list *)util_common_calloc_s(sizeof(struct attach_net_conf)); ++ list = (struct attach_net_conf_list *)util_common_calloc_s(sizeof(struct attach_net_conf_list)); + if (list == NULL) { + ERROR("Out of memory"); + return NULL; +-- +2.25.1 + diff --git a/0101-use-isula_clean_path-rather-than-realpath.patch b/0101-use-isula_clean_path-rather-than-realpath.patch new file mode 100644 index 0000000..6ac4423 --- /dev/null +++ b/0101-use-isula_clean_path-rather-than-realpath.patch @@ -0,0 +1,26 @@ +From 60a2b15e0090018b7850b37369964bf62e253419 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Wed, 5 Jun 2024 02:07:23 +0000 +Subject: [PATCH 101/108] use isula_clean_path rather than realpath + +Signed-off-by: jikai <jikai11@huawei.com> +--- + src/cmd/isulad-shim/process.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c +index 18fae03f..dd41c77f 100644 +--- a/src/cmd/isulad-shim/process.c ++++ b/src/cmd/isulad-shim/process.c +@@ -489,7 +489,7 @@ static bool attach_fifopath_security_check(process_t *p, const char *fifopath) + return false; + } + +- if (realpath(fifopath, real_path) == NULL) { ++ if (isula_clean_path(fifopath, real_path, sizeof(real_path)) == NULL) { + ERROR("Failed to get realpath for '%s': %d.", real_path, SHIM_SYS_ERR(errno)); + return false; + } +-- +2.25.1 + diff --git a/0102-fix-false-engine-rootpath-reference.patch b/0102-fix-false-engine-rootpath-reference.patch new file mode 100644 index 0000000..a375a44 --- /dev/null +++ b/0102-fix-false-engine-rootpath-reference.patch @@ -0,0 +1,32 @@ +From fa25a8923e47ed6b65e0bcd08954589f5b26092c Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Wed, 5 Jun 2024 02:10:48 +0000 +Subject: [PATCH 102/108] fix false engine rootpath reference + +Signed-off-by: jikai <jikai11@huawei.com> +--- + src/daemon/config/isulad_config.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c +index 80689bce..d7b54498 100644 +--- a/src/daemon/config/isulad_config.c ++++ b/src/daemon/config/isulad_config.c +@@ -398,12 +398,12 @@ char *conf_get_sandbox_rootpath(void) + ERROR("Get rootpath failed"); + return epath; + } +- if (strlen(rootpath) > (PATH_MAX - strlen(ENGINE_ROOTPATH_NAME)) - 2) { ++ if (strlen(rootpath) > (PATH_MAX - strlen(SANDBOX_ROOTPATH_NAME)) - 2) { + ERROR("Root path is too long"); + return epath; + } + // rootpath + "/" + SANDBOX_ROOTPATH_NAME + "/0" +- len = strlen(rootpath) + 1 + strlen(ENGINE_ROOTPATH_NAME) + 1; ++ len = strlen(rootpath) + 1 + strlen(SANDBOX_ROOTPATH_NAME) + 1; + epath = util_smart_calloc_s(sizeof(char), len); + if (epath == NULL) { + ERROR("Out of memory"); +-- +2.25.1 + diff --git a/0103-bugfix-add-note.patch b/0103-bugfix-add-note.patch new file mode 100644 index 0000000..1cdc42b --- /dev/null +++ b/0103-bugfix-add-note.patch @@ -0,0 +1,25 @@ +From 8eea40e09aa34da85cfa191f07cfe7e123c9809d Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Wed, 5 Jun 2024 10:18:06 +0800 +Subject: [PATCH 103/108] bugfix:add note + +Signed-off-by: liuxu <liuxu156@huawei.com> +--- + src/utils/cutils/utils_array.h | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/src/utils/cutils/utils_array.h b/src/utils/cutils/utils_array.h +index 0c4fd217..553c2c13 100644 +--- a/src/utils/cutils/utils_array.h ++++ b/src/utils/cutils/utils_array.h +@@ -30,6 +30,7 @@ void util_free_array_by_len(char **array, size_t len); + + void util_free_array(char **array); + ++// this function just copies the first len elements of array and does not automatically add NULL element in the end. + char **util_copy_array_by_len(char **array, size_t len); + + int util_grow_array(char ***orig_array, size_t *orig_capacity, size_t size, +-- +2.25.1 + diff --git a/0104-bugfix-adapt-network-name-max-len.patch b/0104-bugfix-adapt-network-name-max-len.patch new file mode 100644 index 0000000..0d7c6a3 --- /dev/null +++ b/0104-bugfix-adapt-network-name-max-len.patch @@ -0,0 +1,64 @@ +From 174f7d9d959bd129675651ccf7ef460794188b63 Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Wed, 5 Jun 2024 10:40:22 +0800 +Subject: [PATCH 104/108] bugfix:adapt network name max len + +Signed-off-by: liuxu <liuxu156@huawei.com> +--- + CI/test_cases/network_cases/network_create.sh | 5 +++-- + src/utils/cutils/utils.h | 7 ++++++- + test/cutils/utils_network/utils_network_ut.cc | 4 ++-- + 3 files changed, 11 insertions(+), 5 deletions(-) + +diff --git a/CI/test_cases/network_cases/network_create.sh b/CI/test_cases/network_cases/network_create.sh +index 3107da12..da54fdd0 100755 +--- a/CI/test_cases/network_cases/network_create.sh ++++ b/CI/test_cases/network_cases/network_create.sh +@@ -33,8 +33,9 @@ function test_network_create() + local name1="cni1" + local name2="cni2" + local name3="a" +- for i in $(seq 1 7);do +- name3=${name3}${name3} ++ local basechar="a" ++ for i in $(seq 1 199);do ++ name3=${name3}${basechar} + done + local name4=${name3}b + local invalid_name=".xx" +diff --git a/src/utils/cutils/utils.h b/src/utils/cutils/utils.h +index 4417a165..ce0ca703 100644 +--- a/src/utils/cutils/utils.h ++++ b/src/utils/cutils/utils.h +@@ -99,7 +99,12 @@ int malloc_trim(size_t pad); + #define MAX_IMAGE_REF_LEN 384 + #define MAX_CONTAINER_NAME_LEN 1024 + #define MAX_RUNTIME_NAME_LEN 32 +-#define MAX_NETWORK_NAME_LEN 255 ++/* ++ * Linux limits the length of the file name to 255, ++ * isulad will create file by name "${ISULAD_CNI_NETWORK_CONF_FILE_PRE}${network_name}.conflist" ++ * when create native network,so we limit the length of the network name to 200. ++ */ ++#define MAX_NETWORK_NAME_LEN 200 + + #define LOGIN_USERNAME_LEN 255 + #define LOGIN_PASSWORD_LEN 255 +diff --git a/test/cutils/utils_network/utils_network_ut.cc b/test/cutils/utils_network/utils_network_ut.cc +index 68f6f011..be85e398 100644 +--- a/test/cutils/utils_network/utils_network_ut.cc ++++ b/test/cutils/utils_network/utils_network_ut.cc +@@ -344,8 +344,8 @@ TEST(utils_network, test_net_contain_ip) + TEST(utils_network, test_validate_network_name) + { + ASSERT_EQ(util_validate_network_name(nullptr), false); +- ASSERT_EQ(util_validate_network_name(std::string(255, 'a').c_str()), true); +- ASSERT_EQ(util_validate_network_name(std::string(256, 'a').c_str()), false); ++ ASSERT_EQ(util_validate_network_name(std::string(200, 'a').c_str()), true); ++ ASSERT_EQ(util_validate_network_name(std::string(201, 'a').c_str()), false); + ASSERT_EQ(util_validate_network_name(std::string(".abce").c_str()), false); + } + +-- +2.25.1 + diff --git a/0105-start-sandbox-before-setup-network-by-default.patch b/0105-start-sandbox-before-setup-network-by-default.patch new file mode 100644 index 0000000..df5c44f --- /dev/null +++ b/0105-start-sandbox-before-setup-network-by-default.patch @@ -0,0 +1,140 @@ +From e6b3528acff10fb2bc62e2da0c3754f1e36cbd54 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Wed, 5 Jun 2024 10:04:59 +0800 +Subject: [PATCH 105/108] start sandbox before setup network by default + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/common/cri/cri_helpers.cc | 12 +++++++ + src/daemon/common/cri/cri_helpers.h | 3 ++ + .../cri_pod_sandbox_manager_service.cc | 34 +++++++++++++------ + src/daemon/modules/spec/specs.c | 2 +- + src/utils/cutils/utils_file.c | 2 +- + 5 files changed, 41 insertions(+), 12 deletions(-) + +diff --git a/src/daemon/common/cri/cri_helpers.cc b/src/daemon/common/cri/cri_helpers.cc +index 68d569cc..8117403c 100644 +--- a/src/daemon/common/cri/cri_helpers.cc ++++ b/src/daemon/common/cri/cri_helpers.cc +@@ -47,6 +47,8 @@ const std::string Constants::DOCKER_IMAGEID_PREFIX { "docker://" }; + const std::string Constants::DOCKER_PULLABLE_IMAGEID_PREFIX { "docker-pullable://" }; + const std::string Constants::RUNTIME_READY { "RuntimeReady" }; + const std::string Constants::NETWORK_READY { "NetworkReady" }; ++// Kata 2.x need create network namespace and setup network befoce run podsandbox ++const std::string Constants::NETWORK_SETUP_ANNOTATION_KEY { "cri.sandbox.network.setup.v2" }; + const std::string Constants::POD_CHECKPOINT_KEY { "cri.sandbox.isulad.checkpoint" }; + const std::string Constants::CONTAINER_TYPE_ANNOTATION_KEY { "io.kubernetes.cri.container-type" }; + const std::string Constants::CONTAINER_NAME_ANNOTATION_KEY { "io.kubernetes.cri.container-name" }; +@@ -1140,4 +1142,14 @@ auto GetPodSELinuxLabelOpts(const std::string &selinuxLabel, Errors &error) + return fmtiSuladOpts(selinuxOpts, securityOptSep); + } + ++bool SetupNetworkFirst(const std::map<std::string, std::string> &annotations) ++{ ++ auto iter = annotations.find(CRIHelpers::Constants::NETWORK_SETUP_ANNOTATION_KEY); ++ if (iter == annotations.end()) { ++ return false; ++ } ++ ++ return iter->second == std::string("true"); ++} ++ + } // namespace CRIHelpers +diff --git a/src/daemon/common/cri/cri_helpers.h b/src/daemon/common/cri/cri_helpers.h +index 5c450b32..11a80b45 100644 +--- a/src/daemon/common/cri/cri_helpers.h ++++ b/src/daemon/common/cri/cri_helpers.h +@@ -49,6 +49,7 @@ public: + static const std::string DOCKER_PULLABLE_IMAGEID_PREFIX; + static const std::string RUNTIME_READY; + static const std::string NETWORK_READY; ++ static const std::string NETWORK_SETUP_ANNOTATION_KEY; + static const std::string POD_CHECKPOINT_KEY; + static const size_t MAX_CHECKPOINT_KEY_LEN { 250 }; + static const std::string CONTAINER_TYPE_ANNOTATION_KEY; +@@ -151,6 +152,8 @@ auto GetPodSELinuxLabelOpts(const std::string &selinuxLabel, Errors &error) -> s + auto GetlegacySeccompiSuladOpts(const std::string &seccompProfile, Errors &error) -> std::vector<iSuladOpt>; + + auto GetSeccompiSuladOptsByPath(const char *dstpath, Errors &error) -> std::vector<iSuladOpt>; ++ ++bool SetupNetworkFirst(const std::map<std::string, std::string> &annotations); + }; // namespace CRIHelpers + + #endif // DAEMON_ENTRY_CRI_CRI_HELPERS_H +diff --git a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +index af6b5fff..f852f4df 100644 +--- a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +@@ -655,19 +655,33 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1alpha2::PodSandbox + } + } + +- // Step 7: Setup networking for the sandbox. +- SetupSandboxNetwork(config, response_id, inspect_data, networkOptions, stdAnnos, network_setting_json, error); +- if (error.NotEmpty()) { +- goto cleanup_ns; +- } ++ // Step 7: According to the annotation and network namespace mode, ++ // determine the order of start sandbox and setup network. ++ if (CRIHelpers::SetupNetworkFirst(stdAnnos)) { ++ // Step 7.1: Setup networking for the sandbox, and then start the sandbox container. ++ SetupSandboxNetwork(config, response_id, inspect_data, networkOptions, stdAnnos, network_setting_json, error); ++ if (error.NotEmpty()) { ++ goto cleanup_ns; ++ } + +- // Step 8: Start the sandbox container. +- StartSandboxContainer(response_id, error); +- if (error.NotEmpty()) { +- goto cleanup_network; ++ StartSandboxContainer(response_id, error); ++ if (error.NotEmpty()) { ++ goto cleanup_network; ++ } ++ } else { ++ // Step 7.2: (Default)Start the sandbox container, and then setup networking for the sandbox. ++ StartSandboxContainer(response_id, error); ++ if (error.NotEmpty()) { ++ goto cleanup_ns; ++ } ++ ++ SetupSandboxNetwork(config, response_id, inspect_data, networkOptions, stdAnnos, network_setting_json, error); ++ if (error.NotEmpty()) { ++ goto cleanup_ns; ++ } + } + +- // Step 9: Save network settings json to disk ++ // Step 8: Save network settings json to disk + if (namespace_is_cni(inspect_data->host_config->network_mode)) { + Errors tmpErr; + UpdatePodSandboxNetworkSettings(response_id, network_setting_json, tmpErr); +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index 122f9992..f0538e26 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -1601,7 +1601,7 @@ static int merge_share_network_namespace(const oci_runtime_spec *oci_spec, const + int ret = 0; + char *ns_path = NULL; + +- if (host_spec->network_mode == NULL) { ++ if (host_spec->network_mode == NULL || strlen(host_spec->network_mode) == 0) { + return 0; + } + +diff --git a/src/utils/cutils/utils_file.c b/src/utils/cutils/utils_file.c +index 6fc6852d..90bb156f 100644 +--- a/src/utils/cutils/utils_file.c ++++ b/src/utils/cutils/utils_file.c +@@ -85,7 +85,7 @@ bool util_file_exists(const char *f) + struct stat buf; + int nret; + +- if (f == NULL) { ++ if (f == NULL || strlen(f) == 0) { + return false; + } + +-- +2.25.1 + diff --git a/0106-Revert-use-isula_clean_path-rather-than-realpath.patch b/0106-Revert-use-isula_clean_path-rather-than-realpath.patch new file mode 100644 index 0000000..6115474 --- /dev/null +++ b/0106-Revert-use-isula_clean_path-rather-than-realpath.patch @@ -0,0 +1,28 @@ +From f690c9a2dff298b41dc607e4ea6dd09113a322fb Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Thu, 6 Jun 2024 02:13:36 +0000 +Subject: [PATCH 106/108] Revert "use isula_clean_path rather than realpath" + +This reverts commit 60a2b15e0090018b7850b37369964bf62e253419. + +Signed-off-by: jikai <jikai11@huawei.com> +--- + src/cmd/isulad-shim/process.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c +index dd41c77f..18fae03f 100644 +--- a/src/cmd/isulad-shim/process.c ++++ b/src/cmd/isulad-shim/process.c +@@ -489,7 +489,7 @@ static bool attach_fifopath_security_check(process_t *p, const char *fifopath) + return false; + } + +- if (isula_clean_path(fifopath, real_path, sizeof(real_path)) == NULL) { ++ if (realpath(fifopath, real_path) == NULL) { + ERROR("Failed to get realpath for '%s': %d.", real_path, SHIM_SYS_ERR(errno)); + return false; + } +-- +2.25.1 + diff --git a/0107-bugfix-for-start-sandbox-before-setup-network-by-def.patch b/0107-bugfix-for-start-sandbox-before-setup-network-by-def.patch new file mode 100644 index 0000000..009910f --- /dev/null +++ b/0107-bugfix-for-start-sandbox-before-setup-network-by-def.patch @@ -0,0 +1,28 @@ +From 63f4f9bc9c36825d85a14f6a33102194d30e12a7 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Fri, 7 Jun 2024 02:24:49 +1400 +Subject: [PATCH 107/108] bugfix for start sandbox before setup network by + default + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + .../entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +index f852f4df..bc3f4031 100644 +--- a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +@@ -677,6 +677,9 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1alpha2::PodSandbox + + SetupSandboxNetwork(config, response_id, inspect_data, networkOptions, stdAnnos, network_setting_json, error); + if (error.NotEmpty()) { ++ Errors stopError; ++ StopContainerHelper(response_id, stopError); ++ WARN("Error stop container: %s: %s", response_id.c_str(), stopError.GetCMessage()); + goto cleanup_ns; + } + } +-- +2.25.1 + diff --git a/0108-skip-test-rely-on-docker.io.patch b/0108-skip-test-rely-on-docker.io.patch new file mode 100644 index 0000000..dd146d7 --- /dev/null +++ b/0108-skip-test-rely-on-docker.io.patch @@ -0,0 +1,59 @@ +From d3e7b0b0d19ca4937716d835e3627714157d6cc3 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Tue, 11 Jun 2024 17:14:58 +0800 +Subject: [PATCH 108/108] skip test rely on docker.io + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + CI/test_cases/image_cases/image_search.sh | 4 +++- + CI/test_cases/image_cases/integration_check.sh | 2 +- + CI/test_cases/image_cases/registry.sh | 5 +++-- + 3 files changed, 7 insertions(+), 4 deletions(-) + +diff --git a/CI/test_cases/image_cases/image_search.sh b/CI/test_cases/image_cases/image_search.sh +index 11af02f1..4bf0e099 100755 +--- a/CI/test_cases/image_cases/image_search.sh ++++ b/CI/test_cases/image_cases/image_search.sh +@@ -76,7 +76,9 @@ function test_image_search() + + declare -i ans=0 + +-test_image_search || ((ans++)) ++# unable to pull image from docker.io without agent, skip this test ++# registry API v1 is not implemented in https://3laho3y3.mirror.aliyuncs.com and isula search cannot be tested ++# test_image_search || ((ans++)) + + show_result ${ans} "${curr_path}/${0}" + +diff --git a/CI/test_cases/image_cases/integration_check.sh b/CI/test_cases/image_cases/integration_check.sh +index 6ec3ab52..f340348d 100755 +--- a/CI/test_cases/image_cases/integration_check.sh ++++ b/CI/test_cases/image_cases/integration_check.sh +@@ -27,7 +27,7 @@ image="busybox" + function test_image_info() + { + local ret=0 +- local uimage="docker.io/library/nats" ++ local uimage="nats" + local test="list && inspect image info test => (${FUNCNAME[@]})" + local lid + local cid +diff --git a/CI/test_cases/image_cases/registry.sh b/CI/test_cases/image_cases/registry.sh +index e33983d6..7ea9a0c5 100755 +--- a/CI/test_cases/image_cases/registry.sh ++++ b/CI/test_cases/image_cases/registry.sh +@@ -74,8 +74,9 @@ function isula_pull() + isula run --rm -ti busybox echo hello 2>&1 | grep pulling + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - --pull missing failed" && ((ret++)) + +- isula pull docker.io/library/busybox:latest +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - --pull docker.io/library/busybox:latest failed" && ((ret++)) ++ # Unable to pull image from docker.io without agent, skip this test ++ # isula pull docker.io/library/busybox:latest ++ # [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - --pull docker.io/library/busybox:latest failed" && ((ret++)) + + isula pull 3laho3y3.mirror.aliyuncs.com/library/busybox + fn_check_eq "$?" "0" "isula pull 3laho3y3.mirror.aliyuncs.com/library/busybox" +-- +2.25.1 + diff --git a/0109-modify-default-registry-mirrors-in-ci-test.patch b/0109-modify-default-registry-mirrors-in-ci-test.patch new file mode 100644 index 0000000..f0c1a98 --- /dev/null +++ b/0109-modify-default-registry-mirrors-in-ci-test.patch @@ -0,0 +1,26 @@ +From d6284e5e786e1407c2ce5ef098a39c154650bd38 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Wed, 12 Jun 2024 10:57:39 +0800 +Subject: [PATCH 109/121] modify default registry mirrors in ci test + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + CI/test_cases/container_cases/test_data/daemon.json | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/CI/test_cases/container_cases/test_data/daemon.json b/CI/test_cases/container_cases/test_data/daemon.json +index 2664c6b2..ab7d0360 100644 +--- a/CI/test_cases/container_cases/test_data/daemon.json ++++ b/CI/test_cases/container_cases/test_data/daemon.json +@@ -24,7 +24,7 @@ + "overlay2.override_kernel_check=true" + ], + "registry-mirrors": [ +- "docker.io" ++ "https://3laho3y3.mirror.aliyuncs.com" + ], + "insecure-registries": [ + ], +-- +2.25.1 + diff --git a/0110-add-timestamp-in-PodSandboxStatu-response.patch b/0110-add-timestamp-in-PodSandboxStatu-response.patch new file mode 100644 index 0000000..2d2b276 --- /dev/null +++ b/0110-add-timestamp-in-PodSandboxStatu-response.patch @@ -0,0 +1,25 @@ +From 5087d7501308660970aa9e7c12cf5be7a3d9b063 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Wed, 12 Jun 2024 15:20:17 +0000 +Subject: [PATCH 110/121] add timestamp in PodSandboxStatu response + +Signed-off-by: jikai <jikai11@huawei.com> +--- + src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +index fa726e2c..2a458a6d 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +@@ -910,6 +910,7 @@ void PodSandboxManagerService::PodSandboxStatus(const std::string &podSandboxID, + for (auto &containerStatus : containerStatuses) { + *(reply->add_containers_statuses()) = *containerStatus; + } ++ reply->set_timestamp(util_get_now_time_nanos()); + return; + } + +-- +2.25.1 + diff --git a/0111-bugfix-for-file-param-verify.patch b/0111-bugfix-for-file-param-verify.patch new file mode 100644 index 0000000..2903c4b --- /dev/null +++ b/0111-bugfix-for-file-param-verify.patch @@ -0,0 +1,75 @@ +From d0fd2c2bf87d7befaa8810a70d7eb2061664f02f Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Fri, 14 Jun 2024 09:55:28 +0800 +Subject: [PATCH 111/121] bugfix for file param verify + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/cmd/isula/base/create.c | 19 +++++++++++++++++++ + src/cmd/isula/images/load.c | 6 ++++++ + 2 files changed, 25 insertions(+) + +diff --git a/src/cmd/isula/base/create.c b/src/cmd/isula/base/create.c +index 543b8fd6..b04dddb5 100644 +--- a/src/cmd/isula/base/create.c ++++ b/src/cmd/isula/base/create.c +@@ -292,6 +292,12 @@ static int append_env_variables_to_conf(const char *env_file, isula_container_co + int ret = 0; + size_t file_size; + ++ if (util_dir_exists(env_file)) { ++ COMMAND_ERROR("Env file is a directory: %s", env_file); ++ ret = -1; ++ goto out; ++ } ++ + if (!util_file_exists(env_file)) { + COMMAND_ERROR("env file not exists: %s", env_file); + ret = -1; +@@ -427,6 +433,12 @@ static int append_labels_to_conf(const char *label_file, isula_container_config_ + int ret = 0; + size_t file_size; + ++ if (util_dir_exists(label_file)) { ++ COMMAND_ERROR("Label file is a directory: %s", label_file); ++ ret = -1; ++ goto out; ++ } ++ + if (!util_file_exists(label_file)) { + COMMAND_ERROR("label file not exists: %s", label_file); + ret = -1; +@@ -2357,6 +2369,13 @@ static int create_check_env_target_file(const struct client_arguments *args) + ret = -1; + goto out; + } ++ ++ if (util_dir_exists(env_path)) { ++ COMMAND_ERROR("Env target file is a directory: %s", env_path); ++ ret = -1; ++ goto out; ++ } ++ + if (!util_file_exists(env_path)) { + goto out; + } +diff --git a/src/cmd/isula/images/load.c b/src/cmd/isula/images/load.c +index 314e5d5e..cb39dee7 100644 +--- a/src/cmd/isula/images/load.c ++++ b/src/cmd/isula/images/load.c +@@ -162,6 +162,12 @@ int cmd_load_main(int argc, const char **argv) + g_cmd_load_args.file = file; + } + ++ if (util_dir_exists(g_cmd_load_args.file)) { ++ COMMAND_ERROR("Load file is a directory: %s", g_cmd_load_args.file); ++ ret = -1; ++ exit(exit_code); ++ } ++ + if (!util_file_exists(g_cmd_load_args.file)) { + COMMAND_ERROR("File %s is not exist", g_cmd_load_args.file); + exit(exit_code); +-- +2.25.1 + diff --git a/0112-bugfix-change-cni-log-info.patch b/0112-bugfix-change-cni-log-info.patch new file mode 100644 index 0000000..adc18b1 --- /dev/null +++ b/0112-bugfix-change-cni-log-info.patch @@ -0,0 +1,26 @@ +From 359a6673e01bef937adcc17f99ee94b67caca32e Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Fri, 14 Jun 2024 17:12:58 +0800 +Subject: [PATCH 112/121] bugfix:change cni log info + +Signed-off-by: liuxu <liuxu156@huawei.com> +--- + .../modules/network/cni_operator/libcni/invoke/libcni_exec.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.c b/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.c +index 74d6d74a..1e4a7138 100644 +--- a/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.c ++++ b/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.c +@@ -247,7 +247,7 @@ static char *env_stringify(char *(*pargs)[2], size_t len) + bool invalid_arg = (pargs == NULL || len == 0); + + if (invalid_arg) { +- ERROR("Invalid arguments"); ++ DEBUG("Empty arguments"); + return NULL; + } + +-- +2.25.1 + diff --git a/0113-move-shutdown-handle-after-init-module.patch b/0113-move-shutdown-handle-after-init-module.patch new file mode 100644 index 0000000..5ac3acc --- /dev/null +++ b/0113-move-shutdown-handle-after-init-module.patch @@ -0,0 +1,43 @@ +From 0ae6244c6bfed229a46d300888977a4967e1d718 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Wed, 19 Jun 2024 09:50:51 +0800 +Subject: [PATCH 113/121] move shutdown handle after init module + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/cmd/isulad/main.c | 12 +++++++----- + 1 file changed, 7 insertions(+), 5 deletions(-) + +diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c +index 3e2249d7..52ac3172 100644 +--- a/src/cmd/isulad/main.c ++++ b/src/cmd/isulad/main.c +@@ -1669,11 +1669,6 @@ static int start_daemon_threads() + { + int ret = -1; + +- if (new_shutdown_handler()) { +- ERROR("Create new shutdown handler thread failed"); +- goto out; +- } +- + if (events_module_init() != 0) { + goto out; + } +@@ -1825,6 +1820,13 @@ int main(int argc, char **argv) + goto failure; + } + ++ // after all modules are initialized, enable the shutdown handler to ++ // prevent shutdown handler from cleaning up incompletely initialized modules. ++ if (new_shutdown_handler()) { ++ ERROR("Create new shutdown handler thread failed"); ++ goto failure; ++ } ++ + #ifdef ENABLE_PLUGIN + if (start_plugin_manager()) { + ERROR("Failed to init plugin_manager"); +-- +2.25.1 + diff --git a/0114-bugfix-for-null-pointer-reference.patch b/0114-bugfix-for-null-pointer-reference.patch new file mode 100644 index 0000000..a39467e --- /dev/null +++ b/0114-bugfix-for-null-pointer-reference.patch @@ -0,0 +1,56 @@ +From 701180b53d1c52376f753b94c5cf09987ae789b3 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Tue, 18 Jun 2024 16:02:25 +0800 +Subject: [PATCH 114/121] bugfix for null pointer reference + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/entry/connect/grpc/grpc_service.cc | 14 ++++++++++++-- + 1 file changed, 12 insertions(+), 2 deletions(-) + +diff --git a/src/daemon/entry/connect/grpc/grpc_service.cc b/src/daemon/entry/connect/grpc/grpc_service.cc +index 1d8de922..300af082 100644 +--- a/src/daemon/entry/connect/grpc/grpc_service.cc ++++ b/src/daemon/entry/connect/grpc/grpc_service.cc +@@ -100,7 +100,9 @@ public: + { + // Wait for the server to shutdown. Note that some other thread must be + // responsible for shutting down the server for this call to ever return. +- m_server->Wait(); ++ if (m_server != nullptr) { ++ m_server->Wait(); ++ } + + // Wait for stream server to shutdown + m_criService.Wait(); +@@ -109,7 +111,9 @@ public: + void Shutdown(void) + { + // call CRI to shutdown stream server, shutdown cri first to notify events thread to exit +- m_criService.Shutdown(); ++ if (m_server != nullptr) { ++ m_server->Shutdown(); ++ } + + m_server->Shutdown(); + +@@ -242,10 +246,16 @@ int grpc_server_init(const struct service_arguments *args) + + void grpc_server_wait(void) + { ++ if (g_grpcserver == nullptr) { ++ return; ++ } + g_grpcserver->Wait(); + } + + void grpc_server_shutdown(void) + { ++ if (g_grpcserver == nullptr) { ++ return; ++ } + g_grpcserver->Shutdown(); + } +-- +2.25.1 + diff --git a/0115-bugfix-for-m_criService-shutdown.patch b/0115-bugfix-for-m_criService-shutdown.patch new file mode 100644 index 0000000..8d4d859 --- /dev/null +++ b/0115-bugfix-for-m_criService-shutdown.patch @@ -0,0 +1,33 @@ +From 93b1df1a1d3fcf6d285102f3cc1f79e6241aa393 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Thu, 4 Jul 2024 10:58:38 +0800 +Subject: [PATCH 115/121] bugfix for m_criService shutdown + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/entry/connect/grpc/grpc_service.cc | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/src/daemon/entry/connect/grpc/grpc_service.cc b/src/daemon/entry/connect/grpc/grpc_service.cc +index 300af082..fb5ec3cb 100644 +--- a/src/daemon/entry/connect/grpc/grpc_service.cc ++++ b/src/daemon/entry/connect/grpc/grpc_service.cc +@@ -111,12 +111,12 @@ public: + void Shutdown(void) + { + // call CRI to shutdown stream server, shutdown cri first to notify events thread to exit ++ m_criService.Shutdown(); ++ + if (m_server != nullptr) { + m_server->Shutdown(); + } +- +- m_server->Shutdown(); +- ++ + // Shutdown daemon, this operation should remove socket file. + for (const auto &address : m_socketPath) { + if (address.find(UNIX_SOCKET_PREFIX) == 0) { +-- +2.25.1 + diff --git a/0116-fix-bug-in-ci-test.patch b/0116-fix-bug-in-ci-test.patch new file mode 100644 index 0000000..181fbc7 --- /dev/null +++ b/0116-fix-bug-in-ci-test.patch @@ -0,0 +1,54 @@ +From c7cf33c432b3d9479b2fe365169d4b9a37cae8f7 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Tue, 9 Jul 2024 12:30:01 +0000 +Subject: [PATCH 116/121] fix bug in ci test + +Signed-off-by: jikai <jikai11@huawei.com> +--- + CI/test_cases/container_cases/run.sh | 6 +++--- + CI/test_cases/helpers.sh | 5 +++-- + 2 files changed, 6 insertions(+), 5 deletions(-) + +diff --git a/CI/test_cases/container_cases/run.sh b/CI/test_cases/container_cases/run.sh +index 1bfd388b..ef04b547 100755 +--- a/CI/test_cases/container_cases/run.sh ++++ b/CI/test_cases/container_cases/run.sh +@@ -26,9 +26,9 @@ source ../helpers.sh + function do_test_t() + { + tid=`isula run --runtime $1 -tid --name hostname busybox` +- chostname=`isula exec -it $tid hostname` +- clean_hostname=$(echo "$hostname" | sed 's/[\x01-\x1F\x7F]//g') +- fn_check_eq "${clean_hostname}" "${tid:0:12}" "default hostname is not id of container" ++ # should not use -it option, otherwise the hostname will containe special characters such as '$' or '\r' ++ hostname=`isula exec $tid hostname` ++ fn_check_eq "${hostname}" "${tid:0:12}" "default hostname is not id of container" + isula exec -it hostname env | grep HOSTNAME + fn_check_eq "$?" "0" "check HOSTNAME env failed" + isula stop -t 0 $tid +diff --git a/CI/test_cases/helpers.sh b/CI/test_cases/helpers.sh +index c5eba8a2..0288b4ea 100755 +--- a/CI/test_cases/helpers.sh ++++ b/CI/test_cases/helpers.sh +@@ -52,15 +52,16 @@ function cut_output_lines() { + return $retval + } + ++# use string compare to check the result + function fn_check_eq() { +- if [[ "$1" -ne "$2" ]];then ++ if [ "x$1" != "x$2" ];then + echo "$3" + TC_RET_T=$(($TC_RET_T+1)) + fi + } + + function fn_check_ne() { +- if [[ "$1" -eq "$2" ]];then ++ if [[ "x$1" == "x$2" ]];then + echo "$3" + TC_RET_T=$(($TC_RET_T+1)) + fi +-- +2.25.1 + diff --git a/0117-add-nri-design-doc.patch b/0117-add-nri-design-doc.patch new file mode 100644 index 0000000..276caba --- /dev/null +++ b/0117-add-nri-design-doc.patch @@ -0,0 +1,502 @@ +From 0b55502b6e104df2cb34bbe176d3168456f0b024 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Wed, 6 Mar 2024 11:00:04 +0800 +Subject: [PATCH 117/149] add nri design doc + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + .../detailed/NRI/nri_CreateContainer.svg | 16 + + .../design/detailed/NRI/nri_RunPodSandbox.svg | 16 + + docs/design/detailed/NRI/nri_design.md | 382 ++++++++++++++++++ + docs/design/detailed/NRI/nri_detail.svg | 16 + + docs/design/detailed/NRI/nri_init.svg | 16 + + 5 files changed, 446 insertions(+) + create mode 100644 docs/design/detailed/NRI/nri_CreateContainer.svg + create mode 100644 docs/design/detailed/NRI/nri_RunPodSandbox.svg + create mode 100644 docs/design/detailed/NRI/nri_design.md + create mode 100644 docs/design/detailed/NRI/nri_detail.svg + create mode 100644 docs/design/detailed/NRI/nri_init.svg + +diff --git a/docs/design/detailed/NRI/nri_CreateContainer.svg b/docs/design/detailed/NRI/nri_CreateContainer.svg +new file mode 100644 +index 00000000..3c10bf93 +--- /dev/null ++++ b/docs/design/detailed/NRI/nri_CreateContainer.svg +@@ -0,0 +1,16 @@ ++<svg version="1.1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1386.5730653205294 833.4522030026228" width="1386.5730653205294" height="833.4522030026228"> ++ <!-- svg-source:excalidraw --> ++ ++ <defs> ++ <style class="style-fonts"> ++ @font-face { ++ font-family: "Virgil"; ++ src: url("https://excalidraw.com/Virgil.woff2"); ++ } ++ @font-face { ++ font-family: "Cascadia"; ++ src: url("https://excalidraw.com/Cascadia.woff2"); ++ } ++ </style> ++ </defs> ++ <rect x="0" y="0" width="1386.5730653205294" height="833.4522030026228" fill="#ffffff"></rect><g stroke-linecap="round" transform="translate(473.2278685676763 10) rotate(0 148.5 35)"><path d="M4.66 4.05 C4.66 4.05, 4.66 4.05, 4.66 4.05 M4.66 4.05 C4.66 4.05, 4.66 4.05, 4.66 4.05 M3.08 11.96 C6.5 7.9, 8.48 4.99, 11.61 2.15 M3.08 11.96 C5.97 8.27, 7.9 5.63, 11.61 2.15 M2.17 19.11 C8.4 14.62, 11.87 7.4, 17.91 1 M2.17 19.11 C7.8 13.09, 11.53 6.61, 17.91 1 M1.91 25.51 C4.78 20.71, 8.95 15.02, 23.56 0.6 M1.91 25.51 C7.76 19.69, 13.37 12.45, 23.56 0.6 M2.3 31.15 C6.38 24.81, 14.86 17.86, 28.54 0.96 M2.3 31.15 C7.01 25.32, 13.55 18.4, 28.54 0.96 M2.69 36.79 C10.76 27.55, 20.54 13.65, 34.19 0.57 M2.69 36.79 C11.59 25.24, 21.92 15.27, 34.19 0.57 M2.43 43.19 C9.91 32.83, 18.54 27.33, 39.17 0.93 M2.43 43.19 C14.03 29.81, 26.34 14.24, 39.17 0.93 M2.83 48.83 C20.17 31.44, 34.86 12.4, 44.82 0.53 M2.83 48.83 C17.5 31.84, 32.39 15.7, 44.82 0.53 M3.22 54.48 C16 40.84, 27.55 26.66, 49.8 0.89 M3.22 54.48 C13.2 42.89, 22.32 32.54, 49.8 0.89 M2.31 61.63 C15.97 45.33, 33.57 27.38, 55.45 0.5 M2.31 61.63 C14.83 47.88, 27.13 32.28, 55.45 0.5 M5.32 64.25 C18.39 47.82, 33.02 34.4, 60.43 0.86 M5.32 64.25 C24.31 42.48, 41.69 21.48, 60.43 0.86 M7.69 67.63 C25.22 48.57, 42.54 30.21, 66.08 0.46 M7.69 67.63 C25.16 48.5, 41.44 27.91, 66.08 0.46 M11.36 69.5 C27.95 51.75, 42.18 32.43, 71.06 0.82 M11.36 69.5 C25.88 53.75, 39.66 36.82, 71.06 0.82 M15.04 71.37 C33.18 50.72, 46.83 31.7, 76.71 0.43 M15.04 71.37 C38.4 43.24, 63.04 14.72, 76.71 0.43 M20.68 70.98 C36.42 52.15, 54.39 33.27, 81.69 0.79 M20.68 70.98 C38.17 50.56, 55.76 29.97, 81.69 0.79 M25.67 71.34 C48.86 46.46, 68.06 21.4, 87.34 0.39 M25.67 71.34 C47.45 48.33, 68.08 23.52, 87.34 0.39 M31.31 70.94 C45.46 54.01, 56.72 41.75, 92.32 0.75 M31.31 70.94 C46.05 51.82, 62.72 33.57, 92.32 0.75 M36.3 71.3 C54.73 50.85, 73.65 32.09, 97.97 0.36 M36.3 71.3 C51.16 53.4, 65.33 38.57, 97.97 0.36 M41.94 70.91 C59.89 47.86, 77.36 27.32, 102.95 0.72 M41.94 70.91 C62.03 45.95, 83.05 21.97, 102.95 0.72 M46.93 71.27 C64.11 53.86, 77.18 36.37, 107.94 1.08 M46.93 71.27 C66.53 47.77, 85.06 25.52, 107.94 1.08 M51.91 71.63 C70.22 51.16, 87.25 29.63, 113.58 0.69 M51.91 71.63 C74.37 45.96, 98.78 19.71, 113.58 0.69 M57.56 71.23 C74.54 52.88, 88.33 35.88, 118.57 1.05 M57.56 71.23 C78.16 46.62, 99.25 24.09, 118.57 1.05 M62.54 71.59 C80.12 48.64, 99.93 31.23, 124.21 0.65 M62.54 71.59 C84.62 46.5, 109.1 19.27, 124.21 0.65 M68.19 71.2 C88.38 47.36, 104.83 26.95, 129.2 1.01 M68.19 71.2 C92.96 42.85, 116.47 16.59, 129.2 1.01 M73.17 71.56 C93.97 47, 115.86 23.03, 134.84 0.62 M73.17 71.56 C86.6 55.79, 100.2 39.46, 134.84 0.62 M78.82 71.16 C98.62 49.93, 114.7 28.93, 139.83 0.98 M78.82 71.16 C94.73 52.23, 110.34 34.55, 139.83 0.98 M83.8 71.52 C103.47 45.43, 124.18 23.7, 145.47 0.58 M83.8 71.52 C97.67 56.31, 110.6 42.68, 145.47 0.58 M89.45 71.13 C107.83 47.44, 127.49 25.39, 150.46 0.94 M89.45 71.13 C106.98 50.99, 125.04 32.32, 150.46 0.94 M94.43 71.49 C106.55 54.58, 121.85 38.53, 156.1 0.55 M94.43 71.49 C108.67 55.29, 122 39.93, 156.1 0.55 M100.08 71.1 C115.58 54.58, 129.24 33.79, 161.09 0.91 M100.08 71.1 C116.93 49.56, 136.29 29.01, 161.09 0.91 M105.06 71.46 C127.63 46.07, 150.87 18.26, 166.73 0.51 M105.06 71.46 C117.44 55.43, 131.41 39.08, 166.73 0.51 M110.71 71.06 C125.84 52.24, 142.35 31.81, 171.72 0.87 M110.71 71.06 C127.35 51.8, 143.79 33.6, 171.72 0.87 M115.69 71.42 C134.57 50.19, 152.94 28.19, 177.36 0.48 M115.69 71.42 C135.43 46.82, 158.16 22.72, 177.36 0.48 M121.34 71.03 C143.6 46.74, 161.69 24.58, 182.35 0.84 M121.34 71.03 C138.49 51.19, 155.15 31.15, 182.35 0.84 M126.32 71.39 C143.76 49.5, 165.13 28.24, 187.99 0.44 M126.32 71.39 C149.56 45.42, 170.75 21.07, 187.99 0.44 M131.97 70.99 C148.43 53.11, 164.27 36.01, 192.98 0.8 M131.97 70.99 C154.55 45.45, 176.18 21.44, 192.98 0.8 M136.95 71.35 C156.69 44.53, 178.7 21.81, 197.97 1.16 M136.95 71.35 C154.24 52.8, 169.63 33.46, 197.97 1.16 M141.94 71.71 C157.03 52.98, 176.51 32.4, 203.61 0.77 M141.94 71.71 C155.1 58.32, 168.21 42.61, 203.61 0.77 M147.58 71.32 C169.55 46.19, 193.79 17.04, 208.6 1.13 M147.58 71.32 C165.67 51.64, 182.27 30.99, 208.6 1.13 M152.57 71.68 C164.48 57.17, 177.97 41.95, 214.24 0.74 M152.57 71.68 C170.69 50.79, 190.26 28.83, 214.24 0.74 M158.21 71.28 C173.11 53.82, 186.17 42.19, 219.23 1.1 M158.21 71.28 C174.25 53.5, 189.55 33.99, 219.23 1.1 M163.2 71.64 C186.48 41.17, 209.68 16.62, 224.87 0.7 M163.2 71.64 C176.15 55.17, 189.21 40, 224.87 0.7 M168.84 71.25 C184.96 54.31, 200.83 33.82, 229.86 1.06 M168.84 71.25 C183.74 54.76, 198.47 36.78, 229.86 1.06 M173.83 71.61 C193.89 50.11, 214.08 27, 235.5 0.67 M173.83 71.61 C190.33 53.13, 205.27 35.42, 235.5 0.67 M179.47 71.21 C202.66 44.83, 227.97 17.85, 240.49 1.03 M179.47 71.21 C198.4 47.76, 218.88 25.1, 240.49 1.03 M184.46 71.57 C202.68 48.96, 223.76 26.73, 246.13 0.63 M184.46 71.57 C196.3 56.09, 210.32 41.11, 246.13 0.63 M190.1 71.18 C203 53.25, 218.39 36.01, 251.12 0.99 M190.1 71.18 C202.43 55.27, 216.47 40.68, 251.12 0.99 M195.09 71.54 C219.23 46.03, 240.82 17.21, 256.76 0.6 M195.09 71.54 C210.25 53.76, 226.28 35.5, 256.76 0.6 M200.73 71.15 C222.94 44.7, 245.75 16.12, 261.75 0.96 M200.73 71.15 C213.93 54.7, 227.83 39.2, 261.75 0.96 M205.72 71.51 C225.39 50.87, 242.21 31.23, 267.39 0.56 M205.72 71.51 C220.11 55.11, 234.62 39.04, 267.39 0.56 M211.37 71.11 C231.06 48.69, 247.88 27.63, 272.38 0.92 M211.37 71.11 C230.74 47.89, 251.14 24.35, 272.38 0.92 M216.35 71.47 C230.24 54.39, 248.51 34.61, 278.02 0.53 M216.35 71.47 C236.94 47.65, 258.82 22.86, 278.02 0.53 M221.34 71.83 C236.63 57.06, 249.88 41.05, 283.01 0.89 M221.34 71.83 C237.3 53.22, 253.57 33.63, 283.01 0.89 M226.98 71.44 C248.89 45.61, 272.98 18.33, 286.68 2.76 M226.98 71.44 C242.66 52.25, 260.13 33.02, 286.68 2.76 M231.97 71.8 C249.34 53.45, 261.96 34.79, 291.67 3.12 M231.97 71.8 C247.17 53.87, 263.04 35.94, 291.67 3.12 M237.61 71.4 C256.7 51.37, 273.55 30, 294.03 6.5 M237.61 71.4 C252.75 54.87, 269.38 35.77, 294.03 6.5 M242.6 71.76 C257.36 52.93, 278.19 31.83, 296.4 9.88 M242.6 71.76 C257.7 53.41, 273.71 35.78, 296.4 9.88 M248.24 71.37 C267.08 49.31, 283.81 29.09, 296.13 16.27 M248.24 71.37 C261.81 56.43, 274.93 40.62, 296.13 16.27 M253.23 71.73 C268.58 52.77, 286.03 34.51, 296.53 21.92 M253.23 71.73 C263.21 59.51, 274.5 47.28, 296.53 21.92 M258.87 71.33 C271.5 59.23, 280.63 44.32, 296.27 28.31 M258.87 71.33 C273.22 54.77, 286.46 40.35, 296.27 28.31 M263.86 71.69 C274.11 56.71, 289.8 42.07, 296.66 33.96 M263.86 71.69 C274.41 58.44, 285.58 47.06, 296.66 33.96 M269.5 71.3 C275.96 60.92, 282.85 53.74, 296.4 40.36 M269.5 71.3 C277.92 60.48, 287.31 50.09, 296.4 40.36 M274.49 71.66 C282.24 60.82, 289.64 54.72, 296.8 46 M274.49 71.66 C279.11 64.64, 286.31 59.54, 296.8 46 M280.79 70.51 C287 65.5, 289.4 60.74, 297.19 51.64 M280.79 70.51 C284.72 65.31, 291.01 59.71, 297.19 51.64 M287.09 69.36 C290.02 66.73, 292.91 60.92, 297.58 57.28 M287.09 69.36 C290.14 65.3, 294.22 61.52, 297.58 57.28" stroke="#ffc9c9" stroke-width="0.5" fill="none"></path><path d="M17.5 0 M17.5 0 C101.96 -0.85, 186.55 -0.51, 279.5 0 M17.5 0 C90.11 -1.35, 164.04 -2.03, 279.5 0 M279.5 0 C292.31 1.41, 297.72 4.27, 297 17.5 M279.5 0 C290.9 -1.65, 295.24 4.06, 297 17.5 M297 17.5 C297.69 23.98, 295.78 32.84, 297 52.5 M297 17.5 C297.76 24.88, 297.99 31.52, 297 52.5 M297 52.5 C296.78 64.92, 292.58 71.69, 279.5 70 M297 52.5 C296.2 62.63, 291.88 68.86, 279.5 70 M279.5 70 C178.07 68.77, 77.7 69.8, 17.5 70 M279.5 70 C186.76 70.43, 93.47 70.03, 17.5 70 M17.5 70 C6.63 69.3, 0.77 65.53, 0 52.5 M17.5 70 C3.84 70.52, -0.73 64.96, 0 52.5 M0 52.5 C1.96 41.93, 0.9 36.4, 0 17.5 M0 52.5 C-0.4 42.65, -0.08 33.99, 0 17.5 M0 17.5 C-1.7 6.03, 5.38 -1.88, 17.5 0 M0 17.5 C-1.46 7.53, 6.61 1.53, 17.5 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(603.0578780281255 32.5) rotate(0 18.66999053955078 12.5)"><text x="18.66999053955078" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">CRI</text></g><g mask="url(#mask-r_HqSGYDu1Se27X58_a8_)" stroke-linecap="round"><g transform="translate(622.1469858883454 79.78275368430513) rotate(0 1.0082239731011668 36.00664331455255)"><path d="M-1.11 -1 C-0.63 11.1, 2.53 59.48, 3.12 71.93 M0.51 1.09 C0.77 13.38, 1.25 61.24, 1.95 73.02" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(622.1469858883454 79.78275368430513) rotate(0 1.0082239731011668 36.00664331455255)"><path d="M-10.85 44.45 C-7.16 52.38, -0.69 63.19, 0.77 72.96 M-10.12 45.4 C-7.21 53.17, -2.25 61.8, 2.36 72.25" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(622.1469858883454 79.78275368430513) rotate(0 1.0082239731011668 36.00664331455255)"><path d="M9.66 43.81 C6.72 51.76, 6.57 62.78, 0.77 72.96 M10.39 44.76 C7.47 52.85, 6.6 61.67, 2.36 72.25" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask id="mask-r_HqSGYDu1Se27X58_a8_"><rect x="0" y="0" fill="#fff" width="725.6949756424976" height="252.56058571555513"></rect><rect x="544.3210509558517" y="103.67166969993013" fill="#000" width="159.19985961914062" height="25" opacity="1"></rect></mask><g transform="translate(544.3210509558512 103.67166969993013) rotate(0 78.83415890559513 12.117727298927548)"><text x="79.59992980957031" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">CreateContainer</text></g><g stroke-linecap="round" transform="translate(452.81365255501146 153.56058571555513) rotate(0 176 36)"><path d="M18 0 M18 0 C102.63 2.05, 184.36 2.06, 334 0 M18 0 C107.11 -1.32, 196.88 -1.37, 334 0 M334 0 C347.74 1.78, 352.82 4.46, 352 18 M334 0 C344.19 -0.08, 351.55 4.58, 352 18 M352 18 C353.07 24.17, 352.66 35.61, 352 54 M352 18 C351.83 28.57, 351.35 41.73, 352 54 M352 54 C351.6 64.58, 346.29 70.3, 334 72 M352 54 C351.68 65.08, 344.99 73.63, 334 72 M334 72 C235.65 71.14, 137.14 71.97, 18 72 M334 72 C267.55 72.23, 201.12 72.65, 18 72 M18 72 C4.98 72.26, 1.53 64.55, 0 54 M18 72 C5.94 72.11, -0.63 64.54, 0 54 M0 54 C0.23 41.01, 1.58 26.04, 0 18 M0 54 C-0.94 45.24, -0.41 34.77, 0 18 M0 18 C-1.54 6.9, 7.93 0.68, 18 0 M0 18 C2.17 4.71, 5.36 1.25, 18 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(573.8036961951482 177.06058571555513) rotate(0 55.00995635986328 12.5)"><text x="55.00995635986328" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">CRI module</text></g><g mask="url(#mask-OpknWNg9Sznc-QhpTZdV_)" stroke-linecap="round"><g transform="translate(821.1732677211501 387.78532718011866) rotate(0 165.60605119295678 -10.532439672289911)"><path d="M-0.04 -0.19 C27.89 -3.27, 112.41 -18.53, 167.49 -19.17 C222.57 -19.81, 303.27 -6.55, 330.42 -4.04 M-1.51 -1.34 C26.76 -4.69, 113.9 -20.67, 169.61 -20.87 C225.32 -21.08, 305.98 -5.56, 332.73 -2.57" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(821.1732677211501 387.78532718011866) rotate(0 165.60605119295678 -10.532439672289911)"><path d="M303.03 1.65 C313.45 -0.15, 321.31 -1.95, 333.07 -0.83 M302.56 3.49 C314.56 0.76, 325.37 -0.14, 332.95 -2.19" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(821.1732677211501 387.78532718011866) rotate(0 165.60605119295678 -10.532439672289911)"><path d="M305.99 -18.66 C315.4 -14.43, 322.39 -10.2, 333.07 -0.83 M305.52 -16.82 C316.45 -11.66, 326.11 -4.66, 332.95 -2.19" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask id="mask-OpknWNg9Sznc-QhpTZdV_"><rect x="0" y="0" fill="#fff" width="1252.7207974659775" height="507.7607488336844"></rect><rect x="900.3920654031149" y="342.80990552655294" fill="#000" width="178.79989624023438" height="50" opacity="1"></rect></mask><g transform="translate(900.3920654031153 342.80990552655294) rotate(0 86.38725351099174 34.44298198127581)"><text x="89.39994812011719" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">CRI sandbox and </text><text x="89.39994812011719" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">container info</text></g><g stroke-linecap="round" transform="translate(1161.141470437926 372.24409573119283) rotate(0 102 27)"><path d="M13.5 0 M13.5 0 C50.46 -0.21, 86.78 3.4, 190.5 0 M13.5 0 C81.61 1.33, 151.09 0.74, 190.5 0 M190.5 0 C197.86 1.71, 204.46 5.27, 204 13.5 M190.5 0 C197.57 0.4, 206.2 4.97, 204 13.5 M204 13.5 C202.01 21.89, 205.26 33.64, 204 40.5 M204 13.5 C203.9 19.29, 203.6 26.59, 204 40.5 M204 40.5 C204.23 51.12, 199.7 54.11, 190.5 54 M204 40.5 C202.24 49.47, 199.09 51.94, 190.5 54 M190.5 54 C132.29 54.49, 77.26 53.11, 13.5 54 M190.5 54 C135.08 52.76, 79.44 52.08, 13.5 54 M13.5 54 C5.93 53.14, -1.7 49.58, 0 40.5 M13.5 54 C4.95 55.64, -1.38 48.75, 0 40.5 M0 40.5 C1.55 31.26, -0.13 21.91, 0 13.5 M0 40.5 C0.68 30.63, -0.54 22.42, 0 13.5 M0 13.5 C-0.51 2.69, 6.08 1.49, 13.5 0 M0 13.5 C-1.37 6.55, 5.26 0.64, 13.5 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(1203.5915131625354 386.74409573119283) rotate(0 59.549957275390625 12.5)"><text x="59.549957275390625" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">NRI convert</text></g><g mask="url(#mask-DM-zvSyh5rcHa85d0V_b5)" stroke-linecap="round"><g transform="translate(1152.8566925680036 413.1035137103745) rotate(0 -167.40297087769568 9.746459734911696)"><path d="M1.04 1.07 C-27.2 4, -114.42 17.05, -170.57 17.78 C-226.72 18.52, -308.42 7.65, -335.85 5.49 M0.13 0.59 C-28.12 3.62, -115.66 17.8, -171.26 18.84 C-226.86 19.87, -306.16 9, -333.45 6.78" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(1152.8566925680036 413.1035137103745) rotate(0 -167.40297087769568 9.746459734911696)"><path d="M-302.57 0.3 C-311.9 -0.34, -316.44 2.36, -334.69 5.15 M-305.2 -0.35 C-314.6 2.61, -325.96 4.98, -333.92 5.82" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(1152.8566925680036 413.1035137103745) rotate(0 -167.40297087769568 9.746459734911696)"><path d="M-304.68 20.71 C-313.6 15.84, -317.72 14.31, -334.69 5.15 M-307.31 20.06 C-315.94 15.14, -326.49 9.65, -333.92 5.82" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask id="mask-DM-zvSyh5rcHa85d0V_b5"><rect x="0" y="0" fill="#fff" width="1587.7603540329699" height="531.8099055265529"></rect><rect x="892.3920654031153" y="406.80990552655294" fill="#000" width="178.79989624023438" height="50" opacity="1"></rect></mask><g transform="translate(892.3920654031149 406.80990552655294) rotate(0 93.06165628719282 16.040067918733257)"><text x="89.39994812011719" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">NRI sandbox and </text><text x="89.39994812011719" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">container info</text></g><g mask="url(#mask-Z3OCoE3xkKxAvM-4ZpGdR)" stroke-linecap="round"><g transform="translate(536.975886201901 438.4405519618804) rotate(0 -1.3703625478365211 71.73214034558487)"><path d="M0.97 -0.29 C0.36 23.68, -1.8 119.64, -2.47 144.02 M0.01 -1.48 C-0.9 22.56, -3.58 120.69, -3.71 144.95" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(536.975886201901 438.4405519618804) rotate(0 -1.3703625478365211 71.73214034558487)"><path d="M-13.93 115.64 C-11.47 125.83, -4.36 138.54, -5.12 145.23 M-13.89 117.28 C-11.28 123.52, -9.2 128.86, -4.47 144.93" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(536.975886201901 438.4405519618804) rotate(0 -1.3703625478365211 71.73214034558487)"><path d="M6.59 116.02 C0.93 126.22, -0.07 138.78, -5.12 145.23 M6.63 117.66 C4.83 123.73, 2.49 129, -4.47 144.93" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask id="mask-Z3OCoE3xkKxAvM-4ZpGdR"><rect x="0" y="0" fill="#fff" width="638.8837341569981" height="683.6130377275504"></rect><rect x="429.47204309593417" y="486.02679484471537" fill="#000" width="213.09983825683594" height="50" opacity="1"></rect></mask><g transform="translate(429.4720430959346 486.02679484471537) rotate(0 106.13348055813003 24.145897462749872)"><text x="106.54991912841797" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">call CreateContainer </text><text x="106.54991912841797" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">for all manager plugin</text></g><g stroke-linecap="round" transform="translate(425.72318012936944 586.2855617672712) rotate(0 192 28.5)"><path d="M14.25 0 M14.25 0 C105.24 1.89, 193.62 2.33, 369.75 0 M14.25 0 C90.39 1.37, 167.17 1.43, 369.75 0 M369.75 0 C379.45 0.11, 382.47 4.72, 384 14.25 M369.75 0 C378.84 -2.06, 384.41 5.71, 384 14.25 M384 14.25 C382.93 21.99, 383.98 33.3, 384 42.75 M384 14.25 C383.72 20.09, 384.71 26.78, 384 42.75 M384 42.75 C382.3 52.33, 379.64 58.42, 369.75 57 M384 42.75 C382.62 51.5, 379.62 58.64, 369.75 57 M369.75 57 C294.42 55.21, 219.59 57.43, 14.25 57 M369.75 57 C250.76 56.07, 131.97 55.14, 14.25 57 M14.25 57 C6.33 58.49, -1.19 54.04, 0 42.75 M14.25 57 C5.51 57.64, -0.07 53.45, 0 42.75 M0 42.75 C-0.12 34.56, 1.64 25.73, 0 14.25 M0 42.75 C-0.34 37.25, -0.92 29.6, 0 14.25 M0 14.25 C0.31 6.49, 5.54 -0.15, 14.25 0 M0 14.25 C-0.33 6.35, 4.46 -0.68, 14.25 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(592.613210036596 602.2855617672712) rotate(0 25.109970092773438 12.5)"><text x="25.109970092773438" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">plugin</text></g><g mask="url(#mask-PDz157tTqh314uxFblmw3)" stroke-linecap="round"><g transform="translate(633.5577914951557 645.6455478774196) rotate(0 -75.62674011992544 55.1054164023335)"><path d="M0.17 -1.02 C-24.83 8.49, -145.64 37.52, -150.25 56.23 C-154.86 74.94, -48.06 102.04, -27.49 111.23 M-1.2 1.06 C-26.35 10.81, -146.73 39.47, -151.28 57.52 C-155.84 75.56, -49.32 100.71, -28.53 109.34" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(633.5577914951557 645.6455478774196) rotate(0 -75.62674011992544 55.1054164023335)"><path d="M-60.2 110.32 C-50.9 109.32, -40.29 110.08, -26.91 109.53 M-58.68 109.7 C-49.16 110.63, -40.11 108.84, -28.35 108.94" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(633.5577914951557 645.6455478774196) rotate(0 -75.62674011992544 55.1054164023335)"><path d="M-54 90.76 C-46.57 95.25, -37.7 101.48, -26.91 109.53 M-52.48 90.14 C-45 97.04, -37.85 101.22, -28.35 108.94" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask id="mask-PDz157tTqh314uxFblmw3"><rect x="0" y="0" fill="#fff" width="883.6396737870232" height="856.0236866925115"></rect><rect x="375.01600167154993" y="677.35602210468" fill="#000" width="216.91981506347656" height="50" opacity="1"></rect></mask><g transform="translate(375.01600167154993 677.35602210468) rotate(0 182.9150497036801 23.394942175073083)"><text x="108.45990753173828" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">CreateContainerReque</text><text x="108.45990753173828" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">st</text></g><g stroke-linecap="round" transform="translate(432.55646260007256 764.4522030026228) rotate(0 165.5 29.5)"><path d="M3.93 3.41 C3.93 3.41, 3.93 3.41, 3.93 3.41 M3.93 3.41 C3.93 3.41, 3.93 3.41, 3.93 3.41 M3.01 10.57 C4.87 7.44, 7.67 6.29, 10.23 2.26 M3.01 10.57 C4.5 8.33, 6.69 6.55, 10.23 2.26 M2.75 16.96 C7.13 12.33, 8.02 10.43, 16.52 1.11 M2.75 16.96 C7.65 10.99, 12.35 5.52, 16.52 1.11 M2.49 23.36 C8.83 13.95, 18.59 5.31, 21.51 1.47 M2.49 23.36 C5.7 19.53, 10.62 14.33, 21.51 1.47 M2.88 29 C9.34 20.31, 16.94 12.23, 27.15 1.08 M2.88 29 C12.81 19.16, 21.21 8.73, 27.15 1.08 M3.27 34.65 C11.83 27.9, 16.46 19.33, 32.14 1.44 M3.27 34.65 C14.66 21.56, 25.48 10.44, 32.14 1.44 M3.01 41.04 C17.3 26.69, 30.04 12.19, 37.78 1.05 M3.01 41.04 C12.19 31.74, 18.96 23.19, 37.78 1.05 M3.41 46.69 C18.18 31.54, 30.85 13.16, 42.77 1.41 M3.41 46.69 C13.95 35.8, 23.38 24.81, 42.77 1.41 M3.15 53.09 C21.94 34.4, 38.44 13.93, 48.41 1.01 M3.15 53.09 C14.09 40.23, 26.35 26.23, 48.41 1.01 M6.17 55.71 C15.06 44.25, 27.43 31.38, 53.4 1.37 M6.17 55.71 C20.84 39.66, 34.89 21.29, 53.4 1.37 M8.53 59.09 C18.38 45.84, 28.49 34.51, 58.39 1.73 M8.53 59.09 C18.87 46.72, 29.48 35.39, 58.39 1.73 M12.86 60.2 C34.46 36.9, 54.41 13.1, 64.03 1.34 M12.86 60.2 C29.74 40.96, 47.49 19.58, 64.03 1.34 M18.5 59.81 C31.91 44.43, 49.28 26.85, 69.02 1.7 M18.5 59.81 C32.14 45.95, 43.11 30.76, 69.02 1.7 M23.49 60.17 C43.56 36.55, 60.38 16.41, 74.66 1.3 M23.49 60.17 C35.58 46.12, 48.61 32.16, 74.66 1.3 M28.48 60.53 C49.93 38.95, 65.57 16.54, 79.65 1.66 M28.48 60.53 C48.15 38.62, 65.77 16.32, 79.65 1.66 M34.12 60.13 C44.86 45.81, 54.45 33.02, 85.29 1.27 M34.12 60.13 C46.58 46.09, 60.39 29.89, 85.29 1.27 M39.11 60.49 C57.08 42.54, 69.69 22.58, 90.28 1.63 M39.11 60.49 C48.62 48.84, 59.76 36.4, 90.28 1.63 M44.75 60.1 C56.27 44.25, 69.99 33.19, 95.92 1.23 M44.75 60.1 C61.74 42.18, 77.28 22.74, 95.92 1.23 M49.74 60.46 C65.23 41.4, 80.46 23.15, 100.91 1.59 M49.74 60.46 C69.02 38.29, 86.77 17.26, 100.91 1.59 M55.38 60.07 C73.82 35.19, 97.44 14.77, 106.55 1.2 M55.38 60.07 C71.15 42.42, 85.84 22.99, 106.55 1.2 M60.37 60.43 C75.36 43.66, 89.71 24.55, 111.54 1.56 M60.37 60.43 C77.71 41.06, 93.93 22.15, 111.54 1.56 M66.01 60.03 C77.99 44.87, 90.65 30.46, 117.18 1.16 M66.01 60.03 C77.44 47.75, 88.42 33.67, 117.18 1.16 M71 60.39 C89.06 39.25, 106.46 17.4, 122.17 1.52 M71 60.39 C89.82 37.4, 108.21 15.06, 122.17 1.52 M76.64 60 C97.81 39.83, 114.68 18.71, 127.81 1.13 M76.64 60 C95.65 38.28, 113.21 17.05, 127.81 1.13 M81.63 60.36 C91.39 46.61, 103.25 34.78, 132.8 1.49 M81.63 60.36 C101.84 36.53, 121.59 14.23, 132.8 1.49 M87.27 59.96 C99.04 48.55, 109.28 34.92, 138.44 1.09 M87.27 59.96 C108.41 37.11, 128.33 13.29, 138.44 1.09 M92.26 60.32 C105.67 44.43, 115.95 32.95, 143.43 1.46 M92.26 60.32 C108.55 41.87, 124.43 23.82, 143.43 1.46 M97.9 59.93 C110.18 46.1, 119.48 34.68, 148.42 1.82 M97.9 59.93 C110.6 44.76, 123.43 30.19, 148.42 1.82 M102.89 60.29 C114.82 46.71, 125.82 33.25, 154.06 1.42 M102.89 60.29 C117.47 43.49, 132.24 26.56, 154.06 1.42 M108.53 59.89 C120.02 43.66, 131.7 32.95, 159.05 1.78 M108.53 59.89 C120.37 47.28, 131.41 32.61, 159.05 1.78 M113.52 60.25 C134.24 37.43, 153.47 13.7, 164.69 1.39 M113.52 60.25 C127.05 43.53, 141.29 28.74, 164.69 1.39 M118.5 60.61 C130.6 48.64, 140.85 33.44, 169.68 1.75 M118.5 60.61 C134.87 41.43, 152.58 21.86, 169.68 1.75 M124.15 60.22 C144.1 38.87, 162.06 16.4, 175.32 1.35 M124.15 60.22 C138.69 43.4, 150.95 27.65, 175.32 1.35 M129.13 60.58 C146.33 44.13, 161.49 24.61, 180.31 1.71 M129.13 60.58 C143.54 43.44, 158.03 28.43, 180.31 1.71 M134.78 60.18 C147.43 45.33, 164.46 30.14, 185.95 1.32 M134.78 60.18 C149.65 42.89, 164.57 26.64, 185.95 1.32 M139.76 60.54 C152.24 47.76, 164.28 32.76, 190.94 1.68 M139.76 60.54 C156.45 40.69, 173.61 21.71, 190.94 1.68 M145.41 60.15 C164.24 37.31, 185.26 16.58, 196.58 1.28 M145.41 60.15 C165 40.08, 181.41 19.09, 196.58 1.28 M150.39 60.51 C170.67 38.51, 191.16 14.74, 201.57 1.64 M150.39 60.51 C162.44 46.27, 172.4 32.13, 201.57 1.64 M156.04 60.12 C177.76 36.22, 195.93 11.88, 207.21 1.25 M156.04 60.12 C173.87 41.17, 190.68 22, 207.21 1.25 M161.02 60.48 C171.36 47.54, 183.74 34.8, 212.2 1.61 M161.02 60.48 C176.92 43.64, 191.67 26.78, 212.2 1.61 M166.67 60.08 C184.21 42.59, 199.7 21.98, 217.84 1.21 M166.67 60.08 C181.8 42.88, 196.37 26.76, 217.84 1.21 M171.65 60.44 C189.22 38.55, 205.63 19.44, 222.83 1.57 M171.65 60.44 C189.23 40.47, 205.19 22.05, 222.83 1.57 M177.3 60.05 C193.61 38.82, 210.15 21.15, 228.47 1.18 M177.3 60.05 C195.33 39.27, 211.59 19.9, 228.47 1.18 M182.28 60.41 C195.83 44.22, 207.18 31.12, 233.46 1.54 M182.28 60.41 C196.4 45.45, 209.66 29.24, 233.46 1.54 M187.93 60.01 C200.45 44.96, 213.29 31.49, 238.44 1.9 M187.93 60.01 C206.46 36.81, 226.51 14.06, 238.44 1.9 M192.91 60.37 C207.93 43.38, 220.25 27.7, 244.09 1.5 M192.91 60.37 C212.24 40.36, 230.02 18.78, 244.09 1.5 M198.56 59.98 C210.55 44.58, 225.66 28.8, 249.07 1.86 M198.56 59.98 C217.64 38.37, 236.65 15.17, 249.07 1.86 M203.54 60.34 C219.91 41.67, 233.31 25.73, 254.72 1.47 M203.54 60.34 C214.61 47.01, 225.58 36.01, 254.72 1.47 M209.19 59.94 C225.91 39.27, 243.63 21.42, 259.7 1.83 M209.19 59.94 C223.75 43.95, 239.39 26.8, 259.7 1.83 M214.17 60.3 C231.06 41.96, 247.21 23.36, 265.35 1.44 M214.17 60.3 C228.86 43.14, 243.27 26.97, 265.35 1.44 M219.16 60.66 C235.85 42.44, 249.7 27.22, 270.33 1.8 M219.16 60.66 C235.34 43.58, 250.39 24.16, 270.33 1.8 M224.8 60.27 C242.09 42.49, 255.61 26.44, 275.98 1.4 M224.8 60.27 C236.84 47.15, 248.64 33.06, 275.98 1.4 M229.79 60.63 C245.11 41.3, 263.74 25.46, 280.96 1.76 M229.79 60.63 C241.06 46.34, 252.66 35.24, 280.96 1.76 M235.44 60.23 C250.61 42.63, 264.55 26.28, 286.61 1.37 M235.44 60.23 C254.96 38.22, 273.68 14.92, 286.61 1.37 M240.42 60.59 C257.27 42.88, 273.32 24.38, 291.59 1.73 M240.42 60.59 C253.21 43.75, 267.97 28.04, 291.59 1.73 M246.07 60.2 C263.17 42.15, 276.43 26.95, 297.24 1.33 M246.07 60.2 C257.3 48.15, 268.18 34.54, 297.24 1.33 M251.05 60.56 C266.66 43.62, 281.04 24.38, 302.22 1.69 M251.05 60.56 C267.93 40.66, 283.38 23.39, 302.22 1.69 M256.7 60.17 C269.49 46.75, 278.02 34.57, 307.87 1.3 M256.7 60.17 C272.15 41.98, 286.5 24.1, 307.87 1.3 M261.68 60.53 C278 42.64, 290.23 26.73, 312.85 1.66 M261.68 60.53 C273.99 46.23, 284.71 32.1, 312.85 1.66 M267.33 60.13 C287 36.9, 303.73 18.85, 318.5 1.26 M267.33 60.13 C285.14 39.42, 304.41 17.14, 318.5 1.26 M272.31 60.49 C288.88 40.1, 305.45 24.5, 322.83 2.38 M272.31 60.49 C288.27 42.7, 304.26 24.47, 322.83 2.38 M277.96 60.1 C287.82 46.17, 300.48 34.7, 326.5 4.25 M277.96 60.1 C288.65 47.89, 299.1 35.43, 326.5 4.25 M282.94 60.46 C295.8 45.77, 306.18 34.41, 330.18 6.12 M282.94 60.46 C295.37 45.1, 307.27 30.92, 330.18 6.12 M288.59 60.06 C296.97 47.33, 309.32 36.64, 331.89 10.25 M288.59 60.06 C300.43 47.66, 311.2 35.38, 331.89 10.25 M293.57 60.42 C308.95 45.24, 324.36 26.49, 332.28 15.89 M293.57 60.42 C303.33 50.21, 311 39.74, 332.28 15.89 M299.22 60.03 C309.55 49.63, 318.22 37.92, 332.02 22.29 M299.22 60.03 C310.25 47.95, 320.54 37.15, 332.02 22.29 M304.2 60.39 C310.84 52.47, 314.67 46.09, 332.41 27.93 M304.2 60.39 C315.95 47.46, 325.98 34.56, 332.41 27.93 M309.19 60.75 C315.42 52.27, 321.38 46.7, 332.81 33.58 M309.19 60.75 C314.57 55.64, 318.41 49.08, 332.81 33.58 M314.83 60.35 C322.32 52.8, 329.07 44.85, 332.55 39.98 M314.83 60.35 C320.65 54.47, 325.59 48.7, 332.55 39.98 M319.16 61.47 C323.27 55.73, 327.99 50.67, 330.97 47.88 M319.16 61.47 C321.57 57.23, 325.19 53.95, 330.97 47.88" stroke="#ffc9c9" stroke-width="0.5" fill="none"></path><path d="M14.75 0 M14.75 0 C117.82 0.46, 217.51 2.11, 316.25 0 M14.75 0 C130.76 0.41, 246.62 0.76, 316.25 0 M316.25 0 C325.52 0.8, 330.06 3.99, 331 14.75 M316.25 0 C324.6 1.72, 331.33 4.59, 331 14.75 M331 14.75 C331.99 24.69, 330.17 33.06, 331 44.25 M331 14.75 C330.85 25.67, 330.48 37.52, 331 44.25 M331 44.25 C329.17 54.02, 325.72 58.8, 316.25 59 M331 44.25 C333.28 52.05, 326.05 60.93, 316.25 59 M316.25 59 C238.67 56.44, 160.85 58.65, 14.75 59 M316.25 59 C225.32 57.95, 134.66 56.59, 14.75 59 M14.75 59 C6.89 58.5, 1.42 55.57, 0 44.25 M14.75 59 C4.59 61.18, 0.43 54.2, 0 44.25 M0 44.25 C1.79 34.66, -0.19 24.57, 0 14.75 M0 44.25 C0.7 37.08, -0.96 28.76, 0 14.75 M0 14.75 C1.73 3.83, 5.74 0.21, 14.75 0 M0 14.75 C0.56 5.6, 5.93 -1.74, 14.75 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(556.7865041039788 781.4522030026228) rotate(0 41.26995849609375 12.5)"><text x="41.26995849609375" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">nri plugin</text></g><g mask="url(#mask-T8TvRDaxAzGXakclfcKun)" stroke-linecap="round"><g transform="translate(696.8092679679364 754.85602210468) rotate(0 33.78493068825151 -55.79026121423067)"><path d="M0.12 0.06 C11.19 -7.6, 65.03 -26.4, 66.58 -45.02 C68.14 -63.63, 18.9 -100.8, 9.45 -111.64 M-1.28 -0.95 C10.16 -8.49, 67.17 -25.16, 68.81 -43.49 C70.45 -61.81, 18.33 -99.41, 8.57 -110.89" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(696.8092679679364 754.85602210468) rotate(0 33.78493068825151 -55.79026121423067)"><path d="M36.39 -100.94 C27.53 -103.31, 17.14 -106.18, 9.4 -110.74 M35.77 -99.28 C31.46 -102.55, 25.1 -104.08, 8.3 -111.53" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(696.8092679679364 754.85602210468) rotate(0 33.78493068825151 -55.79026121423067)"><path d="M22.52 -85.81 C17.86 -92.91, 11.78 -100.49, 9.4 -110.74 M21.91 -84.15 C20.35 -90.41, 16.78 -94.97, 8.3 -111.53" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask id="mask-T8TvRDaxAzGXakclfcKun"><rect x="0" y="0" fill="#fff" width="864.3092679679369" height="965.4264824420889"></rect><rect x="656.559367150066" y="684.85602210468" fill="#000" width="215.4998016357422" height="50" opacity="1"></rect></mask><g transform="translate(656.559367150066 684.85602210468) rotate(0 74.03483150612215 14.209738785769332)"><text x="107.7499008178711" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">CreateContainerRespo</text><text x="107.7499008178711" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">nse</text></g><g stroke-linecap="round" transform="translate(430.79679440932546 371.4405519618804) rotate(0 190 33)"><path d="M16.5 0 M16.5 0 C107.26 -2.18, 198.74 -2.45, 363.5 0 M16.5 0 C133.06 0.04, 250.98 0.52, 363.5 0 M363.5 0 C374.9 -0.76, 380.65 5.81, 380 16.5 M363.5 0 C376.55 0.63, 381.91 5.15, 380 16.5 M380 16.5 C381.71 25.36, 377.93 29.38, 380 49.5 M380 16.5 C380.09 29.55, 380.12 42.56, 380 49.5 M380 49.5 C381.15 61.84, 373.99 65.79, 363.5 66 M380 49.5 C379.3 58.99, 375.88 67.1, 363.5 66 M363.5 66 C242.62 68.58, 121.81 66.89, 16.5 66 M363.5 66 C280.77 63.54, 197.75 63.24, 16.5 66 M16.5 66 C5.65 66.94, 0.16 61.93, 0 49.5 M16.5 66 C7.1 67.4, -1.63 61.09, 0 49.5 M0 49.5 C-1.7 36.66, -1.17 25.2, 0 16.5 M0 49.5 C-0.27 37.92, -0.19 26.9, 0 16.5 M0 16.5 C1.26 7.19, 6.17 -1.1, 16.5 0 M0 16.5 C-1.61 4.56, 5.57 -2.1, 16.5 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(555.2268404908684 391.9405519618804) rotate(0 65.56995391845703 12.5)"><text x="65.56995391845703" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">NRI Adaption</text></g><g mask="url(#mask-YXhXcLar9N4V9Y240KE-m)" stroke-linecap="round"><g transform="translate(643.9981699359926 376.67945901843814) rotate(0 2.2108034655113897 -71.38695292503371)"><path d="M1.08 -0.08 C1.35 -23.68, 1.67 -117.57, 1.91 -141.14 M0.19 -1.17 C0.82 -25.12, 3.97 -118.97, 4.23 -142.7" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(643.9981699359926 376.67945901843814) rotate(0 2.2108034655113897 -71.38695292503371)"><path d="M13.69 -115.53 C12.38 -123.66, 8.62 -131.13, 3.63 -142.91 M14.08 -114.66 C10.65 -123.33, 8.69 -130.45, 4.25 -142.47" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(643.9981699359926 376.67945901843814) rotate(0 2.2108034655113897 -71.38695292503371)"><path d="M-6.83 -116.01 C-1.45 -124.17, 1.47 -131.48, 3.63 -142.91 M-6.44 -115.14 C-3.68 -123.64, 0.54 -130.6, 4.25 -142.47" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask id="mask-YXhXcLar9N4V9Y240KE-m"><rect x="0" y="0" fill="#fff" width="746.6648366026595" height="618.9016676772917"></rect><rect x="601.1515411111232" y="280.56835468901136" fill="#000" width="88.35992431640625" height="50" opacity="1"></rect></mask><g transform="translate(601.1515411111227 280.56835468901136) rotate(0 45.05743229038103 24.72415140439307)"><text x="44.179962158203125" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">update </text><text x="44.179962158203125" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">container</text></g><g mask="url(#mask-HmCIsBeuFWHlVOsQt6qUi)" stroke-linecap="round"><g transform="translate(479.55375261828453 228.2350417007292) rotate(0 -0.03879652257978705 71.16499273528325)"><path d="M-0.75 -0.65 C-0.74 23.18, -0.56 119.18, -0.43 142.98 M1.06 1.63 C0.99 25.12, -0.91 117.62, -1.14 141.26" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(479.55375261828453 228.2350417007292) rotate(0 -0.03879652257978705 71.16499273528325)"><path d="M-12.04 113.67 C-7.1 121.89, -5.9 126.16, -0.71 142.21 M-11.79 113.65 C-7.22 122.56, -5.56 130.57, -0.98 141.54" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(479.55375261828453 228.2350417007292) rotate(0 -0.03879652257978705 71.16499273528325)"><path d="M8.48 113.99 C8.04 122, 3.87 126.18, -0.71 142.21 M8.73 113.97 C7.07 122.64, 2.49 130.56, -0.98 141.54" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask id="mask-HmCIsBeuFWHlVOsQt6qUi"><rect x="0" y="0" fill="#fff" width="580.4370435285393" height="470.4405519618804"></rect><rect x="389.7121590430397" y="274.33779683130433" fill="#000" width="178.79989624023438" height="50" opacity="1"></rect></mask><g transform="translate(389.7121590430402 274.33779683130433) rotate(0 89.8027970526648 25.062237604708116)"><text x="89.39994812011719" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">CRI sandbox and </text><text x="89.39994812011719" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">container info</text></g><g mask="url(#mask-5TzJvxa8XXpouNS8y8_c-)" stroke-linecap="round"><g transform="translate(768.44266863391 366.0127923517739) rotate(0 0.9788965578827629 -66.0210469606036)"><path d="M1.2 1.16 C1.28 -21.03, 1.74 -110.9, 1.69 -133.2 M0.36 0.72 C0.18 -21.24, 0.21 -109.03, 0.7 -131.6" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(768.44266863391 366.0127923517739) rotate(0 0.9788965578827629 -66.0210469606036)"><path d="M12.63 -103.45 C6.95 -115.95, 3.89 -124.27, 0.26 -132.06 M10.3 -103.96 C9.69 -109.06, 6 -115.37, 0.92 -131.98" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(768.44266863391 366.0127923517739) rotate(0 0.9788965578827629 -66.0210469606036)"><path d="M-7.89 -103.65 C-5.41 -116.27, -0.32 -124.51, 0.26 -132.06 M-10.22 -104.15 C-6.63 -109.01, -6.11 -115.29, 0.92 -131.98" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask id="mask-5TzJvxa8XXpouNS8y8_c-"><rect x="0" y="0" fill="#fff" width="870.220419284951" height="599.3461256851097"></rect><rect x="694.6616068256412" y="274.346125685106" fill="#000" width="149.33987426757812" height="50" opacity="1"></rect></mask><g transform="translate(694.6616068256412 274.346125685106) rotate(0 74.7599583661513 25.64561970606428)"><text x="74.66993713378906" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">merged adjust </text><text x="74.66993713378906" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">container info</text></g><g mask="url(#mask-jMAkxvunSk8Kf09Z2hkZx)" stroke-linecap="round"><g transform="translate(721.1107163682634 586.5516590009674) rotate(0 -0.4676759824310466 -73.98739256346926)"><path d="M0.88 0.49 C0.78 -23.92, -0.76 -122.09, -1.39 -146.74 M-0.12 -0.3 C-0.21 -25.02, -1.18 -124.17, -1.81 -148.47" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(721.1107163682634 586.5516590009674) rotate(0 -0.4676759824310466 -73.98739256346926)"><path d="M9.73 -118.8 C7.76 -127.75, 4.1 -133.68, -0.12 -146.48 M8.59 -119.48 C7.06 -127.65, 4.32 -134.1, -1.88 -148.45" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(721.1107163682634 586.5516590009674) rotate(0 -0.4676759824310466 -73.98739256346926)"><path d="M-10.79 -118.46 C-8 -127.64, -6.9 -133.65, -0.12 -146.48 M-11.93 -119.14 C-8.6 -127.32, -6.47 -133.85, -1.88 -148.45" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask id="mask-jMAkxvunSk8Kf09Z2hkZx"><rect x="0" y="0" fill="#fff" width="823.4897365127028" height="834.0531404997237"></rect><rect x="678.6812542086413" y="500.3009182515898" fill="#000" width="82.47990417480469" height="25" opacity="1"></rect></mask><g transform="translate(678.6812542086409 500.3009182515898) rotate(0 41.96178617719124 12.263348185908399)"><text x="41.239952087402344" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">response</text></g><g stroke-linecap="round" transform="translate(10 367.86430358667076) rotate(0 85.5 31)"><path d="M15.5 0 M15.5 0 C57.83 0.85, 96.88 -2.13, 155.5 0 M15.5 0 C57.67 1.09, 100.8 1.67, 155.5 0 M155.5 0 C167.06 -0.85, 169.99 3.5, 171 15.5 M155.5 0 C164.13 2.19, 171.84 3.13, 171 15.5 M171 15.5 C169.13 22.04, 169.73 29.09, 171 46.5 M171 15.5 C170.15 25.35, 170.82 33.88, 171 46.5 M171 46.5 C172.7 55.05, 165.09 61.99, 155.5 62 M171 46.5 C170.91 58.17, 163.57 64.24, 155.5 62 M155.5 62 C128.2 62.6, 97.04 63.71, 15.5 62 M155.5 62 C121.46 60.43, 87.21 60.88, 15.5 62 M15.5 62 C5.72 62.14, -1.08 58.19, 0 46.5 M15.5 62 C5.22 60.96, 2.08 58.37, 0 46.5 M0 46.5 C-0.65 39.75, -0.43 33.15, 0 15.5 M0 46.5 C0.01 39.25, 0.5 30.59, 0 15.5 M0 15.5 C-1.46 3.88, 6.17 0.77, 15.5 0 M0 15.5 C-0.52 6.56, 7.18 -0.37, 15.5 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(50.18004608154297 386.36430358667076) rotate(0 45.31995391845703 12.5)"><text x="45.31995391845703" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">nri result</text></g><g mask="url(#mask-4RCmohsVVI9SO6fB5yUmi)" stroke-linecap="round"><g transform="translate(424.62461331906 391.1155846942793) rotate(0 -120.74312644979341 0.3665053932409137)"><path d="M0.09 -0.65 C-40.09 -0.84, -200.64 -0.98, -241.05 -0.83 M-1.33 1.63 C-41.59 1.61, -201.22 1.25, -241.57 0.72" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(424.62461331906 391.1155846942793) rotate(0 -120.74312644979341 0.3665053932409137)"><path d="M-214.39 -7.99 C-221.69 -7.22, -228.38 -1.96, -242.95 0.22 M-213.56 -8.86 C-223.07 -6.99, -231.96 -3, -240.97 0.91" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(424.62461331906 391.1155846942793) rotate(0 -120.74312644979341 0.3665053932409137)"><path d="M-214.53 12.53 C-221.85 7.16, -228.49 6.28, -242.95 0.22 M-213.7 11.66 C-223.08 6.81, -231.93 4.09, -240.97 0.91" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask id="mask-4RCmohsVVI9SO6fB5yUmi"><rect x="0" y="0" fill="#fff" width="766.492738359477" height="491.97315376004826"></rect><rect x="215.41063594289426" y="378.1868001613948" fill="#000" width="176.55982971191406" height="25" opacity="1"></rect></mask><g transform="translate(215.4106359428945 378.1868001613948) rotate(0 88.47085092637212 13.295289926125406)"><text x="88.27991485595703" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">all plugin response</text></g><g mask="url(#mask-sEsNwMSpvJ2hI4B6_JuJm)" stroke-linecap="round"><g transform="translate(192.96021259129907 424.18105656420084) rotate(0 115.60485285280083 -0.1854056624488294)"><path d="M-0.48 0.13 C37.98 0.43, 192.9 0.79, 231.68 0.73 M1.48 -0.85 C39.72 -0.79, 192.5 -1.31, 230.91 -1.01" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(192.96021259129907 424.18105656420084) rotate(0 115.60485285280083 -0.1854056624488294)"><path d="M202.91 9.74 C214.72 6.84, 223.77 -0.01, 232.79 -1.12 M203.07 9.88 C209.13 7.46, 215.04 4.68, 231.71 -1.16" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(192.96021259129907 424.18105656420084) rotate(0 115.60485285280083 -0.1854056624488294)"><path d="M202.94 -10.78 C214.55 -5.88, 223.58 -4.94, 232.79 -1.12 M203.1 -10.64 C209.22 -8.66, 215.12 -7.05, 231.71 -1.16" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask id="mask-sEsNwMSpvJ2hI4B6_JuJm"><rect x="0" y="0" fill="#fff" width="524.3234338101422" height="524.1810565642008"></rect><rect x="233.9718860669318" y="399.18105656420084" fill="#000" width="149.33987426757812" height="50" opacity="1"></rect></mask><g transform="translate(233.97188606693135 399.18105656420084) rotate(0 74.59317937716833 24.81459433755117)"><text x="74.66993713378906" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">merged adjust </text><text x="74.66993713378906" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">container info</text></g><g stroke-linecap="round" transform="translate(982.57624395671 150.32484493454922) rotate(0 176 36)"><path d="M18 0 M18 0 C83.28 2.44, 145.92 0.68, 334 0 M18 0 C141.03 -1, 262.43 -1.98, 334 0 M334 0 C345.4 0.7, 352.55 6.51, 352 18 M334 0 C344.08 1.01, 350.83 6.08, 352 18 M352 18 C351.99 23.64, 350.23 33.77, 352 54 M352 18 C351.45 28.1, 351.62 38.81, 352 54 M352 54 C352.82 67.53, 347.83 71.88, 334 72 M352 54 C354.2 66.49, 347.43 72.94, 334 72 M334 72 C243.69 72.91, 156.94 71.37, 18 72 M334 72 C266.14 70.86, 197.87 70.81, 18 72 M18 72 C6.79 71.38, -1.63 64.67, 0 54 M18 72 C7.48 70.18, 2.01 65.47, 0 54 M0 54 C1.84 46.51, -1.95 39.49, 0 18 M0 54 C-0.89 45.11, -0.27 35.1, 0 18 M0 18 C-0.71 4.56, 6.93 1.23, 18 0 M0 18 C1.55 8.15, 4.77 0.62, 18 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(1083.7163120109092 173.82484493454922) rotate(0 74.85993194580078 12.5)"><text x="74.85993194580078" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">create excutor</text></g><g mask="url(#mask-vf0tIy_vFKkEe40SrHqli)" stroke-linecap="round"><g transform="translate(810.6143070708038 184.60105550989647) rotate(0 85.85473115192053 0.5034335892464696)"><path d="M0.81 -0.23 C29.11 0.09, 141.36 0.79, 169.82 0.96 M-0.23 -1.4 C28.43 -0.8, 143.38 1.85, 171.93 2.4" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(810.6143070708038 184.60105550989647) rotate(0 85.85473115192053 0.5034335892464696)"><path d="M143.15 10.16 C151.79 9.96, 160 6.01, 171.38 1.26 M143.5 12.25 C150.41 9.15, 160.09 6.06, 172.12 2.16" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(810.6143070708038 184.60105550989647) rotate(0 85.85473115192053 0.5034335892464696)"><path d="M143.59 -10.36 C151.94 -4.41, 160.02 -2.23, 171.38 1.26 M143.94 -8.27 C150.59 -5.73, 160.15 -3.17, 172.12 2.16" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask id="mask-vf0tIy_vFKkEe40SrHqli"><rect x="0" y="0" fill="#fff" width="1081.57624395671" height="285.74847354302347"></rect><rect x="839.5053249522339" y="172.6747645264595" fill="#000" width="113.17990112304688" height="25" opacity="1"></rect></mask><g transform="translate(839.5053249522334 172.6747645264595) rotate(0 56.96371327049064 12.429724572683426)"><text x="56.58995056152344" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">req->adjust</text></g><g mask="url(#mask-9dl0saM1Gcj0ZLfACPrTs)" stroke-linecap="round"><g transform="translate(1307.8133771964197 228.85359375618555) rotate(0 2.1264746066437965 72.7339411229691)"><path d="M0.14 -0.02 C0.86 24.25, 4.76 122.28, 5.5 146.54 M-1.24 -1.07 C-0.71 22.74, 4.16 120.02, 5 144.63" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1307.8133771964197 228.85359375618555) rotate(0 2.1264746066437965 72.7339411229691)"><path d="M-6.49 118.33 C-1.04 127.8, 1.86 140.65, 5.73 144.45 M-6.2 117.69 C-3.11 126, 0.78 135.77, 4.3 144.93" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1307.8133771964197 228.85359375618555) rotate(0 2.1264746066437965 72.7339411229691)"><path d="M14.02 117.45 C11.34 127.3, 6.11 140.49, 5.73 144.45 M14.3 116.81 C10.58 125.59, 7.65 135.65, 4.3 144.93" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask id="mask-9dl0saM1Gcj0ZLfACPrTs"><rect x="0" y="0" fill="#fff" width="1412.4528706321387" height="474.99763698134757"></rect><rect x="1243.6931825080292" y="276.92561536876656" fill="#000" width="132.8798828125" height="50" opacity="1"></rect></mask><g transform="translate(1243.6931825080292 276.92561536876656) rotate(0 66.24666929503405 24.66191951038809)"><text x="66.43994140625" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">adjust </text><text x="66.43994140625" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">container info</text></g><g mask="url(#mask-zRXmjeqsDL6tlclFuSErW)" stroke-linecap="round"><g transform="translate(1190.279496296159 370.358143545629) rotate(0 -4.540293284720292 -71.94612488596567)"><path d="M-0.24 -1.13 C-1.45 -25.2, -6.99 -121.11, -8.37 -144.79 M-1.82 0.9 C-3.01 -23.03, -7.66 -119.41, -8.84 -143.76" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1190.279496296159 370.358143545629) rotate(0 -4.540293284720292 -71.94612488596567)"><path d="M0.89 -114.58 C-1.16 -120.62, -0.9 -127.57, -10.76 -141.89 M1.94 -116.27 C-0.19 -122.86, -3.91 -132.14, -8.4 -144.01" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1190.279496296159 370.358143545629) rotate(0 -4.540293284720292 -71.94612488596567)"><path d="M-19.61 -113.59 C-17.42 -119.79, -12.91 -126.95, -10.76 -141.89 M-18.56 -115.28 C-14.88 -122.3, -12.78 -131.86, -8.4 -144.01" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask id="mask-zRXmjeqsDL6tlclFuSErW"><rect x="0" y="0" fill="#fff" width="1299.558483167598" height="614.1824400529313"></rect><rect x="1147.040042533291" y="273.44599529197785" fill="#000" width="77.19992065429688" height="50" opacity="1"></rect></mask><g transform="translate(1147.0400425332914 273.44599529197785) rotate(0 38.69916047814763 24.966023367685466)"><text x="38.59996032714844" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">adjust </text><text x="38.59996032714844" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">oci spec</text></g></svg> +\ No newline at end of file +diff --git a/docs/design/detailed/NRI/nri_RunPodSandbox.svg b/docs/design/detailed/NRI/nri_RunPodSandbox.svg +new file mode 100644 +index 00000000..c7b96968 +--- /dev/null ++++ b/docs/design/detailed/NRI/nri_RunPodSandbox.svg +@@ -0,0 +1,16 @@ ++<svg version="1.1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 890.3367625296412 937.7656935634695" width="890.3367625296412" height="937.7656935634695"> ++ <!-- svg-source:excalidraw --> ++ ++ <defs> ++ <style class="style-fonts"> ++ @font-face { ++ font-family: "Virgil"; ++ src: url("https://excalidraw.com/Virgil.woff2"); ++ } ++ @font-face { ++ font-family: "Cascadia"; ++ src: url("https://excalidraw.com/Cascadia.woff2"); ++ } ++ </style> ++ </defs> ++ <rect x="0" y="0" width="890.3367625296412" height="937.7656935634695" fill="#ffffff"></rect><g stroke-linecap="round" transform="translate(35.35858416316478 10) rotate(0 148.5 35)"><path d="M4.66 4.05 C4.66 4.05, 4.66 4.05, 4.66 4.05 M4.66 4.05 C4.66 4.05, 4.66 4.05, 4.66 4.05 M3.08 11.96 C4.55 9.92, 6.55 8.05, 11.61 2.15 M3.08 11.96 C4.3 9.7, 7.04 7.4, 11.61 2.15 M2.17 19.11 C8.79 11.78, 15.84 3.23, 17.91 1 M2.17 19.11 C7.95 13.38, 13.65 6.39, 17.91 1 M1.91 25.51 C7.63 20.77, 10.83 14.18, 23.56 0.6 M1.91 25.51 C8.05 19.27, 14.35 11.03, 23.56 0.6 M2.3 31.15 C6.59 23.09, 14.92 18.63, 28.54 0.96 M2.3 31.15 C7.69 24.59, 14.01 16.89, 28.54 0.96 M2.69 36.79 C10.93 29.86, 14.11 19.97, 34.19 0.57 M2.69 36.79 C11.28 27.1, 21.66 15.89, 34.19 0.57 M2.43 43.19 C10.7 35.32, 17.75 23.66, 39.17 0.93 M2.43 43.19 C10.11 33.63, 17.94 24.89, 39.17 0.93 M2.83 48.83 C16.46 35.42, 28.67 21.57, 44.82 0.53 M2.83 48.83 C15.84 33.19, 29.02 19.76, 44.82 0.53 M3.22 54.48 C17.28 36.9, 32.66 22.97, 49.8 0.89 M3.22 54.48 C13.55 41.64, 25.13 29.67, 49.8 0.89 M2.31 61.63 C15.14 47.17, 24.5 35.53, 55.45 0.5 M2.31 61.63 C22.22 38.5, 41.54 16.22, 55.45 0.5 M5.32 64.25 C25.84 38.03, 49.9 12.94, 60.43 0.86 M5.32 64.25 C15.77 51.46, 28.24 37.56, 60.43 0.86 M7.69 67.63 C24.07 47.36, 41.8 27.62, 66.08 0.46 M7.69 67.63 C30.73 43.26, 52.02 17.99, 66.08 0.46 M11.36 69.5 C27.2 54.71, 38.79 38.13, 71.06 0.82 M11.36 69.5 C34.2 43.39, 56.41 19.16, 71.06 0.82 M15.04 71.37 C39.29 45.9, 61.78 20.17, 76.71 0.43 M15.04 71.37 C30.59 55.18, 43.54 39.51, 76.71 0.43 M20.68 70.98 C43.26 46.74, 63.84 19.28, 81.69 0.79 M20.68 70.98 C36.65 53.86, 51.48 36.65, 81.69 0.79 M25.67 71.34 C50.58 45.74, 73.18 18.26, 87.34 0.39 M25.67 71.34 C40.88 53.6, 57.34 34.77, 87.34 0.39 M31.31 70.94 C43.22 56.12, 58.53 39.88, 92.32 0.75 M31.31 70.94 C50.33 50.04, 68.67 26.74, 92.32 0.75 M36.3 71.3 C48.53 55.09, 61.14 40.88, 97.97 0.36 M36.3 71.3 C49.07 56.21, 62.1 42.12, 97.97 0.36 M41.94 70.91 C67.45 43.05, 91.29 14.77, 102.95 0.72 M41.94 70.91 C62.12 47.89, 83.23 22.64, 102.95 0.72 M46.93 71.27 C63.43 52.29, 83.93 31.11, 107.94 1.08 M46.93 71.27 C63.25 54.36, 76.79 36.2, 107.94 1.08 M51.91 71.63 C75.91 43.51, 96.6 18.9, 113.58 0.69 M51.91 71.63 C66.6 54.7, 82.21 37.77, 113.58 0.69 M57.56 71.23 C82.79 45.46, 102.13 18.79, 118.57 1.05 M57.56 71.23 C80.91 45.18, 102.21 18.66, 118.57 1.05 M62.54 71.59 C75.49 54.63, 87.34 39.25, 124.21 0.65 M62.54 71.59 C77.73 54.54, 94.21 35.27, 124.21 0.65 M68.19 71.2 C89.25 49.69, 104.96 26.16, 129.2 1.01 M68.19 71.2 C79.7 57.34, 92.84 42.61, 129.2 1.01 M73.17 71.56 C87.19 52.85, 103.48 38.83, 134.84 0.62 M73.17 71.56 C93.55 49.77, 112.41 26.5, 134.84 0.62 M78.82 71.16 C97.39 48.5, 115.74 26.64, 139.83 0.98 M78.82 71.16 C101.74 44.78, 123.13 19.57, 139.83 0.98 M83.8 71.52 C106.34 41.8, 134.14 16.57, 145.47 0.58 M83.8 71.52 C102.77 50.13, 120.68 27, 145.47 0.58 M89.45 71.13 C107.39 50.87, 124.8 28.26, 150.46 0.94 M89.45 71.13 C110.03 48.04, 129.54 25.36, 150.46 0.94 M94.43 71.49 C109.04 53.46, 124.24 36.13, 156.1 0.55 M94.43 71.49 C108.13 56.54, 121.39 39.85, 156.1 0.55 M100.08 71.1 C121.62 45.98, 142.46 20.17, 161.09 0.91 M100.08 71.1 C122.53 43.79, 144.6 17.22, 161.09 0.91 M105.06 71.46 C130.24 46.93, 150.99 21.34, 166.73 0.51 M105.06 71.46 C127.89 45.42, 149.29 19.78, 166.73 0.51 M110.71 71.06 C122.66 54.91, 136.69 40.58, 171.72 0.87 M110.71 71.06 C134.88 42.82, 158.49 16.07, 171.72 0.87 M115.69 71.42 C129.73 57.45, 142.16 41.3, 177.36 0.48 M115.69 71.42 C140.98 43.89, 165 15.36, 177.36 0.48 M121.34 71.03 C137.07 52.41, 149.66 38.25, 182.35 0.84 M121.34 71.03 C140.67 48.97, 159.66 27.34, 182.35 0.84 M126.32 71.39 C141.01 54.82, 152.62 40.74, 187.99 0.44 M126.32 71.39 C141.85 53.06, 157.47 35.28, 187.99 0.44 M131.97 70.99 C146.25 54.82, 159.56 38.7, 192.98 0.8 M131.97 70.99 C149.39 50.87, 166.99 30.68, 192.98 0.8 M136.95 71.35 C150.98 52.1, 165.23 38.42, 197.97 1.16 M136.95 71.35 C151.14 55.95, 164.59 38.51, 197.97 1.16 M141.94 71.71 C166.75 44.22, 190.04 15.82, 203.61 0.77 M141.94 71.71 C158.38 51.67, 175.53 33.54, 203.61 0.77 M147.58 71.32 C161.91 56.8, 174.45 38.97, 208.6 1.13 M147.58 71.32 C167.23 48.44, 188.19 25.15, 208.6 1.13 M152.57 71.68 C176.56 45.71, 198.61 18.53, 214.24 0.74 M152.57 71.68 C169.92 51.61, 185.06 32.56, 214.24 0.74 M158.21 71.28 C178.44 51.46, 196.6 28.5, 219.23 1.1 M158.21 71.28 C175.39 51.03, 192.6 32.88, 219.23 1.1 M163.2 71.64 C178.78 53.53, 198.62 35.09, 224.87 0.7 M163.2 71.64 C181.01 50.93, 198.96 31.2, 224.87 0.7 M168.84 71.25 C183.54 56.01, 197.79 38.47, 229.86 1.06 M168.84 71.25 C188.78 47.7, 209.11 25.09, 229.86 1.06 M173.83 71.61 C196.59 44.15, 221.54 18.89, 235.5 0.67 M173.83 71.61 C197.26 47.23, 217.4 21.96, 235.5 0.67 M179.47 71.21 C203.58 44.82, 227.86 16.69, 240.49 1.03 M179.47 71.21 C193.67 54.28, 205.88 37.55, 240.49 1.03 M184.46 71.57 C210.38 42.87, 232.66 13.81, 246.13 0.63 M184.46 71.57 C205.83 48.71, 226.11 25.54, 246.13 0.63 M190.1 71.18 C202.54 55.96, 216.95 40.89, 251.12 0.99 M190.1 71.18 C208.88 51.09, 226.51 30.9, 251.12 0.99 M195.09 71.54 C216.12 50.11, 235.04 25.55, 256.76 0.6 M195.09 71.54 C213.18 50.81, 230.8 31.19, 256.76 0.6 M200.73 71.15 C221.59 45.4, 241.32 22.47, 261.75 0.96 M200.73 71.15 C221.51 47.4, 240.69 25.27, 261.75 0.96 M205.72 71.51 C225.48 46.33, 245.37 24.8, 267.39 0.56 M205.72 71.51 C227.34 46.6, 247.11 23.18, 267.39 0.56 M211.37 71.11 C227.51 52, 241.37 36.02, 272.38 0.92 M211.37 71.11 C228.01 53.13, 243.88 33.92, 272.38 0.92 M216.35 71.47 C231.54 53.4, 246.98 36.92, 278.02 0.53 M216.35 71.47 C239.06 43.24, 263.39 15.56, 278.02 0.53 M221.34 71.83 C239.2 51.66, 254.27 32.81, 283.01 0.89 M221.34 71.83 C244.52 47.52, 266.13 21.53, 283.01 0.89 M226.98 71.44 C241.48 53.18, 259.02 34.59, 286.68 2.76 M226.98 71.44 C249.56 45.88, 272.07 18.66, 286.68 2.76 M231.97 71.8 C250.82 50.31, 266.73 31.49, 291.67 3.12 M231.97 71.8 C244.82 56.39, 257.53 43.38, 291.67 3.12 M237.61 71.4 C256.28 48.47, 275.98 28.35, 294.03 6.5 M237.61 71.4 C253.95 53.48, 271.31 34.35, 294.03 6.5 M242.6 71.76 C260.31 52.46, 277.27 32.92, 296.4 9.88 M242.6 71.76 C258.01 53.75, 273.17 36.73, 296.4 9.88 M248.24 71.37 C263.97 54.26, 276.86 40.15, 296.13 16.27 M248.24 71.37 C263.42 55.43, 277.48 37.15, 296.13 16.27 M253.23 71.73 C268.1 56.64, 279.26 43.3, 296.53 21.92 M253.23 71.73 C263.48 60.75, 273.47 48.74, 296.53 21.92 M258.87 71.33 C269.79 56.92, 284.08 46.07, 296.27 28.31 M258.87 71.33 C267.14 60.63, 275.72 53.01, 296.27 28.31 M263.86 71.69 C273.85 60.18, 282.63 49.77, 296.66 33.96 M263.86 71.69 C276.54 57.56, 288.25 42.33, 296.66 33.96 M269.5 71.3 C278.71 62.25, 287.18 52.46, 296.4 40.36 M269.5 71.3 C275.8 62.25, 283.85 54.26, 296.4 40.36 M274.49 71.66 C282.75 63.39, 287.39 58.11, 296.8 46 M274.49 71.66 C279.75 66.72, 284.6 60.05, 296.8 46 M280.79 70.51 C285.68 66.32, 289.08 59.69, 297.19 51.64 M280.79 70.51 C286.69 63.45, 290.88 59.13, 297.19 51.64 M287.09 69.36 C290.52 66.54, 290.47 64.71, 297.58 57.28 M287.09 69.36 C290.3 65.34, 292.69 61.48, 297.58 57.28" stroke="#ffc9c9" stroke-width="0.5" fill="none"></path><path d="M17.5 0 M17.5 0 C94.53 -0.02, 168.17 0.86, 279.5 0 M17.5 0 C79.8 -1.75, 140.57 -1.94, 279.5 0 M279.5 0 C292.46 -1.07, 296.41 6.53, 297 17.5 M279.5 0 C288.97 -1.43, 298.73 7.23, 297 17.5 M297 17.5 C299.13 27.13, 297.04 37.14, 297 52.5 M297 17.5 C296.43 27.84, 298 37.77, 297 52.5 M297 52.5 C297.22 62.26, 291.93 69.43, 279.5 70 M297 52.5 C298.69 62.98, 289.89 70.11, 279.5 70 M279.5 70 C214.48 70.76, 150.23 69.78, 17.5 70 M279.5 70 C219.04 70.72, 159.44 70.21, 17.5 70 M17.5 70 C5.93 71.65, -1.89 65.1, 0 52.5 M17.5 70 C5.28 69.13, 1.31 65.79, 0 52.5 M0 52.5 C-1.16 39.21, -1.26 27.52, 0 17.5 M0 52.5 C-0.4 44.47, 0.06 36.43, 0 17.5 M0 17.5 C1.44 5.47, 6.85 0.37, 17.5 0 M0 17.5 C1.7 5.01, 5.58 0.87, 17.5 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(165.188593623614 32.5) rotate(0 18.66999053955078 12.5)"><text x="18.66999053955078" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">CRI</text></g><g mask="url(#mask-x7bA0n7Ehc-konmG6XiGb)" stroke-linecap="round"><g transform="translate(184.27770148383206 79.78275368430513) rotate(0 1.1199047305378826 36.80769423820993)"><path d="M-0.19 0.21 C0.6 12.16, 3.5 60.74, 3.99 72.8 M-1.75 -0.73 C-1.01 11.94, 2.12 61.93, 3.27 74.35" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(184.27770148383206 79.78275368430513) rotate(0 1.1199047305378826 36.80769423820993)"><path d="M-8.73 47.73 C-5.88 52.07, -3.33 61.12, 5.22 72.83 M-8.91 46.75 C-4.43 55.26, 0.04 65.69, 2.31 74.36" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(184.27770148383206 79.78275368430513) rotate(0 1.1199047305378826 36.80769423820993)"><path d="M11.73 46.2 C9.77 50.86, 7.5 60.27, 5.22 72.83 M11.55 45.21 C9.33 54.16, 7.09 65.1, 2.31 74.36" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask id="mask-x7bA0n7Ehc-konmG6XiGb"><rect x="0" y="0" fill="#fff" width="287.82569123798544" height="252.56058571555513"></rect><rect x="111.73175770124078" y="103.67166969993013" fill="#000" width="148.63987731933594" height="25" opacity="1"></rect></mask><g transform="translate(111.73175770124078 103.67166969993013) rotate(0 73.66584851312916 12.918778222584933)"><text x="74.31993865966797" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">RunPodSandbox</text></g><g stroke-linecap="round" transform="translate(14.94436815049994 153.56058571555513) rotate(0 176 36)"><path d="M18 0 M18 0 C124.09 2.41, 232.01 0.5, 334 0 M18 0 C121.75 0.53, 225.56 0.18, 334 0 M334 0 C347.67 1.43, 350.09 6.02, 352 18 M334 0 C346.71 -0.29, 354.24 4.88, 352 18 M352 18 C353.12 26.96, 353.32 40.75, 352 54 M352 18 C352.29 25.71, 351.1 32.77, 352 54 M352 54 C350.77 67.34, 346.15 72.17, 334 72 M352 54 C350.37 65.61, 346.32 70.12, 334 72 M334 72 C236.74 71.21, 139.9 72.01, 18 72 M334 72 C261.62 73.21, 189.26 73.51, 18 72 M18 72 C4.12 73.34, -1.88 66.49, 0 54 M18 72 C3.85 72.4, 2 68.05, 0 54 M0 54 C0.21 41.32, -0.96 27.79, 0 18 M0 54 C-0.92 40.3, 0.87 26.45, 0 18 M0 18 C1.61 5.52, 5.07 -1.93, 18 0 M0 18 C-2.17 7.52, 5.54 -1.63, 18 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(135.93441179063666 177.06058571555513) rotate(0 55.00995635986328 12.5)"><text x="55.00995635986328" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">CRI module</text></g><g mask="url(#mask-K8tCZIyk2EXyYdUuxqVUo)" stroke-linecap="round"><g transform="translate(192.27770148383206 224.67166969993013) rotate(0 -0.04788608622271795 64.55094473573308)"><path d="M-0.91 0.34 C-0.9 21.52, 0.6 106.96, 0.79 128.3 M0.81 -0.53 C0.67 21.33, 0.47 107.96, 0.25 129.64" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(192.27770148383206 224.67166969993013) rotate(0 -0.04788608622271795 64.55094473573308)"><path d="M-9.29 102.7 C-6.58 111.96, -2.81 122.63, -1.25 131.31 M-9.54 101.26 C-5.39 111.4, -2.71 122.51, 0.18 130.17" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(192.27770148383206 224.67166969993013) rotate(0 -0.04788608622271795 64.55094473573308)"><path d="M11.23 102.82 C6.56 111.88, 2.94 122.51, -1.25 131.31 M10.98 101.38 C7.5 111.47, 2.57 122.55, 0.18 130.17" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask id="mask-K8tCZIyk2EXyYdUuxqVUo"><rect x="0" y="0" fill="#fff" width="292.2792459937632" height="453.3541924282599"></rect><rect x="107.64699301060477" y="276.512931064095" fill="#000" width="169.25987243652344" height="25" opacity="1"></rect></mask><g transform="translate(107.64699301060477 276.512931064095) rotate(0 84.58282238700457 12.70968337156819)"><text x="84.62993621826172" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">CRI sandbox info</text></g><g stroke-linecap="round" transform="translate(658.3367625296412 363.34724506793555) rotate(0 111 34.5)"><path d="M17.25 0 M17.25 0 C85.32 0.53, 152.89 1.36, 204.75 0 M17.25 0 C87.22 -0.14, 157.54 -0.5, 204.75 0 M204.75 0 C216.06 0.28, 221.85 6.82, 222 17.25 M204.75 0 C214.52 1.46, 224.1 3.59, 222 17.25 M222 17.25 C222.84 26.41, 222.11 38.29, 222 51.75 M222 17.25 C221.32 29.74, 222.17 41.76, 222 51.75 M222 51.75 C222.1 64.51, 216.71 67.87, 204.75 69 M222 51.75 C219.88 61.33, 215.43 67.37, 204.75 69 M204.75 69 C164.94 68.88, 127.18 67.06, 17.25 69 M204.75 69 C135 69.63, 68 70.35, 17.25 69 M17.25 69 C6.57 67.46, -1.58 63.18, 0 51.75 M17.25 69 C5.3 67.58, -1.88 65.21, 0 51.75 M0 51.75 C0.22 42.52, 0.27 31.08, 0 17.25 M0 51.75 C-0.78 42.89, -0.04 31.99, 0 17.25 M0 17.25 C0.29 4.05, 5.47 -0.8, 17.25 0 M0 17.25 C-1.01 7.38, 6.01 1.86, 17.25 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(709.7868052542506 385.34724506793555) rotate(0 59.549957275390625 12.5)"><text x="59.549957275390625" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">NRI convert</text></g><g mask="url(#mask--wrxjs_xJlI6TVvnP5COo)" stroke-linecap="round"><g transform="translate(654.3719981519619 410.5519856603887) rotate(0 -126.84245074879391 0.009490662336702371)"><path d="M1 0.86 C-41.53 0.87, -212.33 -0.53, -254.69 -0.84 M0.07 0.26 C-42.15 0.56, -210.28 0.71, -252.58 0.7" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(654.3719981519619 410.5519856603887) rotate(0 -126.84245074879391 0.009490662336702371)"><path d="M-222.96 -11.49 C-234.09 -5.21, -244 -0.36, -253.55 0.52 M-225.15 -8.94 C-232.34 -7.14, -241.87 -2.26, -252.37 1" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(654.3719981519619 410.5519856603887) rotate(0 -126.84245074879391 0.009490662336702371)"><path d="M-222.95 9.04 C-233.93 8.29, -243.84 6.12, -253.55 0.52 M-225.14 11.58 C-232.31 6.94, -241.85 5.38, -252.37 1" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask id="mask--wrxjs_xJlI6TVvnP5COo"><rect x="0" y="0" fill="#fff" width="1007.9142232648026" height="511.40129610153326"></rect><rect x="450.48095151351026" y="397.62733043981643" fill="#000" width="154.2398681640625" height="25" opacity="1"></rect></mask><g transform="translate(450.48095151351026 397.62733043981643) rotate(0 77.04859588965769 12.934145882908979)"><text x="77.11993408203125" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">nri sandbox info</text></g><g stroke-linecap="round" transform="translate(10 366.1875410203811) rotate(0 190 33)"><path d="M16.5 0 M16.5 0 C87.26 0.58, 157.72 -0.82, 363.5 0 M16.5 0 C124.9 -0.8, 234.89 -1.83, 363.5 0 M363.5 0 C374.95 1.42, 380.42 6.09, 380 16.5 M363.5 0 C372.24 -0.16, 378.59 7.04, 380 16.5 M380 16.5 C379.69 26.93, 378.39 36.9, 380 49.5 M380 16.5 C379.15 27.65, 379.72 39.76, 380 49.5 M380 49.5 C378.12 61.96, 374.44 64.82, 363.5 66 M380 49.5 C379.93 60.13, 372.34 67.54, 363.5 66 M363.5 66 C292.17 65.02, 221.55 64.42, 16.5 66 M363.5 66 C285.96 65.52, 208.14 66.48, 16.5 66 M16.5 66 C5.96 66.77, -1.68 60.85, 0 49.5 M16.5 66 C7.7 66.47, 1.85 59.95, 0 49.5 M0 49.5 C1.64 40.73, -1.1 33.05, 0 16.5 M0 49.5 C-0.39 43.12, 0.16 36.12, 0 16.5 M0 16.5 C0.2 5.61, 3.97 -0.03, 16.5 0 M0 16.5 C-0.41 3.44, 5.91 0.96, 16.5 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(134.43004608154297 386.6875410203811) rotate(0 65.56995391845703 12.5)"><text x="65.56995391845703" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">NRI Adaption</text></g><g mask="url(#mask-IJbcakQRvHVBERb-IzN8V)" stroke-linecap="round"><g transform="translate(191.37748251763867 438.52083112047876) rotate(0 1.2565344676615382 101.85002309368747)"><path d="M0.27 0.85 C0.68 34.95, 3.25 169.76, 3.56 203.45 M-1.04 0.25 C-0.87 34.07, 1.83 167.51, 2.74 201.51" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(191.37748251763867 438.52083112047876) rotate(0 1.2565344676615382 101.85002309368747)"><path d="M-6.74 173.98 C-3.56 182.5, -0.03 194.17, 4.08 201.66 M-8.02 172.74 C-4.94 182.84, -1.4 190.27, 1.82 200.68" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(191.37748251763867 438.52083112047876) rotate(0 1.2565344676615382 101.85002309368747)"><path d="M13.78 173.51 C8.92 182.26, 4.43 194.12, 4.08 201.66 M12.5 172.28 C9.24 182.49, 6.45 190.06, 1.82 200.68" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask id="mask-IJbcakQRvHVBERb-IzN8V"><rect x="0" y="0" fill="#fff" width="294.6846566480016" height="741.6127470780584"></rect><rect x="87.09113580596465" y="490.06678909926904" fill="#000" width="211.87986755371094" height="100" opacity="1"></rect></mask><g transform="translate(87.09113580596465 490.06678909926904) rotate(0 105.54288117933555 50.304065114897185)"><text x="105.93993377685547" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">call </text><text x="105.93993377685547" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">RUN_POD_SANDBOX</text><text x="105.93993377685547" y="50" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">StateChange for all </text><text x="105.93993377685547" y="75" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">manager plugin</text></g><g stroke-linecap="round" transform="translate(19.767828191946137 643.4519123531309) rotate(0 173 28.5)"><path d="M14.25 0 M14.25 0 C116.04 1.34, 218.97 2.3, 331.75 0 M14.25 0 C112.38 -0.19, 210.34 0.83, 331.75 0 M331.75 0 C341.71 -1.13, 344.16 3.08, 346 14.25 M331.75 0 C340.43 -1.63, 343.84 6.42, 346 14.25 M346 14.25 C345.51 20.83, 347.17 29.29, 346 42.75 M346 14.25 C346.6 24.39, 346.15 31.97, 346 42.75 M346 42.75 C344.42 52.18, 340.86 55.77, 331.75 57 M346 42.75 C344.12 54.21, 341.77 57.88, 331.75 57 M331.75 57 C263.49 60.27, 194.32 58.48, 14.25 57 M331.75 57 C266.96 57.98, 202.56 57.88, 14.25 57 M14.25 57 C4.47 56.2, -0.88 53.67, 0 42.75 M14.25 57 C5.01 58.86, 0.23 52.37, 0 42.75 M0 42.75 C-1.79 36.74, 0.84 30.16, 0 14.25 M0 42.75 C-0.56 37.15, 0.71 30.57, 0 14.25 M0 14.25 C-0.06 4.85, 4.2 -1.27, 14.25 0 M0 14.25 C1.65 3.76, 2.8 0.09, 14.25 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(167.6578580991727 659.4519123531309) rotate(0 25.109970092773438 12.5)"><text x="25.109970092773438" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">plugin</text></g><g stroke-linecap="round"><g transform="translate(194.9509815388708 701.7008043063797) rotate(0 -1.8727292897347638 20.04266170563824)"><path d="M0.09 0.1 C-0.43 6.56, -3.35 33.1, -3.84 39.54 M-1.32 -0.89 C-1.4 5.69, -1.21 34.23, -1.57 40.97" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(194.9509815388708 701.7008043063797) rotate(0 -1.8727292897347638 20.04266170563824)"><path d="M-7.86 20.71 C-6.71 26.89, -6.46 31.22, -0.31 41.43 M-8.39 21.41 C-7.57 27.49, -5.1 30.76, -1.6 40.81" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(194.9509815388708 701.7008043063797) rotate(0 -1.8727292897347638 20.04266170563824)"><path d="M5.77 20.96 C3.67 27.1, 0.66 31.36, -0.31 41.43 M5.24 21.66 C2.74 27.56, 1.9 30.78, -1.6 40.81" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(19.38669368170781 868.7656935634695) rotate(0 165.5 29.5)"><path d="M3.93 3.41 C3.93 3.41, 3.93 3.41, 3.93 3.41 M3.93 3.41 C3.93 3.41, 3.93 3.41, 3.93 3.41 M3.01 10.57 C3.78 8.82, 5.75 6.11, 10.23 2.26 M3.01 10.57 C5.22 7.6, 8.86 4.95, 10.23 2.26 M2.75 16.96 C7.5 11.27, 11.04 4.62, 16.52 1.11 M2.75 16.96 C7.18 10.98, 14.29 4.9, 16.52 1.11 M2.49 23.36 C7.85 13.24, 18.34 6.4, 21.51 1.47 M2.49 23.36 C6.68 18.11, 10.74 14.2, 21.51 1.47 M2.88 29 C10.32 18.75, 17.75 11.23, 27.15 1.08 M2.88 29 C8.01 22.03, 13.23 16.75, 27.15 1.08 M3.27 34.65 C8.96 26.11, 17.7 17.35, 32.14 1.44 M3.27 34.65 C9.01 26.96, 16.88 19.78, 32.14 1.44 M3.01 41.04 C12.92 29.62, 21.59 21.06, 37.78 1.05 M3.01 41.04 C14.83 29.11, 25.34 16.08, 37.78 1.05 M3.41 46.69 C12.07 36.04, 26.08 20.94, 42.77 1.41 M3.41 46.69 C18.55 29.47, 33.69 11.02, 42.77 1.41 M3.15 53.09 C17.31 37.56, 26.6 25.46, 48.41 1.01 M3.15 53.09 C18.33 33.66, 35.08 17.23, 48.41 1.01 M6.17 55.71 C23.65 37.07, 37.7 20.21, 53.4 1.37 M6.17 55.71 C16.29 43.46, 25.24 33.41, 53.4 1.37 M8.53 59.09 C21.72 40.63, 36.37 26.39, 58.39 1.73 M8.53 59.09 C22.3 44.39, 34.6 29.45, 58.39 1.73 M12.86 60.2 C31.04 38.36, 51.13 18.27, 64.03 1.34 M12.86 60.2 C31.04 39.43, 47.26 20.22, 64.03 1.34 M18.5 59.81 C33.24 40.72, 50.87 21.33, 69.02 1.7 M18.5 59.81 C36.05 38.18, 55.19 16.95, 69.02 1.7 M23.49 60.17 C39.02 41.95, 53.07 24.26, 74.66 1.3 M23.49 60.17 C41.42 40.49, 61.29 18.43, 74.66 1.3 M28.48 60.53 C40.18 44.7, 54.64 29.73, 79.65 1.66 M28.48 60.53 C39.94 48.29, 50.54 36.59, 79.65 1.66 M34.12 60.13 C51.01 42.4, 64.68 21.96, 85.29 1.27 M34.12 60.13 C51.41 41.06, 68.62 20.07, 85.29 1.27 M39.11 60.49 C58.09 34.99, 80.97 14.81, 90.28 1.63 M39.11 60.49 C59.62 37.84, 78.79 16.38, 90.28 1.63 M44.75 60.1 C55.11 47.13, 65.45 33.43, 95.92 1.23 M44.75 60.1 C60.73 41.75, 74.49 25.91, 95.92 1.23 M49.74 60.46 C67.68 42.05, 80.39 24.83, 100.91 1.59 M49.74 60.46 C62.92 45.77, 77.58 29.82, 100.91 1.59 M55.38 60.07 C66.45 46.26, 77.8 32.23, 106.55 1.2 M55.38 60.07 C74.15 39.13, 92.39 18.54, 106.55 1.2 M60.37 60.43 C75.59 41.3, 92.34 20.43, 111.54 1.56 M60.37 60.43 C74.27 44.94, 88.21 27.42, 111.54 1.56 M66.01 60.03 C76.64 46.67, 87.59 36.6, 117.18 1.16 M66.01 60.03 C86.46 37.17, 106.63 13.3, 117.18 1.16 M71 60.39 C87.52 43.4, 101.25 27.25, 122.17 1.52 M71 60.39 C86.38 44.88, 99.69 27.41, 122.17 1.52 M76.64 60 C88.44 45.84, 101.51 31.91, 127.81 1.13 M76.64 60 C95.82 39.89, 114.12 17.37, 127.81 1.13 M81.63 60.36 C101.72 39.14, 118.84 17.74, 132.8 1.49 M81.63 60.36 C96.5 41.69, 111.89 24.52, 132.8 1.49 M87.27 59.96 C98.11 46.24, 112.97 32.64, 138.44 1.09 M87.27 59.96 C106.58 38.13, 125.16 16.64, 138.44 1.09 M92.26 60.32 C107.99 45.49, 122.27 28.3, 143.43 1.46 M92.26 60.32 C105.81 45.55, 116.54 33.06, 143.43 1.46 M97.9 59.93 C117.32 39.04, 132.2 18.85, 148.42 1.82 M97.9 59.93 C109.31 46.45, 120.94 32.23, 148.42 1.82 M102.89 60.29 C117.23 44.71, 126.81 31.56, 154.06 1.42 M102.89 60.29 C113.22 47.13, 125.4 35.75, 154.06 1.42 M108.53 59.89 C128.08 38.51, 143.48 20.75, 159.05 1.78 M108.53 59.89 C122.18 43.31, 138.33 26.98, 159.05 1.78 M113.52 60.25 C125 48.56, 136.45 35.63, 164.69 1.39 M113.52 60.25 C129.44 39.5, 148.2 21.44, 164.69 1.39 M118.5 60.61 C129.24 46.66, 141.09 33.42, 169.68 1.75 M118.5 60.61 C128.77 48.12, 139.79 35.73, 169.68 1.75 M124.15 60.22 C139.46 42.64, 156.38 24.92, 175.32 1.35 M124.15 60.22 C141.71 39.58, 160.82 18.08, 175.32 1.35 M129.13 60.58 C139.15 48.25, 152.04 35.76, 180.31 1.71 M129.13 60.58 C149.5 38.26, 168.01 16.44, 180.31 1.71 M134.78 60.18 C149.4 42.98, 164.48 24.29, 185.95 1.32 M134.78 60.18 C147 45.82, 159.24 31.09, 185.95 1.32 M139.76 60.54 C159.22 38.6, 176.32 16.59, 190.94 1.68 M139.76 60.54 C155.94 43.46, 169.52 25.43, 190.94 1.68 M145.41 60.15 C163.67 39.69, 181.5 19.34, 196.58 1.28 M145.41 60.15 C156.3 48.24, 165.8 37.4, 196.58 1.28 M150.39 60.51 C167.74 39.29, 185.52 19.12, 201.57 1.64 M150.39 60.51 C160.2 47.55, 171.09 35.8, 201.57 1.64 M156.04 60.12 C173.38 40.37, 190.53 19.2, 207.21 1.25 M156.04 60.12 C167.06 45.9, 179.19 32.46, 207.21 1.25 M161.02 60.48 C176.39 41.66, 192.53 25.13, 212.2 1.61 M161.02 60.48 C173.5 47.07, 185.66 33.01, 212.2 1.61 M166.67 60.08 C181.83 41.45, 195.38 28.26, 217.84 1.21 M166.67 60.08 C188.03 36.25, 207.53 13.82, 217.84 1.21 M171.65 60.44 C187.43 42.29, 208.26 21.96, 222.83 1.57 M171.65 60.44 C187.1 44.01, 200.95 29.3, 222.83 1.57 M177.3 60.05 C192.21 43.27, 204.35 29.26, 228.47 1.18 M177.3 60.05 C192.3 40.59, 208.9 22.17, 228.47 1.18 M182.28 60.41 C194.4 46.6, 205.86 33.72, 233.46 1.54 M182.28 60.41 C198.11 41.31, 213.78 23.69, 233.46 1.54 M187.93 60.01 C207.51 39.05, 227.59 16.39, 238.44 1.9 M187.93 60.01 C203.15 42.43, 218.25 24.99, 238.44 1.9 M192.91 60.37 C204.22 48.69, 215.79 37.99, 244.09 1.5 M192.91 60.37 C206.63 43.65, 220.95 28.56, 244.09 1.5 M198.56 59.98 C209.54 47.29, 220.79 37.85, 249.07 1.86 M198.56 59.98 C210.31 45.44, 223.2 32.46, 249.07 1.86 M203.54 60.34 C216.16 46.74, 230.05 32.5, 254.72 1.47 M203.54 60.34 C221.64 37.84, 241.62 16.46, 254.72 1.47 M209.19 59.94 C222.17 48.53, 234.07 32.57, 259.7 1.83 M209.19 59.94 C221.7 44.65, 235.23 30.92, 259.7 1.83 M214.17 60.3 C227 46.99, 235.35 35.22, 265.35 1.44 M214.17 60.3 C234.31 35.93, 255.65 12.34, 265.35 1.44 M219.16 60.66 C237.12 42.08, 249.85 22.86, 270.33 1.8 M219.16 60.66 C235.01 43.36, 249.99 25.82, 270.33 1.8 M224.8 60.27 C233.85 47.07, 248.43 36.06, 275.98 1.4 M224.8 60.27 C235.61 47.95, 245.97 34.93, 275.98 1.4 M229.79 60.63 C241.17 45.42, 251.28 32.13, 280.96 1.76 M229.79 60.63 C247.63 39.56, 266.36 19.73, 280.96 1.76 M235.44 60.23 C251.23 43, 266.83 23.64, 286.61 1.37 M235.44 60.23 C247.05 46.29, 259.75 33.6, 286.61 1.37 M240.42 60.59 C253.85 44.03, 267.76 30.19, 291.59 1.73 M240.42 60.59 C259.67 38.32, 278.58 17.36, 291.59 1.73 M246.07 60.2 C259.27 43.52, 276.74 27.22, 297.24 1.33 M246.07 60.2 C265.27 37.58, 283.79 15.14, 297.24 1.33 M251.05 60.56 C261.6 46.04, 277.32 31.44, 302.22 1.69 M251.05 60.56 C260.97 48.11, 272.35 36.57, 302.22 1.69 M256.7 60.17 C271.97 46.16, 282.07 29.64, 307.87 1.3 M256.7 60.17 C272.13 42.98, 286.92 26.2, 307.87 1.3 M261.68 60.53 C277.73 42.68, 295.63 20.05, 312.85 1.66 M261.68 60.53 C275.08 44.54, 287.85 30.69, 312.85 1.66 M267.33 60.13 C279.22 44.41, 295.84 29.81, 318.5 1.26 M267.33 60.13 C285.2 40.2, 303.6 18.53, 318.5 1.26 M272.31 60.49 C293.1 38.88, 311.75 16.22, 322.83 2.38 M272.31 60.49 C281.92 48.69, 292.77 37.5, 322.83 2.38 M277.96 60.1 C292.66 43.03, 307.49 27.15, 326.5 4.25 M277.96 60.1 C290.49 46.78, 302.36 33.34, 326.5 4.25 M282.94 60.46 C293.48 48.19, 308.05 34.89, 330.18 6.12 M282.94 60.46 C298.44 43.45, 311.9 27.77, 330.18 6.12 M288.59 60.06 C298.97 49.02, 308.19 37.74, 331.89 10.25 M288.59 60.06 C305.81 40.42, 323.04 21.08, 331.89 10.25 M293.57 60.42 C302.48 50.16, 309.22 40.54, 332.28 15.89 M293.57 60.42 C303.21 48.58, 313.72 35.44, 332.28 15.89 M299.22 60.03 C306.96 49.87, 315.93 38.63, 332.02 22.29 M299.22 60.03 C309.31 47.69, 320.43 36.73, 332.02 22.29 M304.2 60.39 C309.09 54.93, 317.06 47.84, 332.41 27.93 M304.2 60.39 C309.73 52.62, 316.02 45.36, 332.41 27.93 M309.19 60.75 C312.96 54.82, 316.83 51.25, 332.81 33.58 M309.19 60.75 C316.45 52.68, 322.26 45.75, 332.81 33.58 M314.83 60.35 C319.89 53.48, 326.81 47.03, 332.55 39.98 M314.83 60.35 C320.09 54.76, 326.02 48.76, 332.55 39.98 M319.16 61.47 C323.31 55.33, 326.3 53.19, 330.97 47.88 M319.16 61.47 C321.46 57.99, 325.08 54.5, 330.97 47.88" stroke="#ffc9c9" stroke-width="0.5" fill="none"></path><path d="M14.75 0 M14.75 0 C90.24 -0.84, 163.71 1.42, 316.25 0 M14.75 0 C94.46 1.24, 174.76 1.71, 316.25 0 M316.25 0 C324.38 0.19, 330.55 3.04, 331 14.75 M316.25 0 C324.62 1.7, 331.78 6.45, 331 14.75 M331 14.75 C330.09 20.5, 331.98 28.07, 331 44.25 M331 14.75 C330.53 25.04, 331.28 36.93, 331 44.25 M331 44.25 C331.1 55.01, 327.19 59.09, 316.25 59 M331 44.25 C328.94 53.97, 324.79 59.56, 316.25 59 M316.25 59 C247.21 57.94, 175.83 59.03, 14.75 59 M316.25 59 C248.56 61.9, 181.29 60.91, 14.75 59 M14.75 59 C6.28 60.71, 1.04 53.33, 0 44.25 M14.75 59 C4.16 57.67, -2 53.85, 0 44.25 M0 44.25 C-1.04 34.94, 1.38 28.49, 0 14.75 M0 44.25 C-0.44 36.21, -0.17 26.86, 0 14.75 M0 14.75 C-1.76 6.75, 5.69 -0.23, 14.75 0 M0 14.75 C-0.6 3.48, 3.45 -0.75, 14.75 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(143.61673518561406 885.7656935634695) rotate(0 41.26995849609375 12.5)"><text x="41.26995849609375" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">nri plugin</text></g><g stroke-linecap="round" transform="translate(15.611948117769998 745.2189051470195) rotate(0 173 28.5)"><path d="M14.25 0 M14.25 0 C119.07 -2.65, 225.34 -3.03, 331.75 0 M14.25 0 C134.69 0.55, 254.74 0.46, 331.75 0 M331.75 0 C342.82 1.43, 347.58 4.51, 346 14.25 M331.75 0 C342.01 0.4, 344.41 4.75, 346 14.25 M346 14.25 C347.77 23.93, 347.33 33.4, 346 42.75 M346 14.25 C346.58 23.32, 345.22 32.98, 346 42.75 M346 42.75 C345.82 50.99, 339.82 57.89, 331.75 57 M346 42.75 C346.44 52.57, 343.02 55.71, 331.75 57 M331.75 57 C236.08 53.9, 141.82 55.23, 14.25 57 M331.75 57 C241.6 54.98, 152.72 55, 14.25 57 M14.25 57 C4.37 56.28, -0.87 50.74, 0 42.75 M14.25 57 C3.21 58.67, -1.24 53.93, 0 42.75 M0 42.75 C-0.21 30.94, 0 18.56, 0 14.25 M0 42.75 C-0.61 35.53, 0.76 27.03, 0 14.25 M0 14.25 C1.65 3.64, 4.83 0.38, 14.25 0 M0 14.25 C-0.98 3.72, 4.54 2.02, 14.25 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(26.742136106051248 761.2189051470195) rotate(0 161.86981201171875 12.5)"><text x="161.86981201171875" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">isula-rust-extensions-plugin client</text></g><g stroke-linecap="round"><g transform="translate(193.48920462632282 806.1360531347664) rotate(0 0.5828912998827604 32.13722384666016)"><path d="M-0.41 -0.14 C-0.23 10.57, 0.43 53.41, 0.36 64.09 M1.58 -1.26 C1.7 9.65, -0.42 54.47, -0.42 65.53" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(193.48920462632282 806.1360531347664) rotate(0 0.5828912998827604 32.13722384666016)"><path d="M-10.16 37.63 C-8 42.42, -5.96 49.26, -1.67 63.77 M-9.73 37.48 C-6.99 45.75, -3.62 54.01, -0.12 66.51" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(193.48920462632282 806.1360531347664) rotate(0 0.5828912998827604 32.13722384666016)"><path d="M10.35 38.17 C7.63 42.76, 4.78 49.47, -1.67 63.77 M10.78 38.03 C7.23 46.15, 4.32 54.24, -0.12 66.51" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask></mask><g mask="url(#mask-MnhzEbIaIqh74s_zosROW)" stroke-linecap="round"><g transform="translate(402.05735358504376 377.54606753379994) rotate(0 121.57534873310851 -2.01877893778601)"><path d="M-0.66 -0.01 C39.99 -0.54, 202.98 -1.73, 243.81 -2.09 M1.19 -1.06 C41.67 -1.46, 202.34 -4.08, 242.77 -4.03" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(402.05735358504376 377.54606753379994) rotate(0 121.57534873310851 -2.01877893778601)"><path d="M214.65 6.17 C223.4 4.15, 231.51 2.45, 241.01 -2.68 M214.97 6.2 C225.88 1.81, 235.85 -2.01, 242.46 -4.71" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(402.05735358504376 377.54606753379994) rotate(0 121.57534873310851 -2.01877893778601)"><path d="M214.47 -14.35 C223.47 -10.34, 231.62 -6.03, 241.01 -2.68 M214.79 -14.32 C225.82 -11.02, 235.86 -7.15, 242.46 -4.71" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask id="mask-MnhzEbIaIqh74s_zosROW"><rect x="0" y="0" fill="#fff" width="746.0551613629477" height="479.98187432259056"></rect><rect x="439.42632125573414" y="363.8281641394051" fill="#000" width="169.25987243652344" height="25" opacity="1"></rect></mask><g transform="translate(439.42632125573414 363.8281641394051) rotate(0 84.20638106241813 11.699124456608843)"><text x="84.62993621826172" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">CRI sandbox info</text></g></svg> +\ No newline at end of file +diff --git a/docs/design/detailed/NRI/nri_design.md b/docs/design/detailed/NRI/nri_design.md +new file mode 100644 +index 00000000..1f68df5d +--- /dev/null ++++ b/docs/design/detailed/NRI/nri_design.md +@@ -0,0 +1,382 @@ ++| Author | zhongtao | ++| ------ | --------------------- | ++| Date | 2024-07-21 | ++| Email | zhongtao17@huawei.com | ++# 方案目标 ++## NRI介绍 ++ ++[NRI](https://github.com/containerd/nri) (Node Resource Interface), 是用于控制节点资源的公共接口, 是CRI兼容的容器运行时插件扩展的通用框架。它为扩展插件提供了跟踪容器状态，并对其配置进行有限修改的基本机制。允许将用户某些自定的逻辑插入到OCI兼容的运行时中，此逻辑可以对容器进行受控更改，或在容器生命周期的某些时间点执行 OCI 范围之外的额外操作。例如，用于改进设备和其他容器资源的分配和管理。 ++ ++NRI 定义了插件的api：https://github.com/containerd/nri/blob/main/pkg/api/api.proto,目前支持的api 版本为0.6.1 ++ ++NRI 插件能够订阅Pod和容器生命周期事件： ++ ++### 订阅Pod生命周期事件 ++ ++NRI能够订阅Pod生命周期事件，包括： ++- creation ++- stopping ++- removal ++ ++相关的API如下： ++```go ++// 当有pod在节点上创建时，NRI插件将收到该事件 ++func (p *plugin) RunPodSandbox(pod *api.PodSandbox) error { ++ return nil ++} ++// 当有pod在节点上停止时，NRI插件将收到该事件 ++func (p *plugin) StopPodSandbox(pod *api.PodSandbox) error { ++ return nil ++} ++ ++// 当有pod在节点上移除时，NRI插件将收到该事件 ++func (p *plugin) RemovePodSandbox(pod *api.PodSandbox) error { ++ return nil ++} ++``` ++ ++### 订阅容器生命周期事件 ++ ++NRI能够订阅容器生命周期事件，包括： ++ ++- creation ( * ) ++- post-creation ++- starting ++- post-start ++- updating ( * ) ++- post-update ++- stopping ( * ) ++- removal ++ ++( * )后缀的表示可以根据插件的返回信息更新容器的oci spec。相关的API如下： ++ ++```go ++// 创建容器 ++func (p *plugin) CreateContainer(pod *api.PodSandbox, container *api.Container) (*api.ContainerAdjustment, []*api.ContainerUpdate, error) { ++ return nil, nil, nil ++} ++// 容器创建以后 ++func (p *plugin) PostCreateContainer(pod *api.PodSandbox, container *api.Container) error { ++ return nil ++} ++// 容器启动之前 ++func (p *plugin) StartContainer(pod *api.PodSandbox, container *api.Container) error { ++ return nil ++} ++// 容器启动之后 ++func (p *plugin) PostStartContainer(pod *api.PodSandbox, container *api.Container) error { ++ return nil ++} ++// 容器更新时 ++func (p *plugin) UpdateContainer(pod *api.PodSandbox, container *api.Container) ([]*api.ContainerUpdate, error) { ++ return nil, nil ++} ++// 容器更新后 ++func (p *plugin) PostUpdateContainer(pod *api.PodSandbox, container *api.Container) error { ++ return nil ++} ++// 容器停止时 ++func (p *plugin) StopContainer(pod *api.PodSandbox, container *api.Container) ([]*api.ContainerUpdate, error) { ++ return nil, nil ++} ++// 容器移除时 ++func (p *plugin) RemoveContainer(pod *api.PodSandbox, container *api.Container) error { ++ return nil ++} ++``` ++ ++### NRI插件可获得的信息 ++ ++```proto ++// Pod metadata that is considered relevant for a plugin. ++message PodSandbox { ++ string id = 1; ++ string name = 2; ++ string uid = 3; ++ string namespace = 4; ++ map<string, string> labels = 5; ++ map<string, string> annotations = 6; ++ string runtime_handler = 7; ++ LinuxPodSandbox linux = 8; ++ uint32 pid = 9; // for NRI v1 emulation ++} ++ ++// Container metadata that is considered relevant for a plugin. ++message Container { ++ string id = 1; ++ string pod_sandbox_id = 2; ++ string name = 3; ++ ContainerState state = 4; ++ map<string, string> labels = 5; ++ map<string, string> annotations = 6; ++ repeated string args = 7; ++ repeated string env = 8; ++ repeated Mount mounts = 9; ++ Hooks hooks = 10; ++ LinuxContainer linux = 11; ++ uint32 pid = 12; // for NRI v1 emulation ++ repeated POSIXRlimit rlimits = 13; ++} ++``` ++ ++### NRI 插件可更新资源 ++ ++#### 可更新的pod资源 ++ ++```proto ++// PodSandbox linux-specific metadata ++message LinuxPodSandbox { ++ LinuxResources pod_overhead = 1; ++ LinuxResources pod_resources = 2; ++ string cgroup_parent = 3; ++ string cgroups_path = 4; // for NRI v1 emulation ++ repeated LinuxNamespace namespaces = 5; // for NRI v1 emulation ++ LinuxResources resources = 6; // for NRI v1 emulation ++} ++ ++// Container (linux) resources. ++message LinuxResources { ++ LinuxMemory memory = 1; ++ LinuxCPU cpu = 2; ++ repeated HugepageLimit hugepage_limits = 3; ++ OptionalString blockio_class = 4; ++ OptionalString rdt_class = 5; ++ map<string, string> unified = 6; ++ repeated LinuxDeviceCgroup devices = 7; // for NRI v1 emulation ++} ++ ++// Memory-related parts of (linux) resources. ++message LinuxMemory { ++ OptionalInt64 limit = 1; ++ OptionalInt64 reservation = 2; ++ OptionalInt64 swap = 3; ++ OptionalInt64 kernel = 4; ++ OptionalInt64 kernel_tcp = 5; ++ OptionalUInt64 swappiness = 6; ++ OptionalBool disable_oom_killer = 7; ++ OptionalBool use_hierarchy = 8; ++} ++ ++// CPU-related parts of (linux) resources. ++message LinuxCPU { ++ OptionalUInt64 shares = 1; ++ OptionalInt64 quota = 2; ++ OptionalUInt64 period = 3; ++ OptionalInt64 realtime_runtime = 4; ++ OptionalUInt64 realtime_period = 5; ++ string cpus = 6; ++ string mems = 7; ++} ++ ++// Container huge page limit. ++message HugepageLimit { ++ string page_size = 1; ++ uint64 limit = 2; ++} ++ ++// Container rlimits ++message POSIXRlimit { ++ string type = 1; ++ uint64 hard = 2; ++ uint64 soft = 3; ++} ++ ++// A linux namespace. ++message LinuxNamespace { ++ string type = 1; ++ string path = 2; ++} ++``` ++ ++#### 可更新的container资源 ++ ++```proto ++// Requested adjustments to a container being created. ++message ContainerAdjustment { ++ map<string, string> annotations = 2; ++ repeated Mount mounts = 3; ++ repeated KeyValue env = 4; ++ Hooks hooks = 5; ++ LinuxContainerAdjustment linux = 6; ++ repeated POSIXRlimit rlimits = 7; ++} ++ ++// A container mount. ++message Mount { ++ string destination = 1; ++ string type = 2; ++ string source = 3; ++ repeated string options = 4; ++} ++ ++// Adjustments to (linux) resources. ++message LinuxContainerAdjustment { ++ repeated LinuxDevice devices = 1; ++ LinuxResources resources = 2; ++ string cgroups_path = 3; ++} ++ ++``` ++ ++## NRI 优势 ++NRI(Node Resource Interface), 是用于控制节点资源的公共接口, 是CRI兼容的容器运行时插件扩展的通用框架。它为扩展插件提供了跟踪容器状态，并对其配置进行有限修改的基本机制。 ++ ++在NRI规范之前，k8s场景下资源管理的流程与CRI管理流程是相互独立的，在管理面上的存在一些一致性的问题，NRI的推出则解决了这些问题： ++- NRI利用容器引擎来捕获生命周期，无需额外组件监控 ++- Pod资源管理流程与CRI原有流程归一化，减少管理及维护成本 ++- 在Pod管理流程中直接插入调度策略，不存在延迟 ++- Kubernetes获取的资源信息与底层信息保持一致 ++ ++简单来说，NRI的出现减少了管理维护成本，消除了调度延迟，规范了信息一致性 ++ ++## 需求目标 ++ ++实现iSulad对符合NRI 规范的nri插件的支持，使得对于iSulad管理的sandbox与容器，用户可以在不修改容器运行时源代码的情况下添加自定义逻辑，减少对容器资源的管理成本，且保证插件获取信息与容器引擎信息一致。 ++ ++# 总体设计 ++ ++```mermaid ++flowchart TD ++classDef background fill:#FFFFFF,stroke:#216,stroke-width:2px,color:#fff,stroke-dasharray:none; ++ ++classDef oldmodule fill:#DCDCDC,stroke:#216,stroke-width:2px,color:#fff,stroke-dasharray: none; ++ ++classDef newmodule fill:#a5d8ff,stroke:#216,stroke-width:2px,color:#fff,stroke-dasharray: 5 5; ++ ++classDef submodule fill:#87CEFA,stroke:#216,stroke-width:2px,color:#fff,stroke-dasharray: 5 5; ++ ++subgraph isulad ++ subgraph NRI ++ A[NRI Adaption] ++ B[NRI Helpers] ++ C[NRI plugin ops] ++ D[NRI Result] ++ E[Plugin] ++ end ++ ++ subgraph common ++ subgraph NRICommon ++ F[NRI Convert] ++ G[NRI Spec] ++ H[NRI utils] ++ end ++ end ++ ++ S[isulad init] --> A ++ A --> config ++ ICRI[CRI module] --> A ++ NRI --> NRICommon ++ A --> E ++ A --> B ++ A --> C ++ A --> D ++ I[executor] --> G ++ ICRI[CRI module] --> I ++ ++ config:::oldmodule ++ I:::oldmodule ++ S:::oldmodule ++ common:::oldmodule ++ ICRI:::oldmodule ++ NRI:::newmodule ++ NRICommon:::newmodule ++ A:::submodule ++ B:::submodule ++ C:::submodule ++ D:::submodule ++ E:::submodule ++ F:::submodule ++ G:::submodule ++ H:::submodule ++end ++isulad:::background ++OCRI[CRI API] --> ICRI ++E --> R ++R[isula-rust-extensions] --> P[nri plugin] ++P --> R ++``` ++模块简要介绍： ++1. NRI Adaption: （1）管理所有NRI插件；（2）与isula-rust-extension进行交互，调用isula-rust-extension的注册callback函数，告知isula-rust-extension接收到plugin注册与update请求时调用的函数 ++2. NRI Helpers： C++实现的nri帮助函数 ++3. NRI plugin ops： 提供转换后的adaption init函数以及提供需要注册到isula-rust-extension的callback函数 ++4. NRI Result：处理所有插件的adjust与update，得到一个合并后的结果，若插件修改冲突则报错 ++5. Plugin：定义plugin 类，用于定义plugin支持的相关操作以及存储的信息, 根据插件是否关注event来决定是否调用此plugin，需调用isula-rust-extension中plugin相关函数实现 ++6. NRI convert module:实现 CRI 结构体与 NRI结构体之间的转换、实现iSulad内部存储container的结构体与NRI结构体之间的转换 ++7. NRI Spec：实现nri修改oci spec的函数 ++8. NRI utils： 提供处理init与copy nri相关结构体的工具函数 ++ ++# 接口描述 ++ ++## 配置 ++```json ++# add support for NRI plugin. ++"nri-support": true, ++# Allow connections from externally launched NRI plugins. ++"disable_connections": true, ++# plugin_config_path is the directory to search for plugin-specific configuration. ++"plugin_config_path": "/etc/nri/conf.d" ++# plugin_path is the directory to search for plugins to launch on startup. ++"plugin_path": "/opt/nri/plugins" ++# plugin_registration_timeout is the timeout for a plugin to register after connection. ++"plugin_registration_timeout": "5s" ++# plugin_requst_timeout is the timeout for a plugin to handle an event/request. ++"plugin_request_timeout": "2s" ++# socket_path is the path of the NRI socket to create for plugins to connect to. ++"socket_path" = "/var/run/nri/nri.sock" ++``` ++# 详细设计 ++ ++ ++## 初始化流程 ++ ++ ++ ++## 响应RunPodSandbox事件流程 ++ ++ ++StopPodSandbox/RemovePodSandbox/PostCreateContainer/StartContainer/PostStartContainer/PostUpdateContainer/RemoveContainer 与RunPodSandbox处理流程相同，仅event种类变化 ++ ++## 响应CreateContainer事件流程 ++ ++ ++ ++ ++# 约束限制 ++## 接口参数限制 ++暂不支持修改以下参数： ++```proto ++// Container to evict(IOW unsolicitedly stop). ++ContainerEviction evict; ++ ++// ref:https://github.com/containerd/containerd/pull/5490 ++// - User defines blockio classes, for example: ThrottledIO and LowLatency. Class names are not restricted, and the number of classes is not limited. ++// iSulad not support ++OptionalString blockio_class; ++ ++// iSulad now not support the following hook types ++message Hooks { ++ repeated Hook create_runtime = 2; ++ repeated Hook create_container = 3; ++ repeated Hook start_container = 4; ++} ++``` ++tips: ++以下参数iSulad支持，containerd暂不支持 ++```proto ++// POSIX rlimits can be used control the resources a process can consume. ++POSIXRlimit rlimits ++``` ++ ++## 使用限制 ++1. 对于插件异常退出场景，iSulad目前仅在下次调用出错时打印日志（ ++2. 对于external 注册的plugin，若iSulad退出，iSulad不对其进行强制kill，生命周期与iSulad无关。 ++ ++# 测试点 ++ ++1.对于订阅不同生命周期事件的nri插件支持 ++ ++2.稳定性：某个插件异常退出时，iSulad中插件的行为是否存在异常残留以及是否影响其他插件功能。 ++ ++3.稳定性：iSulad退出后，iSulad自己拉取的nri插件是否正常被kill退出。 ++ ++4.注册方式：测试iSulad启动时拉取注册nri插件与external 注册nri插件 +diff --git a/docs/design/detailed/NRI/nri_detail.svg b/docs/design/detailed/NRI/nri_detail.svg +new file mode 100644 +index 00000000..69c0d053 +--- /dev/null ++++ b/docs/design/detailed/NRI/nri_detail.svg +@@ -0,0 +1,16 @@ ++<svg version="1.1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1856.451299601144 927.7272485224321" width="1856.451299601144" height="927.7272485224321"> ++ <!-- svg-source:excalidraw --> ++ ++ <defs> ++ <style class="style-fonts"> ++ @font-face { ++ font-family: "Virgil"; ++ src: url("https://excalidraw.com/Virgil.woff2"); ++ } ++ @font-face { ++ font-family: "Cascadia"; ++ src: url("https://excalidraw.com/Cascadia.woff2"); ++ } ++ </style> ++ </defs> ++ <rect x="0" y="0" width="1856.451299601144" height="927.7272485224321" fill="#ffffff"></rect><g stroke-linecap="round" transform="translate(666.016561039748 10) rotate(0 540.5 24.5)"><path d="M12.25 0 M12.25 0 C313.44 -3.5, 615.92 -4.63, 1068.75 0 M12.25 0 C350.65 -2.48, 688.9 -2.36, 1068.75 0 M1068.75 0 C1075.76 -0.09, 1079.73 3.41, 1081 12.25 M1068.75 0 C1076.81 -0.09, 1080.53 4.89, 1081 12.25 M1081 12.25 C1080.52 18, 1082.35 25.6, 1081 36.75 M1081 12.25 C1080.54 20.68, 1082.07 29.46, 1081 36.75 M1081 36.75 C1079.69 43.72, 1078.19 50.96, 1068.75 49 M1081 36.75 C1080.93 43.01, 1075.36 51.16, 1068.75 49 M1068.75 49 C657.77 50.28, 248.48 51.02, 12.25 49 M1068.75 49 C707.33 48.28, 345.7 47.58, 12.25 49 M12.25 49 C5.22 48.67, -0.97 44.74, 0 36.75 M12.25 49 C5.86 49.04, -0.84 44.13, 0 36.75 M0 36.75 C-0.94 30.33, -0.64 24.82, 0 12.25 M0 36.75 C0.9 29.85, 0.02 20.25, 0 12.25 M0 12.25 C0.73 5.41, 3.05 -0.51, 12.25 0 M0 12.25 C-1.46 2.65, 5.39 1.48, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(1187.8465705001972 22) rotate(0 18.66999053955078 12.5)"><text x="18.66999053955078" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">CRI</text></g><g stroke-linecap="round" transform="translate(1152.3727127527818 211.867619414028) rotate(0 121 25.5)"><path d="M12.75 0 M12.75 0 C99.37 -2.57, 185.29 -2.89, 229.25 0 M12.75 0 C89.99 0.69, 165.89 0.74, 229.25 0 M229.25 0 C236.48 -0.67, 241.91 4.17, 242 12.75 M229.25 0 C237.28 0.8, 241.73 5.35, 242 12.75 M242 12.75 C243.16 17.17, 243.26 24.68, 242 38.25 M242 12.75 C242.71 18.82, 241.18 25.85, 242 38.25 M242 38.25 C243.27 48.71, 237.69 49.34, 229.25 51 M242 38.25 C240.44 48.91, 239.78 52.33, 229.25 51 M229.25 51 C154.96 49.11, 79.05 50.3, 12.75 51 M229.25 51 C169.32 52.21, 111.27 52.72, 12.75 51 M12.75 51 C3.28 50.82, 1.54 46.79, 0 38.25 M12.75 51 C3.41 50.21, -1.33 48.67, 0 38.25 M0 38.25 C-0.34 28.22, 0.5 16.92, 0 12.75 M0 38.25 C-0.34 28.29, 0.18 20.63, 0 12.75 M0 12.75 C-1.03 3.74, 2.98 -1.25, 12.75 0 M0 12.75 C1.3 5.73, 3.04 0.75, 12.75 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(1218.3627563929185 224.867619414028) rotate(0 55.00995635986328 12.5)"><text x="55.00995635986328" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">CRI module</text></g><g stroke-linecap="round" transform="translate(805.7570895247284 622.5638004558205) rotate(0 330.5 34.5)"><path d="M4.59 3.99 C4.59 3.99, 4.59 3.99, 4.59 3.99 M4.59 3.99 C4.59 3.99, 4.59 3.99, 4.59 3.99 M3.02 11.9 C6 8.86, 8.69 4.86, 11.55 2.09 M3.02 11.9 C5.13 9.63, 8.19 6.35, 11.55 2.09 M2.1 19.05 C4.35 13.34, 8.41 11.66, 17.85 0.94 M2.1 19.05 C7.7 12.91, 15.53 5.34, 17.85 0.94 M2.5 24.69 C11.34 16.4, 18.15 9.34, 23.49 0.54 M2.5 24.69 C7.48 17.12, 14.59 11.1, 23.49 0.54 M2.23 31.09 C8.9 22.56, 14.78 14.55, 28.48 0.9 M2.23 31.09 C11.78 19.4, 22.05 8.24, 28.48 0.9 M2.63 36.74 C11.55 27.87, 15.28 20.56, 34.12 0.51 M2.63 36.74 C13.71 22.92, 26.45 10.92, 34.12 0.51 M3.02 42.38 C12.77 31.66, 20.68 21.8, 39.11 0.87 M3.02 42.38 C14.24 30.12, 24.14 16.23, 39.11 0.87 M2.76 48.78 C16.82 31.72, 33.28 14.11, 44.75 0.47 M2.76 48.78 C11.68 37.8, 22.3 27.24, 44.75 0.47 M3.16 54.42 C15.45 39.15, 27.11 26.66, 49.74 0.83 M3.16 54.42 C20.84 34.59, 38.02 14.85, 49.74 0.83 M2.89 60.82 C16.54 47.21, 29.29 32.52, 55.38 0.44 M2.89 60.82 C21.26 39.58, 40.35 18.48, 55.38 0.44 M4.6 64.95 C15.41 50.91, 27.49 37.54, 60.37 0.8 M4.6 64.95 C24.86 42.26, 42.85 21.07, 60.37 0.8 M8.28 66.82 C29.89 41.98, 53.28 14.09, 66.01 0.41 M8.28 66.82 C24.23 49.34, 37.68 31.63, 66.01 0.41 M11.95 68.69 C33.32 45.71, 53.49 24.36, 71 0.77 M11.95 68.69 C26.51 53.2, 40.39 35.25, 71 0.77 M15.63 70.56 C36.55 45.99, 60.82 20.49, 76.64 0.37 M15.63 70.56 C34.83 50.54, 52.83 28.79, 76.64 0.37 M20.61 70.92 C32.84 55.02, 48.58 39.16, 81.63 0.73 M20.61 70.92 C36.9 53.58, 53.19 33.96, 81.63 0.73 M26.26 70.52 C45.37 48.8, 60.88 32.83, 86.61 1.09 M26.26 70.52 C40.92 53.27, 54.07 37.49, 86.61 1.09 M31.24 70.88 C45.06 55.28, 56.46 43.11, 92.26 0.7 M31.24 70.88 C49.89 48.93, 67.41 27.07, 92.26 0.7 M36.89 70.49 C50.1 56.2, 64.46 39.43, 97.24 1.06 M36.89 70.49 C52.6 52.5, 67.53 36.55, 97.24 1.06 M41.87 70.85 C61.25 49.69, 77.93 30.38, 102.89 0.66 M41.87 70.85 C66.3 43.09, 90.7 15.04, 102.89 0.66 M47.52 70.46 C69.17 48.73, 87.05 22.69, 107.87 1.02 M47.52 70.46 C67.79 44.55, 89.93 21.19, 107.87 1.02 M52.5 70.82 C69.59 53.35, 85.54 33.86, 113.52 0.63 M52.5 70.82 C69.54 52.35, 84.88 31.86, 113.52 0.63 M58.15 70.42 C79.13 47.39, 99.99 20, 118.5 0.99 M58.15 70.42 C81.24 45, 103.25 17.02, 118.5 0.99 M63.13 70.78 C75.84 54.36, 89.36 41.44, 124.15 0.59 M63.13 70.78 C74.99 56.21, 88.54 41.12, 124.15 0.59 M68.78 70.39 C83.76 50.85, 103.67 29.57, 129.13 0.95 M68.78 70.39 C92.46 44.3, 114.68 18.14, 129.13 0.95 M73.76 70.75 C95.4 41.78, 121 15.96, 134.78 0.56 M73.76 70.75 C96.69 43.21, 121.22 15.89, 134.78 0.56 M78.75 71.11 C96.64 49.73, 114.24 26.51, 139.76 0.92 M78.75 71.11 C92.52 55.33, 105.28 40.26, 139.76 0.92 M84.39 70.71 C101.41 49.1, 120.73 32.84, 145.41 0.52 M84.39 70.71 C104.05 47.33, 125.39 25.31, 145.41 0.52 M89.38 71.07 C109.16 47.24, 126.58 28.02, 150.39 0.88 M89.38 71.07 C101.62 57.04, 113.33 43.38, 150.39 0.88 M95.02 70.68 C107.87 54.23, 126.2 35.82, 156.04 0.49 M95.02 70.68 C108.56 52.2, 124.7 36.51, 156.04 0.49 M100.01 71.04 C120.01 47.18, 137.27 24.35, 161.02 0.85 M100.01 71.04 C115.67 53.39, 129.47 35.2, 161.02 0.85 M105.65 70.64 C119.15 56.96, 132.74 38.63, 166.67 0.46 M105.65 70.64 C123.85 50.01, 143.15 27.62, 166.67 0.46 M110.64 71 C128.96 53.5, 141.38 33.14, 171.65 0.82 M110.64 71 C127.11 49.45, 144.93 30.29, 171.65 0.82 M116.28 70.61 C128.82 57.34, 140.59 43.25, 177.3 0.42 M116.28 70.61 C139.86 44.36, 163.08 16.74, 177.3 0.42 M121.27 70.97 C135.19 54.29, 151.18 37.19, 182.28 0.78 M121.27 70.97 C139.89 49.03, 158.83 29.41, 182.28 0.78 M126.91 70.57 C138.33 58.57, 149.47 43.04, 187.27 1.14 M126.91 70.57 C143.08 50.81, 159.43 32.59, 187.27 1.14 M131.9 70.93 C146.48 55.57, 157.18 40.25, 192.91 0.75 M131.9 70.93 C147.88 53.08, 162.49 35.93, 192.91 0.75 M137.54 70.54 C152.28 52.07, 170.3 35.15, 197.9 1.11 M137.54 70.54 C153.83 52, 172.35 30.79, 197.9 1.11 M142.53 70.9 C164.81 46.25, 189.91 18, 203.54 0.71 M142.53 70.9 C164.67 46.47, 185.54 20.58, 203.54 0.71 M148.17 70.51 C161.38 53.03, 179.38 34.88, 208.53 1.07 M148.17 70.51 C164.06 50.16, 182.95 31.62, 208.53 1.07 M153.16 70.87 C174.73 49.26, 191.67 23.22, 214.17 0.68 M153.16 70.87 C166.77 53.77, 182.22 37.34, 214.17 0.68 M158.8 70.47 C176.57 50.03, 193.1 32.96, 219.16 1.04 M158.8 70.47 C173 54.69, 186.22 40.8, 219.16 1.04 M163.79 70.83 C182.24 48.8, 196.42 30.95, 224.8 0.64 M163.79 70.83 C184.8 47.69, 206.07 23.66, 224.8 0.64 M169.43 70.44 C184.65 53.78, 199.39 36.16, 229.79 1 M169.43 70.44 C193.03 45.29, 214.73 19, 229.79 1 M174.42 70.8 C196.5 43.32, 218.93 21.45, 235.43 0.61 M174.42 70.8 C196.72 46.69, 217.43 22.35, 235.43 0.61 M179.41 71.16 C201.46 48.66, 219.18 22.82, 240.42 0.97 M179.41 71.16 C199.63 46.34, 222.44 21.38, 240.42 0.97 M185.05 70.76 C197.5 55.97, 209.97 42.87, 246.06 0.57 M185.05 70.76 C205.59 46.29, 226.52 22.7, 246.06 0.57 M190.04 71.12 C210.59 48.38, 228 29.24, 251.05 0.93 M190.04 71.12 C208.28 50.85, 227.13 29.37, 251.05 0.93 M195.68 70.73 C217.4 45.96, 243.14 20.23, 256.69 0.54 M195.68 70.73 C211.19 52.81, 226.41 36.25, 256.69 0.54 M200.67 71.09 C222.5 42.46, 248.92 14.89, 261.68 0.9 M200.67 71.09 C216.68 54.51, 231.79 36.56, 261.68 0.9 M206.31 70.69 C227.98 44.54, 252.24 19.85, 267.33 0.5 M206.31 70.69 C230.22 43.62, 253.84 15.01, 267.33 0.5 M211.3 71.05 C222.39 55.06, 238.62 40.23, 272.31 0.86 M211.3 71.05 C233.64 44.26, 256.75 18.55, 272.31 0.86 M216.94 70.66 C230.22 53.86, 246.5 35.87, 277.96 0.47 M216.94 70.66 C237.51 45.41, 257.17 21.93, 277.96 0.47 M221.93 71.02 C238.82 53.57, 253.71 32.84, 282.94 0.83 M221.93 71.02 C245.28 43.7, 269.48 16.15, 282.94 0.83 M227.57 70.62 C247.27 47.31, 267.14 24.87, 287.93 1.19 M227.57 70.62 C243.82 51.34, 259.48 33.98, 287.93 1.19 M232.56 70.98 C244.77 58.13, 258.12 41.2, 293.57 0.8 M232.56 70.98 C250.25 50.92, 266.62 31.06, 293.57 0.8 M238.2 70.59 C257.36 51.45, 276.09 29.04, 298.56 1.16 M238.2 70.59 C260.89 44.35, 283.62 16.82, 298.56 1.16 M243.19 70.95 C267.41 45.44, 290.28 20.27, 304.2 0.76 M243.19 70.95 C255.33 56.54, 269.35 39.98, 304.2 0.76 M248.83 70.55 C271.36 41.43, 297.41 16.94, 309.19 1.12 M248.83 70.55 C266.52 48.57, 283.97 29.21, 309.19 1.12 M253.82 70.91 C272.92 47.71, 294.55 24.71, 314.83 0.73 M253.82 70.91 C268.81 53.75, 282.62 36.64, 314.83 0.73 M259.46 70.52 C278 46.18, 302.74 21.44, 319.82 1.09 M259.46 70.52 C277.04 50.96, 292.19 32.3, 319.82 1.09 M264.45 70.88 C278.88 54.09, 294.99 36.02, 325.46 0.69 M264.45 70.88 C283.88 46.31, 304.51 23.82, 325.46 0.69 M270.09 70.49 C284.21 56.63, 299.67 40.68, 330.45 1.05 M270.09 70.49 C292.02 44.76, 315.44 18.87, 330.45 1.05 M275.08 70.85 C296.05 48.4, 316.6 26.4, 336.09 0.66 M275.08 70.85 C288.07 57.03, 299.46 41.88, 336.09 0.66 M280.72 70.45 C293.46 54.8, 310.09 40.99, 341.08 1.02 M280.72 70.45 C297.01 51.33, 315.02 31.09, 341.08 1.02 M285.71 70.81 C298.81 56.28, 311.07 40, 346.72 0.62 M285.71 70.81 C303.22 50.24, 321.28 31.06, 346.72 0.62 M290.7 71.17 C306.68 50.96, 323.13 33.59, 351.71 0.98 M290.7 71.17 C308.79 50.07, 328.21 26.77, 351.71 0.98 M296.34 70.78 C320.95 43.12, 342.03 16.62, 357.35 0.59 M296.34 70.78 C314.9 49.07, 333.11 26.95, 357.35 0.59 M301.33 71.14 C312.27 55.94, 324.44 41.91, 362.34 0.95 M301.33 71.14 C326.15 42.92, 349.94 16.09, 362.34 0.95 M306.97 70.74 C324.43 49.11, 344 29.03, 367.98 0.55 M306.97 70.74 C329.6 47.49, 349.67 22.41, 367.98 0.55 M311.96 71.1 C327.28 53.66, 338.84 40.98, 372.97 0.91 M311.96 71.1 C329.5 49.43, 347.35 29.85, 372.97 0.91 M317.6 70.71 C334.82 49.28, 351.63 28.78, 378.61 0.52 M317.6 70.71 C330.29 56.05, 342.19 43.4, 378.61 0.52 M322.59 71.07 C337.71 54.18, 353.3 37.16, 383.6 0.88 M322.59 71.07 C345.25 45.36, 368.36 16.58, 383.6 0.88 M328.23 70.67 C346.71 48.57, 367.89 25.37, 389.24 0.49 M328.23 70.67 C351.32 42.96, 374.3 15.88, 389.24 0.49 M333.22 71.03 C348.91 51.72, 368.03 34.01, 394.23 0.85 M333.22 71.03 C349.74 52.86, 366.33 34.84, 394.23 0.85 M338.86 70.64 C361.04 48.42, 381.6 21.12, 399.22 1.21 M338.86 70.64 C358.09 48.64, 377.73 24.92, 399.22 1.21 M343.85 71 C362.86 50.37, 381.44 28, 404.86 0.81 M343.85 71 C366.8 45.56, 387.28 21.97, 404.86 0.81 M349.49 70.6 C374.02 42.11, 397.21 17.45, 409.85 1.17 M349.49 70.6 C371.16 43.99, 395.19 17.68, 409.85 1.17 M354.48 70.96 C368.95 55.24, 380.95 39.66, 415.49 0.78 M354.48 70.96 C372.19 51.16, 390.66 29, 415.49 0.78 M360.12 70.57 C377.3 52.58, 393.62 32.12, 420.48 1.14 M360.12 70.57 C374.55 55.07, 388.03 39.59, 420.48 1.14 M365.11 70.93 C378.55 55.99, 394.02 37.04, 426.12 0.74 M365.11 70.93 C385.81 45.21, 407.01 20.15, 426.12 0.74 M370.75 70.54 C385.32 51.03, 404.56 31.17, 431.11 1.1 M370.75 70.54 C386.4 52.63, 402.58 33.33, 431.11 1.1 M375.74 70.9 C395.62 49, 414.09 24.01, 436.75 0.71 M375.74 70.9 C389.06 55.73, 402.09 39.72, 436.75 0.71 M381.38 70.5 C399.3 53.03, 414.06 32.84, 441.74 1.07 M381.38 70.5 C394.41 54.81, 408.77 39.07, 441.74 1.07 M386.37 70.86 C407.35 47.63, 427.44 20.42, 447.38 0.67 M386.37 70.86 C410.86 42.1, 434.34 14.64, 447.38 0.67 M391.35 71.22 C413.15 45.02, 434.9 20.12, 452.37 1.03 M391.35 71.22 C413.81 47.06, 434.81 22.78, 452.37 1.03 M397 70.83 C414.91 51.52, 436.16 29.47, 458.01 0.64 M397 70.83 C410.25 56.74, 423.42 40.6, 458.01 0.64 M401.98 71.19 C420.8 49.64, 442.65 24.89, 463 1 M401.98 71.19 C414.39 55.25, 426.74 41.84, 463 1 M407.63 70.79 C427.08 47.62, 451.77 22.62, 468.64 0.6 M407.63 70.79 C420.38 55.36, 434.68 37.81, 468.64 0.6 M412.61 71.15 C429.53 48.95, 450.9 29.79, 473.63 0.96 M412.61 71.15 C430.53 50.25, 446.86 30.46, 473.63 0.96 M418.26 70.76 C438.23 49.21, 456.77 22.78, 479.27 0.57 M418.26 70.76 C430.34 56.18, 442.91 40.61, 479.27 0.57 M423.24 71.12 C443.14 49.48, 465.05 24.22, 484.26 0.93 M423.24 71.12 C447.39 45.38, 469.74 18.05, 484.26 0.93 M428.89 70.72 C444.93 53.77, 457.72 38.05, 489.9 0.54 M428.89 70.72 C451.72 46.08, 474.14 19.38, 489.9 0.54 M433.87 71.08 C447.78 54.85, 460.43 40.72, 494.89 0.9 M433.87 71.08 C447.9 53.75, 463.91 38.31, 494.89 0.9 M439.52 70.69 C452.24 56.33, 463.29 43.23, 499.87 1.26 M439.52 70.69 C457.25 49.87, 475.08 30.24, 499.87 1.26 M444.5 71.05 C456.65 56.85, 467.47 39.94, 505.52 0.86 M444.5 71.05 C462.66 48.9, 482.69 27.66, 505.52 0.86 M450.15 70.65 C466.34 52.13, 481.31 35.99, 510.5 1.22 M450.15 70.65 C464.56 52.25, 480.9 35.84, 510.5 1.22 M455.13 71.01 C466.71 54.71, 479.72 42.08, 516.15 0.83 M455.13 71.01 C476.84 45.03, 499.51 17.82, 516.15 0.83 M460.78 70.62 C485.48 45.88, 508.88 18.92, 521.14 1.19 M460.78 70.62 C478.94 48.74, 495.84 27.97, 521.14 1.19 M465.76 70.98 C486.94 48.64, 506.66 22.32, 526.78 0.79 M465.76 70.98 C484.69 48.1, 505.38 24.72, 526.78 0.79 M471.41 70.59 C490.02 49.79, 506.99 30.79, 531.77 1.15 M471.41 70.59 C489.34 48.03, 508.33 26.66, 531.77 1.15 M476.39 70.95 C496.71 48.42, 515.46 25.49, 537.41 0.76 M476.39 70.95 C499.78 44.57, 521.25 19.16, 537.41 0.76 M482.04 70.55 C506.46 43.88, 528.69 17.68, 542.4 1.12 M482.04 70.55 C506.27 43.7, 530.7 16.44, 542.4 1.12 M487.02 70.91 C506.4 47.09, 528.41 23.65, 548.04 0.72 M487.02 70.91 C503.82 51.11, 522.02 32.72, 548.04 0.72 M492.01 71.27 C507.98 54.49, 523.29 34.25, 553.03 1.08 M492.01 71.27 C508.15 50.39, 525.61 31.93, 553.03 1.08 M497.65 70.88 C520.98 43.99, 545.95 17.2, 558.67 0.69 M497.65 70.88 C515.23 50.47, 534.17 31.48, 558.67 0.69 M502.64 71.24 C527.47 42.54, 553.36 15.86, 563.66 1.05 M502.64 71.24 C522.76 49.44, 541.02 26.91, 563.66 1.05 M508.28 70.84 C525.03 51.52, 543.58 29.7, 569.3 0.65 M508.28 70.84 C533.35 43.62, 557.09 14.17, 569.3 0.65 M513.27 71.2 C535.12 43.41, 560.43 14.54, 574.29 1.01 M513.27 71.2 C530.63 52.71, 546.26 32.25, 574.29 1.01 M518.92 70.81 C539.73 47.81, 557.56 26.36, 579.93 0.62 M518.92 70.81 C537.62 50.52, 556.35 27.13, 579.93 0.62 M523.9 71.17 C537.3 57.82, 547.97 42.93, 584.92 0.98 M523.9 71.17 C546.03 47.1, 566.85 21.6, 584.92 0.98 M529.55 70.77 C551.04 47.67, 568.26 25.17, 590.56 0.58 M529.55 70.77 C545.03 51.97, 559.88 35.84, 590.56 0.58 M534.53 71.13 C554.79 50.59, 572.1 30.68, 595.55 0.94 M534.53 71.13 C555.56 45.55, 578.56 22.14, 595.55 0.94 M540.18 70.74 C556.98 52.75, 571.61 35.2, 600.53 1.31 M540.18 70.74 C562.82 45.71, 585.85 19.49, 600.53 1.31 M545.16 71.1 C562.76 51.37, 580.1 35.48, 606.18 0.91 M545.16 71.1 C564.7 49.31, 584.62 26.84, 606.18 0.91 M550.81 70.7 C562.69 54.69, 575.12 40.88, 611.16 1.27 M550.81 70.7 C571.32 47.14, 594.11 20.68, 611.16 1.27 M555.79 71.06 C569.58 53.86, 586.7 36.02, 616.81 0.88 M555.79 71.06 C575.08 50.38, 594.69 26.84, 616.81 0.88 M561.44 70.67 C586.19 43.7, 606.24 18.03, 621.79 1.24 M561.44 70.67 C584.55 44.72, 608.24 18.29, 621.79 1.24 M566.42 71.03 C587.04 47.26, 603.65 25.49, 627.44 0.84 M566.42 71.03 C580.24 56.73, 592.51 40.76, 627.44 0.84 M572.07 70.63 C594.73 46.46, 617.32 20.4, 632.42 1.2 M572.07 70.63 C594.82 43.89, 618.49 18.35, 632.42 1.2 M577.05 70.99 C589.7 53.85, 606.9 39.3, 638.07 0.81 M577.05 70.99 C596.24 48.62, 617.07 24.92, 638.07 0.81 M582.7 70.6 C607.76 43.02, 628.8 17.49, 643.05 1.17 M582.7 70.6 C605.08 45.12, 628.09 18.45, 643.05 1.17 M587.68 70.96 C606.74 49.6, 625.23 27.44, 648.7 0.77 M587.68 70.96 C609.86 46.82, 631.79 21.71, 648.7 0.77 M592.67 71.32 C609.08 52.03, 624.92 33.42, 652.37 2.64 M592.67 71.32 C613.56 46.81, 633.64 22.77, 652.37 2.64 M598.31 70.93 C619.66 44.66, 643.68 18.64, 656.05 4.51 M598.31 70.93 C614.69 52.14, 630.59 34.02, 656.05 4.51 M603.3 71.29 C624.01 51.03, 642.62 28.17, 659.72 6.38 M603.3 71.29 C623.99 47.91, 643.05 25.15, 659.72 6.38 M608.94 70.89 C621.49 57.24, 635.41 41.38, 661.43 10.51 M608.94 70.89 C620.96 56.72, 632.13 43.65, 661.43 10.51 M613.93 71.25 C627.12 58.45, 640.33 43.47, 661.17 16.91 M613.93 71.25 C624.38 59.07, 634.21 47.83, 661.17 16.91 M619.57 70.86 C636.04 54.29, 652.45 35.02, 661.56 22.56 M619.57 70.86 C628.23 61.01, 638.76 50.25, 661.56 22.56 M624.56 71.22 C637.65 55.25, 654.28 37.15, 661.3 28.95 M624.56 71.22 C634.86 60.48, 643.46 49.8, 661.3 28.95 M630.2 70.82 C643.19 56.7, 656.81 41.57, 661.69 34.6 M630.2 70.82 C638.48 61.94, 646.64 52.23, 661.69 34.6 M635.19 71.18 C644.66 63.69, 649.59 54.59, 661.43 40.99 M635.19 71.18 C643.34 62.31, 650.6 52.44, 661.43 40.99 M640.83 70.79 C646.55 62.34, 652.91 58.27, 661.83 46.64 M640.83 70.79 C649.27 61.51, 657.03 51.76, 661.83 46.64 M645.82 71.15 C650.49 62.65, 657.76 55.89, 662.22 52.28 M645.82 71.15 C651.07 65.36, 657.91 57.42, 662.22 52.28" stroke="#a5d8ff" stroke-width="0.5" fill="none"></path><path d="M17.25 0 M17.25 0 C197.18 1.66, 376.91 1.87, 643.75 0 M17.25 0 C169.99 -2.43, 323 -2.67, 643.75 0 M643.75 0 C653.64 0.3, 660.75 4.09, 661 17.25 M643.75 0 C654.45 -1.68, 662.81 5.67, 661 17.25 M661 17.25 C662.29 22.82, 660.07 30.53, 661 51.75 M661 17.25 C661.53 28.03, 661.03 38.69, 661 51.75 M661 51.75 C661.59 63.97, 655.27 69.18, 643.75 69 M661 51.75 C660.27 64.8, 655.28 66.77, 643.75 69 M643.75 69 C513.36 68.02, 383.28 69.07, 17.25 69 M643.75 69 C513.03 70.26, 382.81 70.59, 17.25 69 M17.25 69 C4.19 70.68, 1.69 62.73, 0 51.75 M17.25 69 C5.42 68.96, -1.57 65.43, 0 51.75 M0 51.75 C-0.62 42.87, 0.31 33.59, 0 17.25 M0 51.75 C0.89 43.08, 0.02 32.28, 0 17.25 M0 17.25 C0.69 4.47, 4.81 0.42, 17.25 0 M0 17.25 C-1.38 4.14, 4.23 -1.08, 17.25 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(1070.6871356062713 644.5638004558205) rotate(0 65.56995391845703 12.5)"><text x="65.56995391845703" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">NRI Adaption</text></g><g stroke-linecap="round" transform="translate(779.1312811557848 438.2528256168371) rotate(0 183.5 35)"><path d="M4.66 4.05 C4.66 4.05, 4.66 4.05, 4.66 4.05 M4.66 4.05 C4.66 4.05, 4.66 4.05, 4.66 4.05 M3.08 11.96 C6.06 7.97, 7.43 5.63, 11.61 2.15 M3.08 11.96 C6.71 8.15, 8.87 4.44, 11.61 2.15 M2.17 19.11 C7.22 13.46, 16.48 5.1, 17.91 1 M2.17 19.11 C6.59 15.61, 9.61 10.38, 17.91 1 M1.91 25.51 C6.98 21.73, 10.02 15.09, 23.56 0.6 M1.91 25.51 C11.31 15.57, 18.89 5.71, 23.56 0.6 M2.3 31.15 C11.5 18.71, 23.75 10.3, 28.54 0.96 M2.3 31.15 C9.25 23.46, 15.5 15.89, 28.54 0.96 M2.69 36.79 C12.45 24.08, 27.14 8.82, 34.19 0.57 M2.69 36.79 C12.14 27.69, 19.69 17.43, 34.19 0.57 M2.43 43.19 C12.24 31.57, 22.52 20.99, 39.17 0.93 M2.43 43.19 C12.43 29.71, 24.62 18.75, 39.17 0.93 M2.83 48.83 C21 29.25, 34.5 9.21, 44.82 0.53 M2.83 48.83 C15.81 34.95, 27.52 20.17, 44.82 0.53 M3.22 54.48 C14.66 41.09, 30.75 25.04, 49.8 0.89 M3.22 54.48 C12.47 43.91, 22.08 32.36, 49.8 0.89 M2.31 61.63 C19.11 43.93, 35.44 23.01, 55.45 0.5 M2.31 61.63 C17.73 44.5, 31.03 28.31, 55.45 0.5 M5.32 64.25 C23.2 43.38, 40.16 23.98, 60.43 0.86 M5.32 64.25 C21.85 44.73, 39.57 25.88, 60.43 0.86 M7.69 67.63 C30.51 40.85, 50.08 17.41, 66.08 0.46 M7.69 67.63 C28.53 41.83, 51.85 17.2, 66.08 0.46 M11.36 69.5 C26.06 55.53, 36.35 39.38, 71.06 0.82 M11.36 69.5 C31.11 45.41, 51.07 22.8, 71.06 0.82 M15.04 71.37 C34.39 50.84, 49.72 29.96, 76.71 0.43 M15.04 71.37 C31.17 52.16, 46.88 32.55, 76.71 0.43 M20.68 70.98 C34.37 55.55, 45.47 41.15, 81.69 0.79 M20.68 70.98 C35.09 53.71, 49.03 37.59, 81.69 0.79 M25.67 71.34 C47.34 48.4, 67.24 20.56, 87.34 0.39 M25.67 71.34 C44.95 48.22, 65.2 25.66, 87.34 0.39 M31.31 70.94 C53.12 47.42, 74.63 22.49, 92.32 0.75 M31.31 70.94 C53.91 43.46, 77.61 16.29, 92.32 0.75 M36.3 71.3 C58.96 45.81, 77.44 22.91, 97.97 0.36 M36.3 71.3 C53.74 52.44, 70.21 31.65, 97.97 0.36 M41.94 70.91 C62.45 47.83, 82.71 25.16, 102.95 0.72 M41.94 70.91 C66.42 43.04, 91.11 15.18, 102.95 0.72 M46.93 71.27 C59.41 53.68, 74.44 40.43, 107.94 1.08 M46.93 71.27 C69.3 46.96, 90.94 20.98, 107.94 1.08 M51.91 71.63 C68.05 52.97, 84.2 35.46, 113.58 0.69 M51.91 71.63 C68.18 50.69, 87.05 30.06, 113.58 0.69 M57.56 71.23 C81.54 41.17, 107.76 14.32, 118.57 1.05 M57.56 71.23 C76.11 47.63, 95.34 25.97, 118.57 1.05 M62.54 71.59 C80.27 52.43, 95.37 32.81, 124.21 0.65 M62.54 71.59 C80.74 50.94, 98.9 31.1, 124.21 0.65 M68.19 71.2 C88.29 48.68, 107.02 28.02, 129.2 1.01 M68.19 71.2 C82.47 54.25, 97.34 37.85, 129.2 1.01 M73.17 71.56 C98.07 43.97, 123.26 19.09, 134.84 0.62 M73.17 71.56 C90.84 51.94, 107.82 33.01, 134.84 0.62 M78.82 71.16 C101.23 42.72, 125.47 16.65, 139.83 0.98 M78.82 71.16 C97.98 48.88, 116.76 26.36, 139.83 0.98 M83.8 71.52 C106.23 50.2, 125.79 22.64, 145.47 0.58 M83.8 71.52 C105.79 48.36, 125.95 24.58, 145.47 0.58 M89.45 71.13 C110.53 47.45, 131.85 21.16, 150.46 0.94 M89.45 71.13 C106.14 51.85, 123.16 32.06, 150.46 0.94 M94.43 71.49 C109.6 55.59, 119.15 44.34, 156.1 0.55 M94.43 71.49 C118.42 46.2, 140.96 19.57, 156.1 0.55 M100.08 71.1 C118.78 46.28, 139.41 25.7, 161.09 0.91 M100.08 71.1 C121.98 43.96, 144.48 18.35, 161.09 0.91 M105.06 71.46 C117.68 55.15, 130.09 40.54, 166.73 0.51 M105.06 71.46 C125.13 48.16, 144.46 25.86, 166.73 0.51 M110.71 71.06 C122.95 52.94, 139.33 38.81, 171.72 0.87 M110.71 71.06 C132.66 46.65, 154.61 19.85, 171.72 0.87 M115.69 71.42 C131.01 53.59, 149.65 34.57, 177.36 0.48 M115.69 71.42 C135.98 47.57, 154.36 25.27, 177.36 0.48 M121.34 71.03 C139.01 51.36, 153.7 33.2, 182.35 0.84 M121.34 71.03 C142.44 48.64, 161.16 24.85, 182.35 0.84 M126.32 71.39 C139.88 53.85, 156.15 35.09, 187.99 0.44 M126.32 71.39 C149.05 45.43, 170.65 20.88, 187.99 0.44 M131.97 70.99 C142.34 56.9, 155.32 41.13, 192.98 0.8 M131.97 70.99 C149.51 50.44, 167.94 29.89, 192.98 0.8 M136.95 71.35 C159.77 48.46, 179.39 23.06, 198.62 0.41 M136.95 71.35 C152.58 53.58, 167.47 36.27, 198.62 0.41 M141.94 71.71 C161.85 49.64, 178.84 31.31, 203.61 0.77 M141.94 71.71 C165.37 43.52, 188.73 16.69, 203.61 0.77 M147.58 71.32 C165.9 51.54, 181.58 32.74, 208.6 1.13 M147.58 71.32 C171.13 46.91, 192.4 21.22, 208.6 1.13 M152.57 71.68 C173.89 43.09, 197.15 18.55, 214.24 0.74 M152.57 71.68 C169.38 53.22, 184.17 36.31, 214.24 0.74 M158.21 71.28 C176.77 51.99, 191.33 29.07, 219.23 1.1 M158.21 71.28 C181.36 45.98, 202.84 20.57, 219.23 1.1 M163.2 71.64 C182.86 47.61, 207.28 22.13, 224.87 0.7 M163.2 71.64 C180.84 52.17, 196.9 32.28, 224.87 0.7 M168.84 71.25 C190.72 46.6, 213.78 18.82, 229.86 1.06 M168.84 71.25 C188.87 47.38, 209.48 25.03, 229.86 1.06 M173.83 71.61 C195.2 50.09, 216.25 23.81, 235.5 0.67 M173.83 71.61 C195.35 46.89, 217.2 21.13, 235.5 0.67 M179.47 71.21 C201.32 42.98, 227.12 18.37, 240.49 1.03 M179.47 71.21 C200.73 47.87, 218.97 24.76, 240.49 1.03 M184.46 71.57 C201.05 53.09, 218.16 35.21, 246.13 0.63 M184.46 71.57 C202.81 50.81, 221.81 28.88, 246.13 0.63 M190.1 71.18 C201.55 58.97, 214.55 41.37, 251.12 0.99 M190.1 71.18 C207.13 51.77, 223.22 32.99, 251.12 0.99 M195.09 71.54 C216.31 50.75, 232.93 25.35, 256.76 0.6 M195.09 71.54 C211.17 53.64, 228.39 35.97, 256.76 0.6 M200.73 71.15 C224.39 45.24, 245.74 17.18, 261.75 0.96 M200.73 71.15 C217.39 51.73, 233.78 32.69, 261.75 0.96 M205.72 71.51 C222.79 53.84, 241.5 32.27, 267.39 0.56 M205.72 71.51 C223.5 49.68, 241.9 29.97, 267.39 0.56 M211.37 71.11 C235.61 45.76, 259.71 17.72, 272.38 0.92 M211.37 71.11 C228.72 51.59, 246.27 31.54, 272.38 0.92 M216.35 71.47 C240.43 44.04, 263.31 20.55, 278.02 0.53 M216.35 71.47 C239.73 43.57, 264.33 15.36, 278.02 0.53 M222 71.08 C235.59 52.59, 252.31 35.91, 283.01 0.89 M222 71.08 C240.27 49.93, 258.54 29.42, 283.01 0.89 M226.98 71.44 C244.17 51.11, 259.54 33.4, 288.65 0.49 M226.98 71.44 C242.53 53.84, 258.26 35.22, 288.65 0.49 M231.97 71.8 C250.63 49.69, 267.57 31.33, 293.64 0.85 M231.97 71.8 C248.24 52.23, 266.03 32.36, 293.64 0.85 M237.61 71.4 C253.83 53.71, 268.82 36.58, 298.63 1.21 M237.61 71.4 C257.14 47.6, 278.14 25.24, 298.63 1.21 M242.6 71.76 C256.19 58.25, 267.02 42.66, 304.27 0.82 M242.6 71.76 C259.15 52.31, 276.28 34.84, 304.27 0.82 M248.24 71.37 C262.47 58.24, 274.58 43.36, 309.26 1.18 M248.24 71.37 C268.1 48.87, 289.57 24.41, 309.26 1.18 M253.23 71.73 C266.49 56.99, 277.9 42.23, 314.9 0.78 M253.23 71.73 C269.25 51.96, 287.46 34.09, 314.9 0.78 M258.87 71.33 C280.07 49.26, 296.67 27.73, 319.89 1.14 M258.87 71.33 C280.26 45.77, 302.61 18.71, 319.89 1.14 M263.86 71.69 C288.56 43.52, 313.68 16.52, 325.53 0.75 M263.86 71.69 C287.01 45.88, 310.15 17.93, 325.53 0.75 M269.5 71.3 C289.81 51.13, 307.59 27.34, 330.52 1.11 M269.5 71.3 C283.54 55.71, 296.66 41.74, 330.52 1.11 M274.49 71.66 C290.42 56.01, 301.38 38.24, 336.16 0.72 M274.49 71.66 C288.74 53.54, 304.42 34.76, 336.16 0.72 M280.13 71.26 C306.07 45.94, 327.17 17.54, 341.15 1.08 M280.13 71.26 C305.41 42.84, 327.64 16.2, 341.15 1.08 M285.12 71.62 C304.07 48.48, 323.57 25.92, 346.79 0.68 M285.12 71.62 C310.46 44.02, 333.76 15.31, 346.79 0.68 M290.76 71.23 C314.63 44.42, 337.94 18.42, 351.78 1.04 M290.76 71.23 C312.21 45.76, 333.63 21.85, 351.78 1.04 M295.75 71.59 C316.01 46.2, 337.1 25.71, 356.11 2.16 M295.75 71.59 C315.98 47.34, 335.67 24.11, 356.11 2.16 M301.39 71.19 C320.94 47.12, 345.36 22.67, 360.44 3.27 M301.39 71.19 C314.22 56.73, 327.21 40.99, 360.44 3.27 M306.38 71.56 C328.9 46.93, 349.36 22.82, 363.46 5.9 M306.38 71.56 C318.4 56.48, 331.05 40.73, 363.46 5.9 M312.02 71.16 C330.54 50.62, 350.79 26.89, 365.82 9.27 M312.02 71.16 C328.45 52.7, 345.65 33.64, 365.82 9.27 M317.01 71.52 C334.44 49.69, 350.92 30.14, 368.84 11.9 M317.01 71.52 C328.14 59.43, 339.72 43.98, 368.84 11.9 M322.65 71.13 C339.39 52.71, 354.92 37.37, 368.58 18.3 M322.65 71.13 C332.01 58.43, 343.01 48.64, 368.58 18.3 M327.64 71.49 C345.43 52.51, 360.94 35.26, 368.97 23.94 M327.64 71.49 C340.55 55.48, 354.26 40.17, 368.97 23.94 M332.63 71.85 C348.28 57.11, 360.45 39.45, 368.71 30.34 M332.63 71.85 C342.86 61.55, 352.07 50.06, 368.71 30.34 M338.27 71.45 C348.16 59.38, 356.99 48.19, 369.1 35.98 M338.27 71.45 C348.46 58.43, 359.21 48.32, 369.1 35.98 M343.26 71.81 C352.34 62.02, 359.72 51.88, 368.84 42.38 M343.26 71.81 C350.37 63.71, 359.41 55.22, 368.84 42.38 M349.56 70.66 C355.21 66.36, 360.86 59.28, 369.24 48.02 M349.56 70.66 C356.03 63.18, 360.45 56.86, 369.24 48.02 M355.86 69.51 C358.34 65.25, 361.62 61.81, 367.01 56.68 M355.86 69.51 C359.07 65.12, 362.41 61.41, 367.01 56.68" stroke="#a5d8ff" stroke-width="0.5" fill="none"></path><path d="M17.5 0 M17.5 0 C123.83 -1.35, 230.56 -2.4, 349.5 0 M17.5 0 C101.4 0.51, 183.74 0.45, 349.5 0 M349.5 0 C360.99 -1.6, 368.98 4.95, 367 17.5 M349.5 0 C361.17 1.24, 368.04 6.77, 367 17.5 M367 17.5 C365.12 25.55, 367.98 34.8, 367 52.5 M367 17.5 C366.66 24.52, 367.08 32.08, 367 52.5 M367 52.5 C366.19 65.32, 360.47 69.65, 349.5 70 M367 52.5 C367.58 63.53, 359.65 69.83, 349.5 70 M349.5 70 C240.18 68.83, 132.4 68.71, 17.5 70 M349.5 70 C257.28 67.19, 166.01 68.19, 17.5 70 M17.5 70 C7.33 69.89, -0.18 62.76, 0 52.5 M17.5 70 C7.65 70.08, -0.66 64.81, 0 52.5 M0 52.5 C-2.16 42.66, -0.94 31.45, 0 17.5 M0 52.5 C-0.39 45.58, 0.3 36.76, 0 17.5 M0 17.5 C-1.01 4.51, 7.17 0.63, 17.5 0 M0 17.5 C1.38 5.43, 5.71 -1.11, 17.5 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(903.0813238803942 460.7528256168371) rotate(0 59.549957275390625 12.5)"><text x="59.549957275390625" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">NRI convert</text></g><g stroke-linecap="round" transform="translate(776.38983126822 215.37868949510266) rotate(0 103.5 27)"><path d="M13.5 0 M13.5 0 C52.09 0.21, 92.12 -1.92, 193.5 0 M13.5 0 C66.65 0.56, 119.69 0.95, 193.5 0 M193.5 0 C202.26 0.96, 205.07 3.9, 207 13.5 M193.5 0 C201 1.42, 206.11 6.04, 207 13.5 M207 13.5 C205.79 25.45, 205.36 32.58, 207 40.5 M207 13.5 C207.14 19.15, 206.49 27.4, 207 40.5 M207 40.5 C208.77 50.66, 203.47 52.56, 193.5 54 M207 40.5 C204.78 51.06, 202.41 54.97, 193.5 54 M193.5 54 C129.94 52.15, 60.91 54.87, 13.5 54 M193.5 54 C143.28 53.86, 91.16 53.49, 13.5 54 M13.5 54 C3.34 55.67, 1.82 48.77, 0 40.5 M13.5 54 C3.95 53.5, -0.07 49.96, 0 40.5 M0 40.5 C1.73 33.98, -1.79 28.8, 0 13.5 M0 40.5 C-0.5 29.99, 0.71 21.36, 0 13.5 M0 13.5 C-1.05 5.15, 3.99 0.95, 13.5 0 M0 13.5 C-0.62 4.42, 4.36 0.14, 13.5 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(803.7099072569895 229.87868949510266) rotate(0 76.17992401123047 12.5)"><text x="76.17992401123047" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">excution module</text></g><g mask="url(#mask-gFVrjJq6eWhtjg8YcFqka)" stroke-linecap="round"><g transform="translate(1014.7149737796883 510.6391138805511) rotate(0 1.9869564054638431 51.26554403084492)"><path d="M0.58 -0.12 C0.86 17.01, 1.94 85.24, 2.19 102.24 M-0.57 -1.22 C0.06 16.14, 3.68 86.38, 4.54 103.75" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(1014.7149737796883 510.6391138805511) rotate(0 1.9869564054638431 51.26554403084492)"><path d="M-7.32 74.96 C-3.33 84.25, 0.08 94.9, 4.46 103.35 M-8.09 75.82 C-3.9 85.93, 0.27 95.24, 5.32 103.44" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(1014.7149737796883 510.6391138805511) rotate(0 1.9869564054638431 51.26554403084492)"><path d="M13.17 73.93 C10.14 83.67, 6.53 94.67, 4.46 103.35 M12.4 74.78 C9.73 85.34, 7.04 95, 5.32 103.44" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask id="mask-gFVrjJq6eWhtjg8YcFqka"><rect x="0" y="0" fill="#fff" width="1117.599775678401" height="712.9373584136829"></rect><rect x="960.757418979533" y="549.2882361471165" fill="#000" width="110.79991149902344" height="25" opacity="1"></rect></mask><g transform="translate(960.757418979533 549.2882361471165) rotate(0 55.944511205619165 12.616421764279494)"><text x="55.39995574951172" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">NRI struct</text></g><g stroke-linecap="round" transform="translate(33.69590449094301 671.3215654615151) rotate(0 278.57139732724147 100.43218458733327)"><path d="M32 0 M32 0 C203.9 0.65, 375.6 1.13, 525.14 0 M525.14 0 C546.39 -0.41, 557.84 10.43, 557.14 32 M557.14 32 C554.52 80.86, 555.05 128.59, 557.14 168.86 M557.14 168.86 C558.7 189.57, 545.18 199.74, 525.14 200.86 M525.14 200.86 C333.47 200.05, 143.59 199.51, 32 200.86 M32 200.86 C9.01 199.51, 1.87 191.97, 0 168.86 M0 168.86 C-2.59 121.24, -0.75 71.75, 0 32 M0 32 C1.28 10.14, 10.38 1.98, 32 0" stroke="#1e1e1e" stroke-width="1.5" fill="none" stroke-dasharray="8 9"></path></g><g transform="translate(54.112824177100265 679.7948966336871) rotate(0 18.66999053955078 12.5)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">NRI</text></g><g stroke-linecap="round" transform="translate(73.19156283841824 702.6022058658982) rotate(0 112 77)"><path d="M32 0 M32 0 C69.76 -0.34, 108.38 -0.17, 192 0 M32 0 C87.03 1.1, 142.97 1.15, 192 0 M192 0 C214.89 -0.63, 222.7 9.55, 224 32 M192 0 C215.35 -1.79, 222.49 9.29, 224 32 M224 32 C223.22 63.36, 226.76 99.21, 224 122 M224 32 C223.47 64.88, 224.4 96.83, 224 122 M224 122 C225.28 142.81, 213.04 155.98, 192 154 M224 122 C222.17 141.6, 214.64 153.62, 192 154 M192 154 C151.8 154.5, 110.84 154.08, 32 154 M192 154 C130.41 153.2, 69.28 153.62, 32 154 M32 154 C9.27 155.17, 1.45 145.22, 0 122 M32 154 C10.32 152.13, 0.84 144.86, 0 122 M0 122 C-1.02 100.79, 1.51 76.25, 0 32 M0 122 C-0.04 92.04, 0.17 62.22, 0 32 M0 32 C0.9 10.61, 11.39 0.87, 32 0 M0 32 C-1.77 8.93, 11.81 1.96, 32 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(83.68165988431667 729.6022058658982) rotate(0 101.50990295410156 50)"><text x="101.50990295410156" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">pre-installed plugin A</text><text x="101.50990295410156" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">iSulad child </text><text x="101.50990295410156" y="50" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">process，exec by </text><text x="101.50990295410156" y="75" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">iSulad</text></g><g stroke-linecap="round" transform="translate(376.0943065508013 746.5292048024885) rotate(0 91 42.5)"><path d="M21.25 0 M21.25 0 C63.18 0.05, 104.67 -1.65, 160.75 0 M21.25 0 C58.51 -0.19, 97.32 -1.17, 160.75 0 M160.75 0 C173.62 -1.12, 183.75 5.53, 182 21.25 M160.75 0 C173.41 -1.38, 183.46 9.34, 182 21.25 M182 21.25 C183.52 35.63, 182.8 47.45, 182 63.75 M182 21.25 C181.55 31.36, 182.23 40.36, 182 63.75 M182 63.75 C181.71 79.89, 173.33 83.49, 160.75 85 M182 63.75 C183.3 77.53, 173.8 84.79, 160.75 85 M160.75 85 C107.47 83.33, 57.23 86.31, 21.25 85 M160.75 85 C123.98 84.9, 86.56 85.28, 21.25 85 M21.25 85 C8.53 86.89, -0.3 76.29, 0 63.75 M21.25 85 C7.92 86.52, -1.19 77.33, 0 63.75 M0 63.75 C1.55 52.65, 0.91 43.14, 0 21.25 M0 63.75 C0.05 49.04, 0.56 34.19, 0 21.25 M0 21.25 C0.72 7.95, 5.54 -1.51, 21.25 0 M0 21.25 C1.14 9.04, 9.25 -0.25, 21.25 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(386.26438101369195 751.5292048024885) rotate(0 80.82992553710938 37.5)"><text x="80.82992553710938" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">externel plugin N</text><text x="80.82992553710938" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">independent </text><text x="80.82992553710938" y="50" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">process</text></g><g transform="translate(321.02218874392844 775.6637918342876) rotate(0 16.43998718261716 12.5)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">......</text></g><g mask="url(#mask-TzMhPrg1Y6BU8WpydeCbn)" stroke-linecap="round"><g transform="translate(377.5008818220156 318.4974021628568) rotate(0 6.69831740004139 171.20370933175946)"><path d="M-1.16 -0.36 C0.09 23.05, 4.72 83.68, 7.34 141.07 C9.96 198.46, 13.12 310.38, 14.55 344 M0.43 -1.6 C2.08 21.43, 7.45 82.06, 9.64 139.34 C11.82 196.62, 12.37 308.02, 13.55 342.08" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(377.5008818220156 318.4974021628568) rotate(0 6.69831740004139 171.20370933175946)"><path d="M2.13 312.8 C7.21 321.5, 10.44 331.89, 12.92 340.79 M2.07 313.5 C6.25 321.99, 8.22 327.01, 12.87 343.02" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(377.5008818220156 318.4974021628568) rotate(0 6.69831740004139 171.20370933175946)"><path d="M22.64 312.39 C21.82 321.01, 19.13 331.52, 12.92 340.79 M22.59 313.1 C21.77 321.53, 18.73 326.64, 12.87 343.02" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask id="mask-TzMhPrg1Y6BU8WpydeCbn"><rect x="0" y="0" fill="#fff" width="492.51785921797364" height="761.6976710017916"></rect><rect x="339.7624572945682" y="446.3281996378264" fill="#000" width="91.71992492675781" height="25" opacity="1"></rect></mask><g transform="translate(339.7624572945682 446.3281996378264) rotate(0 44.436741927488754 43.37291185678987)"><text x="45.859962463378906" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">client req</text></g><g mask="url(#mask-07-pOcmh6vBYG2RZvt4j5)" stroke-linecap="round"><g transform="translate(1151.3727127527818 236.09966947245084) rotate(0 -83.72160061814031 0.983200256269356)"><path d="M-0.78 0.74 C-28.47 1.12, -139.75 1.91, -167.26 1.91 M1.01 0.08 C-26.77 0.16, -140.59 -0.09, -168.45 0.21" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(1151.3727127527818 236.09966947245084) rotate(0 -83.72160061814031 0.983200256269356)"><path d="M-139.06 -10.92 C-144.71 -6.67, -152.41 -7.41, -169.57 1.96 M-139.66 -9.16 C-146.65 -8.84, -153.55 -4.77, -167.56 0.79" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(1151.3727127527818 236.09966947245084) rotate(0 -83.72160061814031 0.983200256269356)"><path d="M-138.99 9.61 C-144.6 9.68, -152.31 4.76, -169.57 1.96 M-139.59 11.36 C-146.46 7.12, -153.38 6.62, -167.56 0.79" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask id="mask-07-pOcmh6vBYG2RZvt4j5"><rect x="0" y="0" fill="#fff" width="1418.1689844890898" height="337.2037947588742"></rect><rect x="1051.0245875657802" y="224.15173211566253" fill="#000" width="33.89997863769531" height="25" opacity="1"></rect></mask><g transform="translate(1051.0245875657802 224.15173211566253) rotate(0 16.626524568861328 12.931137613057672)"><text x="16.949989318847656" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">call</text></g><g stroke-linecap="round" transform="translate(657.7323680295196 118.07174761311762) rotate(0 594.3594657858122 399.82775045465723)"><path d="M32 0 M32 0 C296.53 2.35, 559.73 1.21, 1156.72 0 M1156.72 0 C1176.93 1.75, 1187.16 9.35, 1188.72 32 M1188.72 32 C1190.56 208, 1190.05 386.08, 1188.72 767.66 M1188.72 767.66 C1190.49 790.15, 1179.02 798.21, 1156.72 799.66 M1156.72 799.66 C928.17 799.99, 698.75 799.36, 32 799.66 M32 799.66 C12.64 798.07, -1.51 790.12, 0 767.66 M0 767.66 C2.04 559.28, 1.13 350.61, 0 32 M0 32 C-1.16 12.33, 12.48 -0.73, 32 0" stroke="#1e1e1e" stroke-width="1.5" fill="none" stroke-dasharray="8 9"></path></g><g transform="translate(699.9331835266487 139.04242423466894) rotate(0 52.09197998046875 22.5)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="36px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">iSulad</text></g><g stroke-linecap="round" transform="translate(824.2320604200497 784.7965642218423) rotate(0 318.5 30.5)"><path d="M4.06 3.53 C4.06 3.53, 4.06 3.53, 4.06 3.53 M4.06 3.53 C4.06 3.53, 4.06 3.53, 4.06 3.53 M3.14 10.68 C5.1 6.36, 9.45 4.15, 10.36 2.38 M3.14 10.68 C5.48 8.4, 7.59 4.61, 10.36 2.38 M2.22 17.83 C7.22 13.52, 12.59 5.33, 16.66 1.23 M2.22 17.83 C7.06 12.93, 9.75 8.67, 16.66 1.23 M2.62 23.48 C9.39 18.56, 15.76 10.31, 21.64 1.59 M2.62 23.48 C6.18 18.31, 10.22 13.16, 21.64 1.59 M3.01 29.12 C12.14 18.78, 21.1 8.79, 27.29 1.2 M3.01 29.12 C10.68 20.8, 17.79 12.98, 27.29 1.2 M2.75 35.52 C11.29 29.69, 15.95 20.46, 32.27 1.56 M2.75 35.52 C13.27 22.81, 24.28 9.92, 32.27 1.56 M3.15 41.16 C13.51 27.69, 25.24 14.23, 37.92 1.16 M3.15 41.16 C10.52 31.21, 18.16 23.43, 37.92 1.16 M3.54 46.8 C13.85 35.56, 24.32 25.09, 42.9 1.52 M3.54 46.8 C16.29 30.82, 30.05 15.22, 42.9 1.52 M2.62 53.96 C19.84 36.21, 34.68 16.55, 48.55 1.13 M2.62 53.96 C18.05 35.82, 35.4 16.71, 48.55 1.13 M4.99 57.34 C20.91 40.92, 35.31 22.98, 53.53 1.49 M4.99 57.34 C22.74 38.24, 41.2 17.62, 53.53 1.49 M7.35 60.71 C20.42 45.67, 32.41 33.49, 59.18 1.09 M7.35 60.71 C27.75 37.78, 48.56 14.86, 59.18 1.09 M12.34 61.07 C29.15 40.27, 46.78 20.61, 64.16 1.45 M12.34 61.07 C31.94 37.36, 53.41 15.17, 64.16 1.45 M17.32 61.43 C31.25 42.48, 45.93 25.94, 69.81 1.06 M17.32 61.43 C34.45 40.92, 52.35 22.23, 69.81 1.06 M22.97 61.04 C38.5 44, 56.92 22.78, 74.79 1.42 M22.97 61.04 C38.96 42.03, 56.72 21.31, 74.79 1.42 M27.95 61.4 C39.95 50.55, 51.51 36.02, 80.44 1.02 M27.95 61.4 C45.43 42.59, 62.15 22.23, 80.44 1.02 M33.6 61.01 C45.05 49.3, 56.49 36.38, 85.43 1.38 M33.6 61.01 C53.89 38.32, 74.25 14.99, 85.43 1.38 M38.58 61.37 C57.07 39.69, 78.89 15.75, 91.07 0.99 M38.58 61.37 C58.02 39.35, 77.82 16.64, 91.07 0.99 M44.23 60.97 C61.28 41.09, 77.37 25.59, 96.06 1.35 M44.23 60.97 C59.63 44.53, 72.99 26.23, 96.06 1.35 M49.21 61.33 C63.23 43.21, 79.95 27.49, 101.7 0.95 M49.21 61.33 C65.28 43.76, 80.36 27.26, 101.7 0.95 M54.86 60.94 C65.52 49.37, 77.72 35.78, 106.69 1.31 M54.86 60.94 C69.62 43.95, 84.53 27.98, 106.69 1.31 M59.84 61.3 C71.75 47.86, 83.51 33.02, 112.33 0.92 M59.84 61.3 C76.76 42.35, 92.37 23.43, 112.33 0.92 M65.49 60.9 C80.57 45.57, 90.67 30.32, 117.32 1.28 M65.49 60.9 C84.01 38.56, 103.74 17.41, 117.32 1.28 M70.47 61.26 C87.67 42.32, 104.12 18.69, 122.3 1.64 M70.47 61.26 C88.23 41.4, 103.67 22.27, 122.3 1.64 M75.46 61.62 C94.22 43.7, 109.32 23.04, 127.95 1.25 M75.46 61.62 C87.12 49.65, 98.4 36.47, 127.95 1.25 M81.1 61.23 C98.06 41.6, 116.22 18.89, 132.93 1.61 M81.1 61.23 C99.26 38.83, 119.03 15.49, 132.93 1.61 M86.09 61.59 C106.17 40.51, 125.5 15.57, 138.58 1.21 M86.09 61.59 C99.38 46.66, 112.41 31.61, 138.58 1.21 M91.73 61.19 C106.27 43.86, 117.19 30.05, 143.56 1.57 M91.73 61.19 C113.78 38.42, 132.8 13.01, 143.56 1.57 M96.72 61.55 C115.68 40.59, 134.62 17.06, 149.21 1.18 M96.72 61.55 C109.96 48.56, 122.58 33.94, 149.21 1.18 M102.36 61.16 C116.57 43.94, 130.07 31.47, 154.19 1.54 M102.36 61.16 C117.86 42.99, 133.74 25.26, 154.19 1.54 M107.35 61.52 C122.27 43.22, 139.65 27.86, 159.84 1.14 M107.35 61.52 C119.6 49.51, 128.97 35.55, 159.84 1.14 M112.99 61.12 C123.23 47.3, 135.86 35.29, 164.82 1.5 M112.99 61.12 C126.12 45.82, 139.7 30.89, 164.82 1.5 M117.98 61.48 C138.54 37.79, 158.61 14.64, 170.47 1.11 M117.98 61.48 C137.4 39.43, 157.38 15.95, 170.47 1.11 M123.62 61.09 C143.24 38.26, 161.24 18.93, 175.45 1.47 M123.62 61.09 C133.88 49.44, 144.97 35.4, 175.45 1.47 M128.61 61.45 C146.58 41.17, 163.01 24.08, 181.1 1.07 M128.61 61.45 C139.56 48.26, 152.09 34.62, 181.1 1.07 M134.25 61.05 C148.07 47.62, 158.58 32.36, 186.08 1.43 M134.25 61.05 C153.08 38.63, 170.99 18.37, 186.08 1.43 M139.24 61.41 C155.76 40.17, 174.58 20.89, 191.73 1.04 M139.24 61.41 C160.67 38.28, 181.71 14.63, 191.73 1.04 M144.88 61.02 C162.21 42.28, 175.34 24.33, 196.71 1.4 M144.88 61.02 C162.8 41.37, 179.34 19.99, 196.71 1.4 M149.87 61.38 C170.23 39.13, 192.34 15.75, 202.36 1 M149.87 61.38 C162.96 47.19, 174.37 32.64, 202.36 1 M155.51 60.99 C176.21 38.59, 196.26 15.07, 207.34 1.36 M155.51 60.99 C173.47 40.14, 194.22 16.89, 207.34 1.36 M160.5 61.35 C183.13 38.45, 203.77 11.7, 212.99 0.97 M160.5 61.35 C171.86 47.84, 183.28 35.75, 212.99 0.97 M166.14 60.95 C185.81 40.73, 203.88 18.99, 217.97 1.33 M166.14 60.95 C181.02 42.51, 198.7 24.36, 217.97 1.33 M171.13 61.31 C190.17 41.78, 206.39 19.34, 223.62 0.93 M171.13 61.31 C187.99 41.28, 206.76 21.1, 223.62 0.93 M176.77 60.92 C193.84 42.59, 212.48 22.62, 228.6 1.29 M176.77 60.92 C190.42 46.56, 201.4 31.93, 228.6 1.29 M181.76 61.28 C201.11 38.33, 221.49 15.35, 233.59 1.65 M181.76 61.28 C198.32 42.58, 213.68 23.88, 233.59 1.65 M186.75 61.64 C200.27 43.4, 215.93 27.87, 239.23 1.26 M186.75 61.64 C199.57 45.12, 214.19 29.41, 239.23 1.26 M192.39 61.24 C210.43 38.61, 230.32 17.26, 244.22 1.62 M192.39 61.24 C210.46 38.77, 230.5 16.1, 244.22 1.62 M197.38 61.6 C214.06 45.44, 225.21 26.68, 249.86 1.23 M197.38 61.6 C212.95 42.6, 230.79 24.35, 249.86 1.23 M203.02 61.21 C220.96 38.34, 239.7 16.96, 254.85 1.59 M203.02 61.21 C217.22 43.32, 232.35 26.52, 254.85 1.59 M208.01 61.57 C221.48 50.46, 228.97 36.13, 260.49 1.19 M208.01 61.57 C224.55 41.4, 240.98 21.51, 260.49 1.19 M213.65 61.17 C226.32 49.09, 236.56 34.15, 265.48 1.55 M213.65 61.17 C232.6 41.54, 249.83 19.43, 265.48 1.55 M218.64 61.53 C230.13 46.14, 244.22 33.69, 271.12 1.16 M218.64 61.53 C232.33 44.78, 246.54 28.33, 271.12 1.16 M224.28 61.14 C237.83 43.75, 253.69 26.33, 276.11 1.52 M224.28 61.14 C241.72 41.26, 258.89 21.12, 276.11 1.52 M229.27 61.5 C242.29 47.18, 254.35 34.23, 281.75 1.12 M229.27 61.5 C247.78 41.11, 265.39 21.02, 281.75 1.12 M234.91 61.1 C247.89 45.35, 261.35 33.25, 286.74 1.48 M234.91 61.1 C254.7 38.24, 274.16 15.6, 286.74 1.48 M239.9 61.46 C253.07 43.8, 268.3 30.45, 292.38 1.09 M239.9 61.46 C257.69 41.2, 273.93 21.99, 292.38 1.09 M245.54 61.07 C262.19 41.23, 275.25 23.41, 297.37 1.45 M245.54 61.07 C263.57 41, 280.72 19.26, 297.37 1.45 M250.53 61.43 C261.86 45.18, 273.56 32.32, 303.01 1.05 M250.53 61.43 C266.39 43.07, 282.73 24.53, 303.01 1.05 M256.17 61.04 C269.84 47.9, 277.29 34.7, 308 1.41 M256.17 61.04 C276.12 38.89, 297.16 15.11, 308 1.41 M261.16 61.4 C272.54 48.05, 286.81 31.62, 313.64 1.02 M261.16 61.4 C277.26 44.05, 291.58 24.87, 313.64 1.02 M266.8 61 C277.68 47.21, 291.24 34.79, 318.63 1.38 M266.8 61 C286.47 35.99, 306.96 12.46, 318.63 1.38 M271.79 61.36 C290.51 39.85, 308.44 21.2, 324.27 0.98 M271.79 61.36 C291.97 37.83, 312.67 15.3, 324.27 0.98 M277.43 60.97 C285.92 51.08, 297.01 37.6, 329.26 1.34 M277.43 60.97 C297.77 38.28, 317.9 14.28, 329.26 1.34 M282.42 61.33 C304.81 38.26, 324.53 14.78, 334.25 1.7 M282.42 61.33 C296.86 45.58, 311.84 27.65, 334.25 1.7 M287.41 61.69 C307.33 40.34, 327.46 14.16, 339.89 1.31 M287.41 61.69 C304.56 42.91, 320.38 24.1, 339.89 1.31 M293.05 61.29 C308.9 42.57, 328.55 21.06, 344.88 1.67 M293.05 61.29 C309.74 44.07, 324.63 26.13, 344.88 1.67 M298.04 61.65 C313.7 45.46, 327.69 26.07, 350.52 1.28 M298.04 61.65 C308.9 47.47, 320.91 34.45, 350.52 1.28 M303.68 61.26 C317.81 44.91, 333.91 24.85, 355.51 1.64 M303.68 61.26 C322.75 38.09, 343.44 17.51, 355.51 1.64 M308.67 61.62 C321.37 47.58, 337.2 29.2, 361.15 1.24 M308.67 61.62 C322.38 45.71, 335.71 28.75, 361.15 1.24 M314.31 61.22 C328.01 47.67, 339.58 31.14, 366.14 1.6 M314.31 61.22 C328.95 44.42, 343.59 27.86, 366.14 1.6 M319.3 61.58 C329.65 47.7, 344.74 31.87, 371.78 1.21 M319.3 61.58 C339.9 37.7, 358.45 16.81, 371.78 1.21 M324.94 61.19 C338.33 50.05, 347.48 36.14, 376.77 1.57 M324.94 61.19 C345.92 37.25, 366.24 12.58, 376.77 1.57 M329.93 61.55 C341.59 49, 354.35 32.89, 382.41 1.17 M329.93 61.55 C348.54 39.51, 367.9 18.18, 382.41 1.17 M335.57 61.15 C355.56 38.06, 376.18 17.19, 387.4 1.53 M335.57 61.15 C352.86 40.96, 370.74 20.65, 387.4 1.53 M340.56 61.51 C356.43 45.89, 369.69 30.6, 393.04 1.14 M340.56 61.51 C359.45 38.66, 380.75 15.45, 393.04 1.14 M346.2 61.12 C361.31 44.18, 374.26 28.17, 398.03 1.5 M346.2 61.12 C363.49 40.3, 380.27 19.34, 398.03 1.5 M351.19 61.48 C368.79 40.05, 389.31 18.89, 403.67 1.1 M351.19 61.48 C365.42 44.82, 379.78 29.08, 403.67 1.1 M356.83 61.09 C370.68 44.99, 385.82 29.75, 408.66 1.46 M356.83 61.09 C369.05 46.3, 382.88 31.76, 408.66 1.46 M361.82 61.45 C381.38 38.35, 400.8 17.27, 414.3 1.07 M361.82 61.45 C375.7 46.12, 388.1 29.64, 414.3 1.07 M367.46 61.05 C379.6 49.48, 387.05 37.03, 419.29 1.43 M367.46 61.05 C384.97 41.29, 401.69 22.18, 419.29 1.43 M372.45 61.41 C385.91 43.96, 400.23 29.79, 424.93 1.03 M372.45 61.41 C385.23 47.78, 396.92 34.53, 424.93 1.03 M378.09 61.02 C393.5 43.66, 412.96 24.66, 429.92 1.39 M378.09 61.02 C394.2 43.85, 408.4 26.53, 429.92 1.39 M383.08 61.38 C397.42 43.4, 415.28 24.29, 434.91 1.75 M383.08 61.38 C393.94 48.72, 405.68 34.65, 434.91 1.75 M388.06 61.74 C402.2 47.52, 414.32 31.14, 440.55 1.36 M388.06 61.74 C401.02 45.43, 415.02 31.33, 440.55 1.36 M393.71 61.34 C409.12 40.84, 424.4 26.03, 445.54 1.72 M393.71 61.34 C412.76 39.51, 432.55 17.86, 445.54 1.72 M398.69 61.7 C415.69 40.38, 433.35 21.38, 451.18 1.33 M398.69 61.7 C417.04 42.2, 434.66 19.98, 451.18 1.33 M404.34 61.31 C415.86 47.28, 425.08 36.02, 456.17 1.69 M404.34 61.31 C424.94 38.56, 443.82 15.84, 456.17 1.69 M409.32 61.67 C424.36 43.9, 436.13 28.39, 461.81 1.29 M409.32 61.67 C425.76 42.17, 443.96 20.94, 461.81 1.29 M414.97 61.27 C430.84 45.89, 443.13 31.62, 466.8 1.65 M414.97 61.27 C429.26 46.21, 441.12 31.84, 466.8 1.65 M419.95 61.63 C433.24 46.81, 444.9 34.6, 472.44 1.26 M419.95 61.63 C440.8 37.13, 462.66 14.15, 472.44 1.26 M425.6 61.24 C439.87 45.9, 455.74 28.45, 477.43 1.62 M425.6 61.24 C439.14 44.61, 454.41 28.72, 477.43 1.62 M430.58 61.6 C449.73 41.64, 470.45 16.58, 483.07 1.22 M430.58 61.6 C448.6 40.3, 466.49 18.89, 483.07 1.22 M436.23 61.2 C455.68 42.85, 472.81 20.81, 488.06 1.58 M436.23 61.2 C453.07 42.65, 468.95 23.6, 488.06 1.58 M441.21 61.56 C460.14 41.72, 475.78 22.2, 493.7 1.19 M441.21 61.56 C453.61 47.98, 463.8 35.04, 493.7 1.19 M446.86 61.17 C466.01 39.46, 484.35 18.44, 498.69 1.55 M446.86 61.17 C463.4 41.93, 479.17 22.04, 498.69 1.55 M451.84 61.53 C468.63 41.02, 483.4 22.07, 504.33 1.15 M451.84 61.53 C465.01 46.81, 476.34 32.47, 504.33 1.15 M457.49 61.13 C470.37 45.69, 483.59 34.67, 509.32 1.51 M457.49 61.13 C477.12 38.95, 497.44 14.23, 509.32 1.51 M462.47 61.49 C475.58 47.29, 487.81 34.99, 514.96 1.12 M462.47 61.49 C478.47 43.85, 493.49 25.31, 514.96 1.12 M468.12 61.1 C484.76 39.98, 504.45 19.73, 519.95 1.48 M468.12 61.1 C485.09 42.93, 500.62 22.65, 519.95 1.48 M473.1 61.46 C490.29 40.96, 507.64 22.65, 525.59 1.08 M473.1 61.46 C488.68 43.39, 503.8 26.55, 525.59 1.08 M478.75 61.07 C491.49 43.54, 505.14 30.68, 530.58 1.44 M478.75 61.07 C489.59 45.94, 502.65 32.48, 530.58 1.44 M483.74 61.43 C503.19 39.56, 519.63 18.59, 535.56 1.8 M483.74 61.43 C498.92 43.25, 514.88 24.61, 535.56 1.8 M488.72 61.79 C508.06 37.79, 528.27 17.11, 541.21 1.41 M488.72 61.79 C499.02 50.53, 508.66 36.83, 541.21 1.41 M494.37 61.39 C505.11 47.45, 518.45 32.55, 546.19 1.77 M494.37 61.39 C509.24 42.83, 525.32 24.49, 546.19 1.77 M499.35 61.75 C517.11 36.49, 538.79 13.87, 551.84 1.37 M499.35 61.75 C518.51 38, 539.59 15.66, 551.84 1.37 M505 61.36 C514.89 49.66, 525.09 38.16, 556.82 1.73 M505 61.36 C516.96 46.49, 530.6 32.52, 556.82 1.73 M509.98 61.72 C526.92 42.45, 540.2 24.47, 562.47 1.34 M509.98 61.72 C524.37 44.62, 539.33 29.34, 562.47 1.34 M515.63 61.32 C525.21 47.55, 538.49 35.35, 567.45 1.7 M515.63 61.32 C530.62 42.85, 546.1 26.81, 567.45 1.7 M520.61 61.68 C534.44 47.05, 547.97 33.5, 573.1 1.31 M520.61 61.68 C535.67 43.44, 551.8 26.55, 573.1 1.31 M526.26 61.29 C544.31 43.3, 561.78 23.25, 578.08 1.67 M526.26 61.29 C540.69 46.55, 554.51 29.06, 578.08 1.67 M531.24 61.65 C546.78 41.76, 567.26 21.96, 583.73 1.27 M531.24 61.65 C549.94 39.56, 569.27 15.9, 583.73 1.27 M536.89 61.25 C554.01 41.76, 575.64 17.84, 588.71 1.63 M536.89 61.25 C556.57 37.72, 576.34 16.31, 588.71 1.63 M541.87 61.61 C556.3 41.65, 574.88 22.87, 594.36 1.24 M541.87 61.61 C560.5 38.89, 580.71 16.47, 594.36 1.24 M547.52 61.22 C562.09 45.48, 575.27 27.18, 599.34 1.6 M547.52 61.22 C562.85 43.66, 576.34 28.22, 599.34 1.6 M552.5 61.58 C573.67 37.73, 590.5 17.37, 604.99 1.2 M552.5 61.58 C567.38 43.04, 583.17 26.77, 604.99 1.2 M558.15 61.18 C575.86 39.89, 594.46 20.64, 609.97 1.56 M558.15 61.18 C570.25 46.09, 582.57 32.8, 609.97 1.56 M563.13 61.54 C577.97 44.39, 596.63 22.41, 615.62 1.17 M563.13 61.54 C574.92 48.44, 587.45 35.71, 615.62 1.17 M568.78 61.15 C584.67 40.07, 602.55 23.38, 620.6 1.53 M568.78 61.15 C586.97 41.02, 604.39 19.98, 620.6 1.53 M573.76 61.51 C591.09 40.18, 611.08 15.26, 626.25 1.13 M573.76 61.51 C592.96 39.18, 613.22 15.5, 626.25 1.13 M579.41 61.12 C597.68 37.4, 619.24 15.86, 629.92 3 M579.41 61.12 C594.72 43.06, 609.64 26.29, 629.92 3 M584.39 61.48 C594.98 48.64, 607.11 35.39, 634.25 4.12 M584.39 61.48 C600.41 41.52, 617.74 23.5, 634.25 4.12 M590.04 61.08 C604.48 43.98, 620.51 25.49, 635.96 8.25 M590.04 61.08 C602.7 48.19, 614.61 33.36, 635.96 8.25 M595.02 61.44 C609.84 43.13, 625.08 24.81, 638.98 10.88 M595.02 61.44 C609.4 45.4, 625.05 27.4, 638.98 10.88 M600.01 61.8 C613.9 45.8, 632.41 28.25, 638.72 17.27 M600.01 61.8 C613.49 46.6, 628.09 29.91, 638.72 17.27 M605.65 61.41 C616.43 52.53, 624.12 42.01, 639.11 22.92 M605.65 61.41 C618.85 46.89, 631.32 32.04, 639.11 22.92 M610.64 61.77 C617.72 54.49, 626.14 46.54, 638.85 29.31 M610.64 61.77 C621.74 49.5, 632.59 36.67, 638.85 29.31 M616.28 61.37 C624.38 51.13, 632.81 44.49, 639.25 34.96 M616.28 61.37 C625.51 50.47, 633.36 41.86, 639.25 34.96 M621.93 60.98 C628.17 53.53, 633.45 46.52, 639.64 40.6 M621.93 60.98 C627.07 54.13, 634.41 47.43, 639.64 40.6 M627.57 60.58 C631.6 55.8, 634.5 53.18, 637.41 49.26 M627.57 60.58 C631.51 56.08, 635.11 52.46, 637.41 49.26" stroke="#a5d8ff" stroke-width="0.5" fill="none"></path><path d="M15.25 0 M15.25 0 C189.47 -0.25, 365.34 0.58, 621.75 0 M15.25 0 C147.92 0.13, 280.15 0.06, 621.75 0 M621.75 0 C630.79 0.19, 636.55 6.25, 637 15.25 M621.75 0 C631.79 0.03, 634.99 5.73, 637 15.25 M637 15.25 C638.01 26.61, 637.52 36.4, 637 45.75 M637 15.25 C637.96 22.88, 636.5 31.75, 637 45.75 M637 45.75 C635.8 57.12, 631.08 60.21, 621.75 61 M637 45.75 C635.31 58.1, 632.35 60.56, 621.75 61 M621.75 61 C447.81 57.84, 273.67 58.05, 15.25 61 M621.75 61 C473.43 63.12, 325.37 62.88, 15.25 61 M15.25 61 C3.47 61.3, -0.25 54.25, 0 45.75 M15.25 61 C4.29 59.32, 1.81 55.84, 0 45.75 M0 45.75 C0.82 37.62, -1.4 31.65, 0 15.25 M0 45.75 C0.1 36.48, -0.4 27.08, 0 15.25 M0 15.25 C0.59 5.81, 5.1 0.18, 15.25 0 M0 15.25 C-0.73 6.64, 5.11 -2.23, 15.25 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(1112.6220903272763 802.7965642218423) rotate(0 30.109970092773438 12.5)"><text x="30.109970092773438" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">plugin </text></g><g stroke-linecap="round" transform="translate(10 131.4823844339371) rotate(0 269.66662597656256 116.06928877397013)"><path d="M32 0 M32 0 C143.1 -2.42, 255.05 -1.28, 507.33 0 M32 0 C220.97 2.15, 410.94 1.99, 507.33 0 M507.33 0 C529.64 -1.44, 537.4 12.02, 539.33 32 M507.33 0 C528.57 0.97, 540.81 10.06, 539.33 32 M539.33 32 C539.48 81.14, 540.66 127.04, 539.33 200.14 M539.33 32 C540.26 80.58, 540.05 130.67, 539.33 200.14 M539.33 200.14 C541.15 220.74, 528.19 231.7, 507.33 232.14 M539.33 200.14 C539.26 221.94, 527.06 233.48, 507.33 232.14 M507.33 232.14 C329.58 232.14, 153.84 231.37, 32 232.14 M507.33 232.14 C377.12 231.11, 246.17 230.97, 32 232.14 M32 232.14 C10.16 233.09, -0.54 221.4, 0 200.14 M32 232.14 C10.53 232.28, 1.04 221.41, 0 200.14 M0 200.14 C-2.24 144.6, 0.97 88.97, 0 32 M0 200.14 C-0.44 150.74, -0.93 102.76, 0 32 M0 32 C-1.79 10.41, 11.22 0.36, 32 0 M0 32 C-0.65 9.56, 10.13 -0.72, 32 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(20.242077341672996 141.64298369524386) rotate(0 102.01988220214844 12.5)"><text x="102.01988220214844" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">isula-rust-extensions</text></g><g stroke-linecap="round" transform="translate(62.108766233767824 181.322216293971) rotate(0 90 62.5)"><path d="M7.66 7.99 C7.66 7.99, 7.66 7.99, 7.66 7.99 M7.66 7.99 C7.66 7.99, 7.66 7.99, 7.66 7.99 M3.46 18.91 C7.14 13.62, 13.91 7.4, 17.9 2.31 M3.46 18.91 C6.12 15.12, 10.62 11.5, 17.9 2.31 M2.55 26.06 C10 21.45, 13.3 12.59, 24.2 1.16 M2.55 26.06 C7.88 19.68, 15.32 13.11, 24.2 1.16 M2.94 31.71 C12.18 21.76, 18.01 12.71, 29.84 0.76 M2.94 31.71 C8.63 24.51, 15.29 17.3, 29.84 0.76 M2.68 38.1 C11.42 29.13, 19.85 19.05, 34.17 1.88 M2.68 38.1 C14.27 26.75, 25.53 13.36, 34.17 1.88 M2.42 44.5 C12.03 29.96, 24.81 16.49, 39.16 2.24 M2.42 44.5 C12.57 31.82, 23.16 19.86, 39.16 2.24 M2.81 50.15 C15.5 37.59, 22.8 25.3, 44.14 2.6 M2.81 50.15 C15.11 35.62, 28.35 21.3, 44.14 2.6 M2.55 56.54 C20.3 35.71, 35.87 16.6, 49.79 2.2 M2.55 56.54 C14.29 42.28, 28.36 26.77, 49.79 2.2 M2.95 62.19 C16.16 46.6, 28.28 35.32, 54.77 2.56 M2.95 62.19 C14.22 50.57, 24.64 38.08, 54.77 2.56 M2.68 68.58 C21.28 45.95, 43.45 25.19, 60.42 2.17 M2.68 68.58 C22.38 47.75, 40.02 26.34, 60.42 2.17 M2.42 74.98 C18.29 54.74, 37.91 34.94, 65.4 2.53 M2.42 74.98 C27.64 46.17, 51.87 17.52, 65.4 2.53 M2.82 80.63 C22.63 55.38, 42.46 34.09, 71.05 2.14 M2.82 80.63 C28.36 49.73, 54.46 19.28, 71.05 2.14 M2.56 87.02 C19.93 64.96, 37.73 44.77, 76.04 2.5 M2.56 87.02 C29.28 56.08, 56.97 26.36, 76.04 2.5 M2.95 92.67 C17.35 72.46, 35.2 55.47, 81.68 2.1 M2.95 92.67 C20.22 73.38, 39.13 53.73, 81.68 2.1 M0.72 101.33 C22.74 76.9, 46.48 50.71, 86.67 2.46 M0.72 101.33 C28.09 69.93, 56.76 38.48, 86.67 2.46 M1.77 106.22 C31.22 71.1, 63.16 36.24, 92.31 2.07 M1.77 106.22 C21.07 82.19, 41.68 60.21, 92.31 2.07 M4.13 109.6 C27.61 83.61, 49.94 58.46, 97.3 2.43 M4.13 109.6 C39.63 69.35, 73.68 29.03, 97.3 2.43 M6.5 112.97 C41.99 75.13, 76.28 33.38, 102.94 2.03 M6.5 112.97 C25.9 89.59, 45.49 67.73, 102.94 2.03 M8.2 117.11 C43.79 77.46, 79.88 35.77, 107.93 2.39 M8.2 117.11 C48.57 71.87, 88.08 27.32, 107.93 2.39 M11.22 119.73 C52.18 74.94, 90.12 30.58, 113.57 2 M11.22 119.73 C51.71 73.8, 92.88 27.25, 113.57 2 M14.9 121.6 C34.94 96.53, 58.52 68.86, 118.56 2.36 M14.9 121.6 C49.31 83.14, 83.56 42.28, 118.56 2.36 M18.57 123.47 C52.22 85.56, 82.32 50.16, 124.2 1.96 M18.57 123.47 C45.8 92.79, 72.74 61.28, 124.2 1.96 M22.9 124.59 C47.46 100.28, 68.18 71.38, 129.19 2.32 M22.9 124.59 C52.63 90.92, 83.43 55.43, 129.19 2.32 M27.89 124.95 C64.71 83.39, 104.07 41.09, 134.83 1.93 M27.89 124.95 C54.79 92.99, 82.92 61.44, 134.83 1.93 M32.22 126.06 C55.43 95.9, 79.34 68.87, 139.82 2.29 M32.22 126.06 C59.88 93.44, 88.75 60.77, 139.82 2.29 M37.86 125.67 C61.76 96.74, 88.64 67.17, 145.46 1.89 M37.86 125.67 C70.57 87.89, 103.02 48.75, 145.46 1.89 M42.85 126.03 C73.07 91.24, 108.18 50.48, 151.1 1.5 M42.85 126.03 C64.98 99.67, 87.78 73.73, 151.1 1.5 M48.5 125.63 C73.25 98.86, 100.79 70.99, 155.43 2.61 M48.5 125.63 C82.03 87.48, 115.7 48.46, 155.43 2.61 M53.48 125.99 C92.26 80.39, 131.34 35.68, 160.42 2.97 M53.48 125.99 C93.47 79.99, 133.4 34.95, 160.42 2.97 M59.13 125.6 C95.88 86.78, 133.12 42.28, 164.75 4.09 M59.13 125.6 C83.94 95.63, 109.42 67.42, 164.75 4.09 M64.11 125.96 C101.61 85.28, 135.9 45.44, 168.43 5.96 M64.11 125.96 C98.05 85.6, 134 44.5, 168.43 5.96 M69.76 125.56 C96.88 93.15, 125.56 61.96, 172.1 7.83 M69.76 125.56 C91.17 98.5, 114.83 71.1, 172.1 7.83 M74.74 125.92 C114.34 79.66, 155.54 35.03, 175.12 10.45 M74.74 125.92 C94.74 103.2, 115.43 79.24, 175.12 10.45 M79.73 126.28 C107.63 92.62, 136.58 62.12, 177.48 13.83 M79.73 126.28 C107.12 95.52, 135.03 63.28, 177.48 13.83 M85.37 125.89 C113.17 92.61, 140.98 57.29, 179.19 17.97 M85.37 125.89 C122.89 82.49, 159.38 40.02, 179.19 17.97 M90.36 126.25 C122.3 91.05, 154.8 55.28, 179.58 23.61 M90.36 126.25 C125.06 86.37, 160.53 44.33, 179.58 23.61 M96 125.85 C121 96.62, 148.34 64.7, 180.63 28.5 M96 125.85 C116.91 100.7, 138.94 76.71, 180.63 28.5 M100.99 126.21 C132.04 91.14, 164.19 52.95, 182.34 32.63 M100.99 126.21 C117.19 107.57, 134.49 88.17, 182.34 32.63 M106.63 125.82 C121.59 103.73, 142.03 87.09, 182.08 39.03 M106.63 125.82 C124.13 105.23, 143.31 83.82, 182.08 39.03 M111.62 126.18 C128.67 108.48, 142.92 90.74, 181.82 45.43 M111.62 126.18 C137.06 95.86, 164.59 65.07, 181.82 45.43 M117.26 125.79 C141.57 97.29, 163.88 74.16, 182.21 51.07 M117.26 125.79 C140.91 99.96, 163.9 72.66, 182.21 51.07 M122.25 126.15 C135.66 110.52, 148.46 98.15, 181.95 57.47 M122.25 126.15 C144.3 100.59, 166.71 76.07, 181.95 57.47 M127.89 125.75 C149.82 104.23, 169.89 77.58, 181.69 63.86 M127.89 125.75 C142.69 106.85, 159.63 88.4, 181.69 63.86 M132.88 126.11 C146.28 112.72, 157.93 96.89, 182.08 69.51 M132.88 126.11 C146.88 110, 159.44 95.05, 182.08 69.51 M138.52 125.72 C148.97 111.91, 158.82 101.02, 181.82 75.91 M138.52 125.72 C152.36 109.53, 165.26 94.17, 181.82 75.91 M143.51 126.08 C156.12 107.98, 170.33 93.13, 181.56 82.3 M143.51 126.08 C153.2 117.04, 161.57 105.26, 181.56 82.3 M149.15 125.68 C157.49 113.01, 167.44 104.78, 181.96 87.95 M149.15 125.68 C160.76 112.35, 170.85 99.42, 181.96 87.95 M155.45 124.53 C165.12 114.49, 170.95 105.61, 181.69 94.34 M155.45 124.53 C164.25 114.89, 175.07 102.27, 181.69 94.34 M161.1 124.14 C168.56 116.87, 172.72 108.73, 180.12 102.25 M161.1 124.14 C165.57 118.81, 171.85 112.22, 180.12 102.25" stroke="#a5d8ff" stroke-width="0.5" fill="none"></path><path d="M31.25 0 M31.25 0 C69.23 2.46, 107.5 1.51, 148.75 0 M31.25 0 C56.85 -0.14, 82.56 0.69, 148.75 0 M148.75 0 C170.15 -0.62, 178.24 12.13, 180 31.25 M148.75 0 C167.39 -0.16, 178.02 9.78, 180 31.25 M180 31.25 C178.14 48.11, 179.5 64.84, 180 93.75 M180 31.25 C179.34 51.46, 179.44 71.56, 180 93.75 M180 93.75 C178.84 115.33, 167.69 125.54, 148.75 125 M180 93.75 C179.78 114.44, 171.29 125.81, 148.75 125 M148.75 125 C119.21 126.91, 90.4 124.62, 31.25 125 M148.75 125 C110.16 125.77, 73.4 126.03, 31.25 125 M31.25 125 C10.42 126.6, 0.12 115.07, 0 93.75 M31.25 125 C9.92 125.7, -0.98 112.66, 0 93.75 M0 93.75 C1.16 70.61, 0.99 48.35, 0 31.25 M0 93.75 C-1.15 80.78, -0.48 65.91, 0 31.25 M0 31.25 C1.97 11.48, 8.67 -1.15, 31.25 0 M0 31.25 C-2.29 11.4, 11.78 0, 31.25 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(76.57883611902173 231.322216293971) rotate(0 75.5299301147461 12.5)"><text x="75.5299301147461" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">Runtime Service</text></g><g stroke-linecap="round" transform="translate(299.10147425003186 176.2622602621841) rotate(0 108.5 67)"><path d="M7.86 8.16 C7.86 8.16, 7.86 8.16, 7.86 8.16 M7.86 8.16 C7.86 8.16, 7.86 8.16, 7.86 8.16 M3.66 19.09 C10.94 11.47, 12.71 8.15, 18.1 2.48 M3.66 19.09 C8.6 14.41, 11.55 9.35, 18.1 2.48 M2.75 26.24 C7.05 19.53, 12.59 13.92, 24.4 1.33 M2.75 26.24 C10.58 19.08, 16.63 9.35, 24.4 1.33 M2.48 32.64 C11.25 20.59, 20.01 12.07, 30.7 0.18 M2.48 32.64 C12.67 21.2, 22.93 8.87, 30.7 0.18 M2.88 38.28 C12.06 28.21, 22.24 15.29, 34.37 2.05 M2.88 38.28 C10.71 29.77, 16.95 22.53, 34.37 2.05 M2.62 44.68 C12.91 31.51, 22.93 22.2, 39.36 2.41 M2.62 44.68 C14.19 31.09, 25.71 17.15, 39.36 2.41 M2.36 51.07 C15.62 33.34, 33.03 14.57, 45 2.02 M2.36 51.07 C18.19 33.61, 34.24 15.72, 45 2.02 M2.75 56.72 C19.25 36.62, 36.38 18.33, 49.99 2.38 M2.75 56.72 C16.33 41.54, 28.64 27.08, 49.99 2.38 M2.49 63.11 C14.16 51.93, 23.78 36.73, 55.63 1.98 M2.49 63.11 C16.1 47.45, 29.24 32.72, 55.63 1.98 M2.88 68.76 C17.04 56.34, 28.27 39.03, 60.62 2.34 M2.88 68.76 C18.2 50.46, 35.11 31.73, 60.62 2.34 M2.62 75.16 C27.09 49.33, 51.77 21.9, 66.26 1.95 M2.62 75.16 C16.49 58.39, 31.19 43.14, 66.26 1.95 M2.36 81.55 C19.7 61.9, 40.1 40.52, 71.25 2.31 M2.36 81.55 C23.87 56.01, 46.17 32.02, 71.25 2.31 M2.76 87.2 C18.89 67.54, 38.73 48.36, 76.89 1.91 M2.76 87.2 C21.79 67.33, 38.61 45.55, 76.89 1.91 M2.49 93.59 C33.59 59.81, 61.35 23.38, 81.88 2.27 M2.49 93.59 C25.45 69.29, 45.95 43.99, 81.88 2.27 M2.89 99.24 C35.81 63.52, 65.62 25.1, 87.52 1.88 M2.89 99.24 C28.88 70.85, 53.66 43.08, 87.52 1.88 M2.63 105.64 C25.64 80.75, 47.34 54.18, 92.51 2.24 M2.63 105.64 C35.5 69.53, 67.1 31.93, 92.51 2.24 M3.02 111.28 C29.06 82.52, 53.75 53.88, 98.15 1.85 M3.02 111.28 C36.59 71.53, 72.64 30.4, 98.15 1.85 M4.07 116.17 C26.17 89.33, 44.62 66.07, 103.14 2.21 M4.07 116.17 C36.33 79.78, 69.51 42.46, 103.14 2.21 M4.47 121.81 C38.86 82.82, 75.37 40.08, 108.78 1.81 M4.47 121.81 C35.08 87.42, 65.12 52.64, 108.78 1.81 M7.49 124.43 C30.03 98.21, 54.75 69.94, 113.77 2.17 M7.49 124.43 C32.23 97.02, 55.41 69.9, 113.77 2.17 M10.51 127.06 C44.8 86.47, 78.96 49.23, 119.41 1.78 M10.51 127.06 C44.66 86.28, 78.69 48.6, 119.41 1.78 M12.87 130.44 C36.88 102.3, 63.47 71.32, 124.4 2.14 M12.87 130.44 C39.44 99.12, 64.51 69.5, 124.4 2.14 M17.2 131.55 C45.28 95.67, 75.07 62.74, 130.04 1.74 M17.2 131.55 C51.81 91.53, 85.16 51.98, 130.04 1.74 M21.53 132.67 C54.45 97.63, 84.74 60.42, 135.03 2.1 M21.53 132.67 C53.42 96.93, 83.57 61.97, 135.03 2.1 M25.2 134.54 C71.41 81.26, 115.79 31.58, 140.67 1.71 M25.2 134.54 C65.26 88.88, 103.54 42.43, 140.67 1.71 M30.19 134.9 C75.07 84.3, 119.44 32.53, 145.66 2.07 M30.19 134.9 C71.31 85.91, 113.57 40.15, 145.66 2.07 M34.52 136.01 C63.68 102.49, 87.4 73.96, 150.65 2.43 M34.52 136.01 C75.66 88.87, 115.7 40.58, 150.65 2.43 M39.51 136.37 C66.07 109.64, 89.22 79.07, 156.29 2.03 M39.51 136.37 C66.84 104.95, 95.7 73.64, 156.29 2.03 M45.15 135.98 C71.31 105.4, 98.35 77.24, 161.28 2.39 M45.15 135.98 C75.76 101.82, 107.47 64.57, 161.28 2.39 M50.14 136.34 C81.12 99.76, 108.17 65.67, 166.92 2 M50.14 136.34 C94.51 86.86, 138.75 36.54, 166.92 2 M55.78 135.94 C95.79 90.35, 136.75 43.97, 171.91 2.36 M55.78 135.94 C83.26 104.85, 110.29 74.74, 171.91 2.36 M60.77 136.3 C89.06 105.19, 114.49 74.81, 177.55 1.96 M60.77 136.3 C104.2 86.48, 148.34 36.8, 177.55 1.96 M66.41 135.91 C105.56 93, 139.84 50.59, 182.54 2.32 M66.41 135.91 C93.5 104.77, 121.36 72.21, 182.54 2.32 M71.4 136.27 C107.43 96.66, 142.42 54.48, 187.52 2.68 M71.4 136.27 C110.25 91.48, 147.8 48.88, 187.52 2.68 M77.04 135.87 C120.11 83.13, 167.46 34.82, 193.82 1.54 M77.04 135.87 C107.09 100.69, 138.85 66.26, 193.82 1.54 M82.03 136.23 C110.51 105.85, 138.09 72.82, 198.15 2.65 M82.03 136.23 C116.14 95.84, 150.16 58.33, 198.15 2.65 M87.67 135.84 C121.4 96.54, 151.59 63.22, 201.83 4.52 M87.67 135.84 C128.4 90.24, 168.84 42.37, 201.83 4.52 M92.66 136.2 C119.31 104.21, 145.09 75.8, 206.16 5.63 M92.66 136.2 C117.22 108.48, 140.95 80.45, 206.16 5.63 M98.3 135.8 C137.25 96.26, 171.18 55.03, 208.52 9.01 M98.3 135.8 C136.13 92.88, 171.89 48.55, 208.52 9.01 M103.29 136.16 C138.49 94.87, 176.61 54.02, 211.54 11.64 M103.29 136.16 C145.3 88.79, 186.06 41.67, 211.54 11.64 M108.93 135.77 C144.99 96.62, 181.72 53.94, 214.56 14.26 M108.93 135.77 C138.27 100.96, 166.76 66.88, 214.56 14.26 M113.92 136.13 C154.77 91.72, 195.39 44.2, 216.27 18.4 M113.92 136.13 C153.14 92.19, 191.88 46.18, 216.27 18.4 M118.91 136.49 C145.48 103.86, 171.55 76.01, 217.32 23.28 M118.91 136.49 C146.69 105.71, 173.59 74.76, 217.32 23.28 M124.55 136.1 C153.06 101.91, 181.6 68.27, 219.68 26.66 M124.55 136.1 C156.06 99.32, 189.05 60.87, 219.68 26.66 M129.54 136.46 C163.97 96.42, 199.99 57.94, 219.42 33.06 M129.54 136.46 C149.03 113.39, 169.71 91.79, 219.42 33.06 M135.18 136.06 C167.05 99.31, 198.53 62.47, 219.16 39.46 M135.18 136.06 C161.29 104.06, 186.79 74.82, 219.16 39.46 M140.17 136.42 C164.6 109.81, 190.61 81.9, 219.55 45.1 M140.17 136.42 C159.86 112.37, 180.77 89.32, 219.55 45.1 M145.81 136.03 C172.43 103.89, 201.05 71.47, 219.29 51.5 M145.81 136.03 C166.11 112.62, 184.36 92.13, 219.29 51.5 M150.8 136.39 C174.9 106.2, 199.34 77.66, 219.68 57.14 M150.8 136.39 C175.15 109, 197.74 83.26, 219.68 57.14 M156.44 135.99 C173.88 118.12, 186.24 97.7, 219.42 63.54 M156.44 135.99 C172.55 117.64, 189.96 98.13, 219.42 63.54 M161.43 136.35 C173.08 120.96, 185.03 108.05, 219.82 69.18 M161.43 136.35 C174.14 121.62, 186.91 106.81, 219.82 69.18 M167.07 135.96 C177.87 122.85, 188.26 108.68, 219.56 75.58 M167.07 135.96 C185.26 115.38, 203.82 95.41, 219.56 75.58 M172.06 136.32 C183.59 123.58, 195.78 107.63, 219.95 81.22 M172.06 136.32 C187.01 118.93, 203.8 101.36, 219.95 81.22 M177.7 135.92 C190.41 120.97, 203.14 106.04, 219.69 87.62 M177.7 135.92 C185.56 126.09, 195.74 115.54, 219.69 87.62 M182.69 136.28 C196.83 117.82, 213.59 102.4, 220.08 93.26 M182.69 136.28 C190.73 124.76, 199.96 115.02, 220.08 93.26 M187.68 136.64 C199 124.28, 207 110.77, 217.2 102.68 M187.68 136.64 C197.04 124.1, 208.12 112.84, 217.2 102.68 M193.97 135.49 C199.94 128.45, 206.02 119.35, 216.28 109.83 M193.97 135.49 C199.35 128.08, 204.7 123.65, 216.28 109.83 M200.93 133.59 C205.75 131.4, 207.68 124.78, 213.39 119.25 M200.93 133.59 C205.92 128.52, 209.5 124.46, 213.39 119.25" stroke="#a5d8ff" stroke-width="0.5" fill="none"></path><path d="M32 0 M32 0 C80.38 -2.28, 124.72 -2.09, 185 0 M32 0 C87.89 0.9, 144.66 1.82, 185 0 M185 0 C207.85 1.39, 215.86 8.98, 217 32 M185 0 C204.54 -1.31, 217.75 8.85, 217 32 M217 32 C218.51 55.12, 219.06 79.07, 217 102 M217 32 C217.4 47.1, 216.08 61.18, 217 102 M217 102 C215.43 121.39, 208.04 132.7, 185 134 M217 102 C217.28 125.59, 208.4 133.47, 185 134 M185 134 C136.99 135.56, 85.61 131.87, 32 134 M185 134 C127.92 134.21, 69.91 134.22, 32 134 M32 134 C10 135.95, 0.07 123.05, 0 102 M32 134 C12.62 135.2, -0.85 123.37, 0 102 M0 102 C0.36 74.89, 0.18 48.96, 0 32 M0 102 C0.51 77.08, 0.99 53.23, 0 32 M0 32 C0.91 12.13, 9.34 -0.1, 32 0 M0 32 C-2.06 10.82, 12.64 0.95, 32 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(350.18152948684826 230.7622602621841) rotate(0 57.419944763183594 12.5)"><text x="57.419944763183594" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">Plugin clinet</text></g><g mask="url(#mask-oKA4s3pyjo4kyorp7tOI-)" stroke-linecap="round"><g transform="translate(820.2310312350744 814.3079863056755) rotate(0 -164.75580456834018 -251.39287292908057)"><path d="M-0.81 1.12 C-19.38 -22.6, -56.59 -59.2, -111.45 -143.37 C-166.32 -227.54, -293.29 -444.1, -330.01 -503.91 M0.96 0.67 C-17.67 -23.32, -56.6 -61.17, -111.84 -145.13 C-167.08 -229.08, -293.76 -443.17, -330.47 -503.05" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(820.2310312350744 814.3079863056755) rotate(0 -164.75580456834018 -251.39287292908057)"><path d="M-305.25 -482.43 C-312.74 -488.98, -322.27 -497.48, -329.12 -503.13 M-307.27 -483.22 C-316.03 -491.47, -322.65 -497.32, -330.96 -503.14" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(820.2310312350744 814.3079863056755) rotate(0 -164.75580456834018 -251.39287292908057)"><path d="M-322.83 -471.84 C-325.01 -481.41, -329.32 -493.05, -329.12 -503.13 M-324.85 -472.62 C-327.66 -484.45, -328.27 -493.93, -330.96 -503.14" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask id="mask-oKA4s3pyjo4kyorp7tOI-"><rect x="0" y="0" fill="#fff" width="1250.8256187304519" height="1417.353712349167"></rect><rect x="690.7678438893104" y="657.7462013274344" fill="#000" width="33.89997863769531" height="25" opacity="1"></rect></mask><g transform="translate(690.7678438893104 657.7462013274344) rotate(0 -35.292617222576155 -94.83108795083945)"><text x="16.949989318847656" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">call</text></g><g mask="url(#mask-ne33bhbs-yLaUby1tf7TI)" stroke-linecap="round"><g transform="translate(79.45526923866078 669.935423773688) rotate(0 -1.5715833556676841 -180.1656832099752)"><path d="M1.06 0.69 C0.6 -59.66, -2.5 -301.02, -3.56 -361.03 M0.16 0.01 C-0.33 -59.71, -3.68 -299.73, -4.2 -359.94" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(79.45526923866078 669.935423773688) rotate(0 -1.5715833556676841 -180.1656832099752)"><path d="M7.54 -330.9 C2.22 -340.23, 2.56 -344.89, -3.36 -358.66 M5.59 -332.63 C4.09 -339.61, 0.15 -347.22, -3.43 -359.92" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(79.45526923866078 669.935423773688) rotate(0 -1.5715833556676841 -180.1656832099752)"><path d="M-12.98 -330.66 C-13.54 -340.22, -8.44 -344.95, -3.36 -358.66 M-14.93 -332.39 C-10.81 -339.64, -9.15 -347.31, -3.43 -359.92" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask id="mask-ne33bhbs-yLaUby1tf7TI"><rect x="0" y="0" fill="#fff" width="183.6009723226037" height="1130.09643998327"></rect><rect x="10.292490023349501" y="464.85491566889687" fill="#000" width="134.1798553466797" height="50" opacity="1"></rect></mask><g transform="translate(10.292490023349501 464.85491566889687) rotate(0 67.59119585964359 24.914824894815865)"><text x="67.08992767333984" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">RegisterPlugin</text><text x="67.08992767333984" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge"></text></g><g mask="url(#mask-vAG_8_uMXXnHUl59qU-Gu)" stroke-linecap="round"><g transform="translate(291.85811222375855 645.7849646818368) rotate(0 -50.35444210175925 -166.92967031517765)"><path d="M0.58 -0.86 C-1.53 -22.31, 5.21 -72.6, -11.64 -127.97 C-28.49 -183.34, -85.83 -298.76, -100.5 -333.08 M-0.57 1.3 C-2.43 -20.54, 7.25 -73.59, -9.53 -129.66 C-26.32 -185.74, -86.38 -301.27, -101.29 -335.16" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(291.85811222375855 645.7849646818368) rotate(0 -50.35444210175925 -166.92967031517765)"><path d="M-81.34 -316.05 C-86.98 -322.27, -95.8 -329.08, -101.82 -335.45 M-79.33 -314.29 C-88.75 -322.76, -96.03 -331.04, -101.66 -335.5" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(291.85811222375855 645.7849646818368) rotate(0 -50.35444210175925 -166.92967031517765)"><path d="M-99.87 -307.22 C-98.36 -316.96, -99.98 -327.2, -101.82 -335.45 M-97.86 -305.46 C-99.82 -317.32, -99.7 -329.12, -101.66 -335.5" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask id="mask-vAG_8_uMXXnHUl59qU-Gu"><rect x="0" y="0" fill="#fff" width="492.3098474632352" height="1079.3672252023462"></rect><rect x="194.7972458348047" y="504.5054142402314" fill="#000" width="173.15985107421875" height="25" opacity="1"></rect></mask><g transform="translate(194.7972458348047 504.5054142402314) rotate(0 46.70642428719461 -25.650119873572294)"><text x="86.57992553710938" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">UpdateContainers</text></g><g mask="url(#mask-t4Ve6UCb4s2ddFOJFMKmq)" stroke-linecap="round"><g transform="translate(174.84757981386832 316.06689886760614) rotate(0 18.06697964380507 186.46561865802823)"><path d="M-1.16 0.81 C-0.49 45.65, -1.83 206.39, 4.58 268.19 C10.99 329.99, 32 354.1, 37.29 371.59 M0.43 0.19 C0.89 44.75, -2.47 204.24, 3.61 266.33 C9.69 328.42, 31.45 355.27, 36.93 372.74" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(174.84757981386832 316.06689886760614) rotate(0 18.06697964380507 186.46561865802823)"><path d="M17.42 351.1 C24.02 360.18, 29.93 366.12, 38.91 371.15 M15.58 351.09 C21.64 356.07, 25.19 360.55, 36.35 373.57" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(174.84757981386832 316.06689886760614) rotate(0 18.06697964380507 186.46561865802823)"><path d="M36.16 342.73 C36.17 354.55, 35.54 363.41, 38.91 371.15 M34.32 342.72 C36.22 349.55, 35.56 355.91, 36.35 373.57" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask id="mask-t4Ve6UCb4s2ddFOJFMKmq"><rect x="0" y="0" fill="#fff" width="311.370283987698" height="787.9704674162213"></rect><rect x="91.97820374459491" y="571.2529728339814" fill="#000" width="174.9998321533203" height="25" opacity="1"></rect></mask><g transform="translate(91.97820374459491 571.2529728339814) rotate(0 100.93635571307848 -68.72045530834703)"><text x="87.49991607666016" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">update failed list</text></g><g mask="url(#mask-9VLM9lSMU1NGhxSIiyzQb)" stroke-linecap="round"><g transform="translate(486.39206745127194 665.8644940354698) rotate(0 -5.3612081416674755 -176.06974041674312)"><path d="M-0.05 0.5 C-0.4 -16.38, -1.61 -41.92, -3.24 -100.49 C-4.86 -159.07, -8.59 -308.84, -9.82 -350.92 M-1.53 -0.28 C-1.87 -17.06, -2.26 -40.4, -3.78 -99.12 C-5.3 -157.85, -9.5 -310.43, -10.67 -352.64" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(486.39206745127194 665.8644940354698) rotate(0 -5.3612081416674755 -176.06974041674312)"><path d="M1.2 -323.47 C-2.6 -330.72, -2.34 -337.69, -12.18 -351.51 M1.13 -324.73 C-3 -333, -6.33 -339.73, -9.76 -353.01" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(486.39206745127194 665.8644940354698) rotate(0 -5.3612081416674755 -176.06974041674312)"><path d="M-19.31 -322.9 C-18.94 -330.26, -14.51 -337.35, -12.18 -351.51 M-19.38 -324.16 C-17.7 -332.58, -15.22 -339.46, -9.76 -353.01" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask id="mask-9VLM9lSMU1NGhxSIiyzQb"><rect x="0" y="0" fill="#fff" width="596.034632297163" height="1117.975238016037"></rect><rect x="441.14604286330905" y="553.1863734075687" fill="#000" width="82.47990417480469" height="25" opacity="1"></rect></mask><g transform="translate(441.14604286330905 553.1863734075687) rotate(0 39.88481644629542 -63.391619788842036)"><text x="41.239952087402344" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">response</text></g><g mask="url(#mask-LZ9ev888BBSucr_xRdptu)" stroke-linecap="round"><g transform="translate(1194.168112626724 267.5152644533837) rotate(0 2.728295080510293 173.14359760798106)"><path d="M0.77 -0.32 C1.73 57.57, 5.04 290.14, 5.74 347.82 M-0.29 -1.53 C0.51 56.01, 3.43 288.3, 4.7 346.32" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(1194.168112626724 267.5152644533837) rotate(0 2.728295080510293 173.14359760798106)"><path d="M-6.55 318.01 C-0.75 324.93, -1.07 335.95, 4.37 345.35 M-6.39 317.96 C-3.26 327.28, 1.34 334.23, 4.46 346.1" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(1194.168112626724 267.5152644533837) rotate(0 2.728295080510293 173.14359760798106)"><path d="M13.97 317.67 C13.69 324.83, 7.31 335.94, 4.37 345.35 M14.13 317.62 C11.29 327.05, 9.92 334.1, 4.46 346.1" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask id="mask-LZ9ev888BBSucr_xRdptu"><rect x="0" y="0" fill="#fff" width="1300.0847276937782" height="714.1446988196667"></rect><rect x="1180.1764308414035" y="428.3299816365252" fill="#000" width="33.89997863769531" height="25" opacity="1"></rect></mask><g transform="translate(1180.1764308414035 428.3299816365252) rotate(0 16.719976865830745 12.328880424839554)"><text x="16.949989318847656" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">call</text></g><g stroke-linecap="round"><g transform="translate(1276.550934970028 58.53551739195791) rotate(0 0.5093983992495623 76.59715198779622)"><path d="M-0.17 1.19 C-0.21 26.47, 0.14 126.22, 0.53 151.37 M-1.72 0.76 C-1.46 26.2, 2.16 127.01, 2.74 152.43" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1276.550934970028 58.53551739195791) rotate(0 0.5093983992495623 76.59715198779622)"><path d="M-6.38 122.97 C-5.9 135.86, -0.68 143.9, 2.56 153.97 M-8.93 125.39 C-4.2 132.65, -2 141.21, 2.71 152.63" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1276.550934970028 58.53551739195791) rotate(0 0.5093983992495623 76.59715198779622)"><path d="M14.13 122.36 C7.14 135.44, 4.89 143.7, 2.56 153.97 M11.58 124.78 C10.12 132.15, 6.13 140.9, 2.71 152.63" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask></mask><g mask="url(#mask-RR6eGbAn7Aj0xyVWgZKVy)" stroke-linecap="round"><g transform="translate(808.8133225597356 442.2790282891556) rotate(0 0.21868587230858338 -82.91315751214916)"><path d="M-0.95 -0.91 C-0.69 -28.95, 1.21 -139.38, 1.39 -167.06 M0.75 1.23 C0.93 -26.12, 1.08 -137.8, 0.8 -165.63" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(808.8133225597356 442.2790282891556) rotate(0 0.21868587230858338 -82.91315751214916)"><path d="M9.64 -136.34 C7.59 -146.56, 4.78 -152.09, 0.84 -166.36 M12.06 -137.84 C8.33 -145.09, 5.78 -152.15, 0.1 -165.04" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(808.8133225597356 442.2790282891556) rotate(0 0.21868587230858338 -82.91315751214916)"><path d="M-10.88 -136.27 C-7.08 -146.35, -4.02 -151.9, 0.84 -166.36 M-8.46 -137.77 C-6.74 -145.21, -3.83 -152.29, 0.1 -165.04" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask id="mask-RR6eGbAn7Aj0xyVWgZKVy"><rect x="0" y="0" fill="#fff" width="909.523029817289" height="709.1361420819394"></rect><rect x="733.1682448530629" y="346.3504713927632" fill="#000" width="151.99986267089844" height="25" opacity="1"></rect></mask><g transform="translate(733.1682448530629 346.3504713927632) rotate(0 75.8637635789812 13.01539938424321)"><text x="75.99993133544922" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">adjust oci spec</text></g><g mask="url(#mask-1ulPStvL1fhx2LLr_N0fq)" stroke-linecap="round"><g transform="translate(1117.9225807913301 693.5564619959187) rotate(0 6.252196245743221 44.06817081043482)"><path d="M0.68 -0.2 C2.59 14.62, 8.63 73.28, 10.52 88.01 M-0.42 -1.35 C1.97 13.74, 10.9 74.35, 12.93 89.49" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1117.9225807913301 693.5564619959187) rotate(0 6.252196245743221 44.06817081043482)"><path d="M-1.53 62.05 C1.54 70.32, 4.88 73.88, 12.24 88.33 M-1.43 62.8 C4.43 73.42, 9.18 83.95, 13.65 90.43" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1117.9225807913301 693.5564619959187) rotate(0 6.252196245743221 44.06817081043482)"><path d="M18.79 59.17 C17.36 68.19, 16.21 72.39, 12.24 88.33 M18.89 59.92 C16.99 71.57, 13.94 83.21, 13.65 90.43" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask id="mask-1ulPStvL1fhx2LLr_N0fq"><rect x="0" y="0" fill="#fff" width="1229.0243431323681" height="881.6728113157524"></rect><rect x="1088.6534851552085" y="725.114636655835" fill="#000" width="69.63995361328125" height="25" opacity="1"></rect></mask><g transform="translate(1088.6534851552085 725.114636655835) rotate(0 35.52129188186484 12.509996150518418)"><text x="34.819976806640625" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">manage</text></g><g stroke-linecap="round" transform="translate(1645.0211967892924 620.782216988212) rotate(0 92 37.5)"><path d="M4.99 4.34 C4.99 4.34, 4.99 4.34, 4.99 4.34 M4.99 4.34 C4.99 4.34, 4.99 4.34, 4.99 4.34 M0.79 15.26 C5.07 12.76, 9 7.47, 11.95 2.43 M0.79 15.26 C2.74 12.17, 5.08 9.05, 11.95 2.43 M1.19 20.91 C8.02 13.22, 14.64 5.93, 18.9 0.53 M1.19 20.91 C6.9 14.83, 12.07 9.25, 18.9 0.53 M0.93 27.31 C8.09 23.02, 11.44 15.31, 24.54 0.14 M0.93 27.31 C9.19 17.13, 17.97 6.8, 24.54 0.14 M1.32 32.95 C9.55 21.93, 19.13 10.94, 29.53 0.5 M1.32 32.95 C7.31 24.71, 13.51 18.57, 29.53 0.5 M1.06 39.35 C10.06 29.62, 19.2 20.69, 35.17 0.1 M1.06 39.35 C12.04 25.47, 23.99 11.94, 35.17 0.1 M0.8 45.74 C15.71 30.56, 28.27 13.53, 40.16 0.46 M0.8 45.74 C13.92 30.21, 29.03 13.68, 40.16 0.46 M1.19 51.39 C15.86 36.41, 29.04 19.89, 45.8 0.07 M1.19 51.39 C17.49 33.91, 34.53 14.91, 45.8 0.07 M0.93 57.78 C13.55 43.25, 25.11 31.58, 50.79 0.43 M0.93 57.78 C20.57 35.73, 40.6 13.71, 50.79 0.43 M1.98 62.67 C19.7 40.84, 38.21 20.16, 56.43 0.03 M1.98 62.67 C22.62 37.77, 45.13 14.41, 56.43 0.03 M4.34 66.05 C19.59 45.61, 35.6 27.53, 61.42 0.39 M4.34 66.05 C23.03 43.81, 42.45 23.36, 61.42 0.39 M5.4 70.94 C23.99 50.46, 45.42 25.79, 67.06 0 M5.4 70.94 C24.54 48.27, 45.55 23.82, 67.06 0 M9.07 72.81 C23.28 59.56, 37.02 42.52, 72.05 0.36 M9.07 72.81 C29.88 50.06, 50.01 25.77, 72.05 0.36 M12.09 75.43 C26.56 60.4, 41 44.02, 77.69 -0.04 M12.09 75.43 C37.71 46.77, 63.33 17.39, 77.69 -0.04 M17.08 75.79 C40.3 48.58, 66.96 19.08, 82.68 0.32 M17.08 75.79 C41.45 48.03, 66.14 19.7, 82.68 0.32 M21.41 76.91 C42.94 51.69, 63.63 30.9, 88.32 -0.07 M21.41 76.91 C41.19 55.51, 58.79 32.33, 88.32 -0.07 M26.39 77.27 C44.57 54.36, 65.5 33.79, 93.31 0.29 M26.39 77.27 C46.61 54.9, 65.88 33.58, 93.31 0.29 M32.04 76.87 C45.93 61.54, 61.34 44.26, 98.3 0.65 M32.04 76.87 C50.99 55.11, 69.96 34.47, 98.3 0.65 M37.02 77.23 C52.28 60.21, 67.27 41.66, 103.94 0.25 M37.02 77.23 C58.54 53.11, 78.78 28.87, 103.94 0.25 M42.67 76.84 C61.47 57.23, 75.31 37.68, 108.93 0.61 M42.67 76.84 C66.46 48.45, 91.35 21.37, 108.93 0.61 M47.65 77.2 C70.08 52.24, 91.65 22.72, 114.57 0.22 M47.65 77.2 C70.33 51.62, 90.61 26.91, 114.57 0.22 M53.3 76.81 C76.56 53.69, 96.17 27.84, 119.56 0.58 M53.3 76.81 C67.86 61.64, 82.01 45.16, 119.56 0.58 M58.28 77.17 C80.45 51.62, 103.78 22.97, 125.2 0.19 M58.28 77.17 C81.99 48.15, 107.49 18.22, 125.2 0.19 M63.93 76.77 C89.03 49.82, 113.46 19.02, 130.19 0.55 M63.93 76.77 C80.58 57.97, 97.09 38.91, 130.19 0.55 M68.91 77.13 C87.33 55.31, 102.23 36.92, 135.83 0.15 M68.91 77.13 C97.1 47.52, 122.14 15.18, 135.83 0.15 M73.9 77.49 C97.92 50.58, 122.01 21.13, 140.82 0.51 M73.9 77.49 C90.65 60.71, 106.62 42.24, 140.82 0.51 M79.54 77.1 C97.83 55.43, 115.3 38.4, 146.46 0.12 M79.54 77.1 C99.81 53.59, 120.33 30.53, 146.46 0.12 M84.53 77.46 C103.65 54.45, 125.17 34.33, 151.45 0.48 M84.53 77.46 C99.89 61.97, 112.3 44.52, 151.45 0.48 M90.17 77.06 C103.72 59.44, 119.78 43.48, 157.09 0.08 M90.17 77.06 C107.04 57.24, 124.45 37.9, 157.09 0.08 M95.16 77.42 C121.41 47.41, 147.06 17.84, 162.08 0.44 M95.16 77.42 C120.04 49.21, 145.37 19.56, 162.08 0.44 M100.8 77.03 C125.46 48.42, 148.6 23.18, 166.41 1.56 M100.8 77.03 C113.91 62.11, 127.88 44.76, 166.41 1.56 M105.79 77.39 C127.94 52.52, 148.44 30.74, 171.4 1.92 M105.79 77.39 C119.65 60.9, 135.1 43.9, 171.4 1.92 M111.43 76.99 C128.25 60.11, 141.79 41.37, 175.73 3.03 M111.43 76.99 C134.72 49.33, 157.2 23.81, 175.73 3.03 M116.42 77.35 C136.57 51.91, 158.98 28.5, 179.4 4.9 M116.42 77.35 C142.05 49.42, 167.22 21.02, 179.4 4.9 M122.06 76.96 C141.64 55.67, 157.01 35.15, 181.11 9.04 M122.06 76.96 C142.46 54.49, 161.46 30.28, 181.11 9.04 M127.05 77.32 C148.67 53.58, 172.07 28.72, 182.82 13.17 M127.05 77.32 C140.94 62.27, 153.11 46.84, 182.82 13.17 M132.69 76.92 C153.89 53.97, 174.44 29.88, 185.84 15.79 M132.69 76.92 C151.13 55.54, 172.36 31.73, 185.84 15.79 M137.68 77.28 C158.5 56.49, 177.31 31.84, 185.57 22.19 M137.68 77.28 C148.06 64.94, 158.49 54, 185.57 22.19 M143.32 76.89 C159.73 60.46, 174.47 42.56, 185.97 27.83 M143.32 76.89 C155.41 61.78, 170.25 46.88, 185.97 27.83 M148.31 77.25 C162.34 63.55, 173.57 46.84, 185.71 34.23 M148.31 77.25 C160.26 63.13, 173.95 48.79, 185.71 34.23 M153.95 76.86 C164.64 66.15, 176.72 53.72, 186.1 39.87 M153.95 76.86 C162.78 68.06, 169.02 58.88, 186.1 39.87 M158.94 77.22 C168.7 64.98, 179.55 52.94, 185.84 46.27 M158.94 77.22 C167.79 67.51, 175.43 57.7, 185.84 46.27 M165.9 75.31 C170.75 67.28, 177.58 61.91, 185.58 52.67 M165.9 75.31 C170.26 68.9, 176.09 63.3, 185.58 52.67 M172.2 74.16 C175.56 68.3, 181.1 63.46, 185.97 58.31 M172.2 74.16 C176.37 67.9, 182.15 61.62, 185.97 58.31" stroke="#a5d8ff" stroke-width="0.5" fill="none"></path><path d="M18.75 0 M18.75 0 C62.14 0.01, 99.91 -1.66, 165.25 0 M18.75 0 C63.92 0.49, 111.29 1.2, 165.25 0 M165.25 0 C179.09 -0.79, 185.48 4.92, 184 18.75 M165.25 0 C177.07 -1.85, 183.91 6.05, 184 18.75 M184 18.75 C184.29 31.91, 185.43 45.12, 184 56.25 M184 18.75 C184.5 27.39, 183.86 38.28, 184 56.25 M184 56.25 C185.25 70.29, 177.47 74.73, 165.25 75 M184 56.25 C182.22 67.83, 176.19 75.83, 165.25 75 M165.25 75 C129.47 76.38, 96.4 75.08, 18.75 75 M165.25 75 C115.87 75.6, 64.92 76, 18.75 75 M18.75 75 C5.49 73.83, -1.29 69.23, 0 56.25 M18.75 75 C6.9 74.28, -2.03 70.73, 0 56.25 M0 56.25 C-0.53 48.17, -1.47 41.28, 0 18.75 M0 56.25 C-0.57 47.91, -0.71 38.66, 0 18.75 M0 18.75 C-0.28 5.47, 6.16 -1.77, 18.75 0 M0 18.75 C-1.33 7.11, 4.07 0.62, 18.75 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(1684.191240734605 645.782216988212) rotate(0 52.8299560546875 12.5)"><text x="52.8299560546875" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">NRI result</text></g><g mask="url(#mask-a5xCWBkOEDg0mBhbi08Xu)" stroke-linecap="round"><g transform="translate(1329.038532780583 615.3599280412636) rotate(0 -7.292721704084215 -178.76528187787972)"><path d="M0.92 0.02 C-1.37 -59.55, -11.75 -298.19, -14.33 -357.55 M-0.05 -1.01 C-2.49 -60.41, -12.9 -296.74, -15.51 -356.23" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1329.038532780583 615.3599280412636) rotate(0 -7.292721704084215 -178.76528187787972)"><path d="M-3.98 -329.25 C-7.57 -336.56, -8.11 -340.55, -16.24 -356.71 M-4.72 -327.93 C-6.48 -335.26, -10.56 -344.73, -15.14 -355.57" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1329.038532780583 615.3599280412636) rotate(0 -7.292721704084215 -178.76528187787972)"><path d="M-24.48 -328.35 C-22.92 -336.04, -18.3 -340.26, -16.24 -356.71 M-25.22 -327.03 C-21.28 -334.65, -19.65 -344.37, -15.14 -355.57" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask id="mask-a5xCWBkOEDg0mBhbi08Xu"><rect x="0" y="0" fill="#fff" width="1442.9294766770977" height="1072.5028069823911"></rect><rect x="1238.82313285381" y="424.2884885706999" fill="#000" width="166.53985595703125" height="25" opacity="1"></rect></mask><g transform="translate(1238.82313285381 424.2884885706999) rotate(0 82.92267822268877 12.30615759268403)"><text x="83.26992797851562" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">update container</text></g><g mask="url(#mask-nUHR_9L9jV7EuSFJ_2S9_)" stroke-linecap="round"><g transform="translate(1642.5089055206602 655.7657258493055) rotate(0 -85.27311736005413 0.9396532236555686)"><path d="M-0.44 -0.41 C-29.27 0.18, -143.77 3.13, -172.08 3.55 M1.53 -1.67 C-26.99 -1.4, -141.06 1.14, -169.73 1.95" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1642.5089055206602 655.7657258493055) rotate(0 -85.27311736005413 0.9396532236555686)"><path d="M-142.48 -10.15 C-150.66 -3.15, -163.59 -1.32, -170.17 1.89 M-141.07 -8.05 C-150.97 -6.14, -159.4 -1.16, -170.25 1.7" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1642.5089055206602 655.7657258493055) rotate(0 -85.27311736005413 0.9396532236555686)"><path d="M-141.98 10.36 C-150.35 9.62, -163.47 3.72, -170.17 1.89 M-140.57 12.46 C-150.48 7.81, -159.07 6.22, -170.25 1.7" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask id="mask-nUHR_9L9jV7EuSFJ_2S9_"><rect x="0" y="0" fill="#fff" width="1913.8913538846825" height="758.3162269196455"></rect><rect x="1497.5477457307386" y="632.0409763844755" fill="#000" width="118.53987121582031" height="50" opacity="1"></rect></mask><g transform="translate(1497.5477457307388 632.0409763844755) rotate(0 59.68804242986721 24.664402688485552)"><text x="59.269935607910156" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">merge all </text><text x="59.269935607910156" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">plugin result</text></g></svg> +\ No newline at end of file +diff --git a/docs/design/detailed/NRI/nri_init.svg b/docs/design/detailed/NRI/nri_init.svg +new file mode 100644 +index 00000000..f1628f65 +--- /dev/null ++++ b/docs/design/detailed/NRI/nri_init.svg +@@ -0,0 +1,16 @@ ++<svg version="1.1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 2823.2323848839956 1797.06763419055" width="2823.2323848839956" height="1797.06763419055"> ++ <!-- svg-source:excalidraw --> ++ ++ <defs> ++ <style class="style-fonts"> ++ @font-face { ++ font-family: "Virgil"; ++ src: url("https://excalidraw.com/Virgil.woff2"); ++ } ++ @font-face { ++ font-family: "Cascadia"; ++ src: url("https://excalidraw.com/Cascadia.woff2"); ++ } ++ </style> ++ </defs> ++ <rect x="0" y="0" width="2823.2323848839956" height="1797.06763419055" fill="#ffffff"></rect><g stroke-linecap="round" transform="translate(127.22668132042895 509.2690219846654) rotate(0 155 26)"><path d="M13 0 M13 0 C97.84 1.56, 180.27 1.06, 297 0 M13 0 C82.01 -0.1, 149.35 0.37, 297 0 M297 0 C307.57 -1.35, 311.68 5.88, 310 13 M297 0 C307.11 -1.56, 311.14 5.32, 310 13 M310 13 C310.25 22.96, 310.98 32.29, 310 39 M310 13 C309.3 20.45, 309.66 27.82, 310 39 M310 39 C310.38 47.07, 304.23 51.59, 297 52 M310 39 C310.26 48.61, 305.79 51.21, 297 52 M297 52 C209.21 52.16, 121.43 52.98, 13 52 M297 52 C239.24 53.33, 180.92 53.51, 13 52 M13 52 C5.54 53.05, -0.44 48.87, 0 39 M13 52 C2.8 49.91, -0.51 47.23, 0 39 M0 39 C-1.83 33.5, 0.01 27.55, 0 13 M0 39 C0.02 28.3, 0.19 18.75, 0 13 M0 13 C0.49 3.38, 4.11 1, 13 0 M0 13 C1.58 4.17, 4.83 0.94, 13 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(201.93675669884692 522.7690219846654) rotate(0 80.28992462158203 12.5)"><text x="80.28992462158203" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">pre_init_daemon</text></g><g stroke-linecap="round"><g transform="translate(439.82676066613215 539.1748226101463) rotate(0 60.909572890781135 -0.07241361157139181)"><path d="M0.41 -0.71 C20.72 -0.94, 102.17 -1.38, 122.65 -1.68 M-0.83 1.54 C19.27 1.51, 101.16 0.22, 121.79 -0.33" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(439.82676066613215 539.1748226101463) rotate(0 60.909572890781135 -0.07241361157139181)"><path d="M92.64 10.8 C103.54 7.79, 112.46 0.8, 121.11 1.57 M94.44 9.82 C100.73 8.19, 107.2 4.5, 121.59 -0.11" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(439.82676066613215 539.1748226101463) rotate(0 60.909572890781135 -0.07241361157139181)"><path d="M92.22 -9.72 C103.22 -5.19, 112.3 -4.64, 121.11 1.57 M94.03 -10.7 C100.58 -7.55, 107.15 -6.47, 121.59 -0.11" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(570.5266996309759 515.6691074338842) rotate(0 157 29)"><path d="M14.5 0 M14.5 0 C102.8 -2.72, 193.53 -0.03, 299.5 0 M14.5 0 C81.29 -2.19, 147.5 -1.35, 299.5 0 M299.5 0 C310.08 -1.49, 313.59 5.28, 314 14.5 M299.5 0 C310.28 1.34, 315.51 4.77, 314 14.5 M314 14.5 C313.43 22.57, 312.7 32.38, 314 43.5 M314 14.5 C313.9 22.75, 313.84 29.84, 314 43.5 M314 43.5 C314.23 54.22, 309.98 58.96, 299.5 58 M314 43.5 C312.92 55.39, 309.32 55.78, 299.5 58 M299.5 58 C213.78 57.06, 129.46 59, 14.5 58 M299.5 58 C198.56 58.61, 98.47 59.16, 14.5 58 M14.5 58 C2.97 59.32, 0.13 51.51, 0 43.5 M14.5 58 C4.7 58.21, -0.79 55.27, 0 43.5 M0 43.5 C-0.89 36.69, 0.36 30.29, 0 14.5 M0 43.5 C0.72 36.11, 0.25 29.27, 0 14.5 M0 14.5 C-1.36 4.42, 3.92 1.56, 14.5 0 M0 14.5 C1.32 5.84, 4.11 -0.48, 14.5 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(577.8568387911322 532.1691074338842) rotate(0 149.66986083984375 12.5)"><text x="149.66986083984375" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">merge_json_confs_into_global</text></g><g transform="translate(548.4266935274603 475.66907691630604) rotate(0 201.44981384277344 12.5)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">get config for nri plugin from deamon.json</text></g><g stroke-linecap="round"><g transform="translate(270.83650872565477 563.0690708127904) rotate(0 -0.05160679541131685 99.8160984170936)"><path d="M1.01 0.93 C0.72 34.09, -0.27 165.06, -0.55 198.15 M0.07 0.37 C-0.4 33.74, -0.9 166.42, -1.11 199.26" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(270.83650872565477 563.0690708127904) rotate(0 -0.05160679541131685 99.8160984170936)"><path d="M-9.68 172.27 C-8.77 183.08, -2.56 194.17, -2.6 198.85 M-10.57 170.99 C-7.81 180.05, -5.27 188.89, -1.45 198.61" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(270.83650872565477 563.0690708127904) rotate(0 -0.05160679541131685 99.8160984170936)"><path d="M10.84 172.37 C3.64 183.32, 1.74 194.36, -2.6 198.85 M9.95 171.1 C6.09 180.24, 2.01 189.04, -1.45 198.61" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(127.7267423555852 774.6690463987279) rotate(0 150.5 22)"><path d="M11 0 M11 0 C73.95 1.02, 138.43 -0.37, 290 0 M11 0 C89.06 -1.14, 167.12 -0.47, 290 0 M290 0 C298.14 0.96, 300.06 5.6, 301 11 M290 0 C297.49 -2.22, 300.93 5.64, 301 11 M301 11 C299.94 18.89, 301.9 23.92, 301 33 M301 11 C300.63 18.73, 299.92 27.51, 301 33 M301 33 C301.13 38.68, 297.22 44.18, 290 44 M301 33 C300.21 42.44, 295.87 42.78, 290 44 M290 44 C210.42 44.42, 129.79 43.87, 11 44 M290 44 C192.51 44.16, 94.16 43.69, 11 44 M11 44 C2.75 45.56, 1.15 41.21, 0 33 M11 44 C2.94 43.52, -1.94 38.93, 0 33 M0 33 C0.63 27.13, 0.9 22.03, 0 11 M0 33 C0.07 28.33, -0.53 22.58, 0 11 M0 11 C1.46 5.05, 3.34 -0.06, 11 0 M0 11 C0.9 3.43, 5.38 1.81, 11 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(193.83681925988208 784.1690463987279) rotate(0 84.38992309570312 12.5)"><text x="84.38992309570312" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">nri_adaption_init</text></g><g stroke-linecap="round"><g transform="translate(433.15041198587664 805.4975260035035) rotate(0 68.70051974178017 -19.62116838704651)"><path d="M-0.86 -0.24 C22.15 -6.37, 115.09 -30.83, 138.26 -37.13 M0.89 -1.42 C23.73 -7.88, 114.62 -33.08, 137.38 -39" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(433.15041198587664 805.4975260035035) rotate(0 68.70051974178017 -19.62116838704651)"><path d="M112.44 -21.52 C121.54 -26.96, 127.85 -32.33, 138.44 -38.19 M112.91 -22.71 C121.43 -27.12, 130.03 -34.67, 136.71 -39.45" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(433.15041198587664 805.4975260035035) rotate(0 68.70051974178017 -19.62116838704651)"><path d="M107.09 -41.33 C117.84 -40.42, 125.85 -39.48, 138.44 -38.19 M107.56 -42.52 C117.85 -40.1, 128.31 -40.75, 136.71 -39.45" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(583.7415679221112 843.1310854109825) rotate(0 180 30)"><path d="M15 0 M15 0 C117.24 0.04, 221.15 0.44, 345 0 M15 0 C129.82 -2.84, 244.53 -1.6, 345 0 M345 0 C355.08 -1.37, 358.66 4.09, 360 15 M345 0 C356.39 1.2, 359.49 6.39, 360 15 M360 15 C359.35 20.14, 361.04 29.13, 360 45 M360 15 C359.64 22.43, 359.35 27.51, 360 45 M360 45 C359.72 53.96, 355.04 60.2, 345 60 M360 45 C360.57 53.9, 354.74 61.15, 345 60 M345 60 C223.92 59.56, 101.44 59.87, 15 60 M345 60 C220.06 60.84, 94.98 60.57, 15 60 M15 60 C4.47 61.69, 0.11 55.82, 0 45 M15 60 C5.18 61.25, 1.23 53.78, 0 45 M0 45 C1.3 33.99, 0.68 26.26, 0 15 M0 45 C0.44 33.82, 0.79 23.64, 0 15 M0 15 C-0.96 3.44, 3.1 -0.8, 15 0 M0 15 C1.84 4.37, 6.89 0.79, 15 0" stroke="#1e1e1e" stroke-width="1" fill="none"></path></g><g transform="translate(589.5216887717206 860.6310854109825) rotate(0 174.21987915039062 12.5)"><text x="174.21987915039062" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">NRIAdaptation::GetInstance()->Init</text></g><g stroke-linecap="round" transform="translate(92.77666911339747 436.21902274760487) rotate(0 183.49998474121105 220.50002098083496)"><path d="M32 0 M32 0 C136.6 1.13, 236.97 0.99, 335 0 M32 0 C148.39 1.18, 266.21 1.89, 335 0 M335 0 C355.89 1.2, 365.67 8.85, 367 32 M335 0 C355.82 -0.44, 364.86 12.18, 367 32 M367 32 C365.22 147.11, 366.22 262.28, 367 409 M367 32 C366.69 125.24, 366.83 218.06, 367 409 M367 409 C366.77 431.33, 357.71 440.86, 335 441 M367 409 C367.5 431.27, 354.77 440.53, 335 441 M335 441 C258.55 441.81, 180.53 441.04, 32 441 M335 441 C262.62 443.16, 190.22 443.26, 32 441 M32 441 C11.74 439.94, 0.73 431.79, 0 409 M32 441 C11.01 442.75, -1.04 431.36, 0 409 M0 409 C-1.61 268.82, -1.7 128.6, 0 32 M0 409 C0.52 266.07, 1.58 123.46, 0 32 M0 32 C1.64 11.35, 9.03 1.42, 32 0 M0 32 C-1.36 11.03, 10.46 -0.77, 32 0" stroke="#1e1e1e" stroke-width="4" fill="none"></path></g><g stroke-linecap="round"><g transform="translate(945.450430743124 888.3072622604932) rotate(0 129.03446739145397 -340.98443489770807)"><path d="M0.08 -1.16 C24.32 -38.77, 101.64 -110.9, 144.86 -224.51 C188.08 -338.12, 240.46 -606.48, 259.41 -682.82 M-1.34 0.85 C22.79 -36.57, 100.56 -112.3, 143.9 -226.09 C187.23 -339.89, 239.58 -605.99, 258.66 -681.93" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(945.450430743124 888.3072622604932) rotate(0 129.03446739145397 -340.98443489770807)"><path d="M260.51 -652.23 C262.91 -659.63, 258.87 -668.58, 257.76 -680.72 M261.78 -653.08 C260.96 -662.83, 258.92 -672.46, 258.73 -682.76" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(945.450430743124 888.3072622604932) rotate(0 129.03446739145397 -340.98443489770807)"><path d="M240.5 -656.76 C248.01 -663.03, 249.04 -670.83, 257.76 -680.72 M241.76 -657.61 C247.98 -665.88, 253 -673.92, 258.73 -682.76" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1212.9521093485976 185.45607793967065) rotate(0 123 22.5)"><path d="M11.25 0 M11.25 0 C81.26 0.12, 147.15 -1.1, 234.75 0 M11.25 0 C67.98 -0.49, 124.52 -1.79, 234.75 0 M234.75 0 C241.8 -0.38, 244.14 5.07, 246 11.25 M234.75 0 C242.4 -1.9, 245.87 3.96, 246 11.25 M246 11.25 C245.15 16.95, 245.16 23.25, 246 33.75 M246 11.25 C245.94 18.73, 246.74 25.14, 246 33.75 M246 33.75 C246.43 42.06, 240.89 44.59, 234.75 45 M246 33.75 C244.95 43.04, 243.57 46, 234.75 45 M234.75 45 C173.69 42.93, 116.19 43.47, 11.25 45 M234.75 45 C159 45.59, 82.65 46.68, 11.25 45 M11.25 45 C4.05 46.52, -0.91 42.14, 0 33.75 M11.25 45 C5.43 46.59, -0.37 41.18, 0 33.75 M0 33.75 C1.59 25.17, -1.54 16.6, 0 11.25 M0 33.75 C0.64 28.24, -1 22.75, 0 11.25 M0 11.25 C-1.19 4.07, 3.57 -0.67, 11.25 0 M0 11.25 C-1.3 2.39, 4.89 0.74, 11.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1241.7321920512343 195.45607793967065) rotate(0 94.21991729736328 12.5)"><text x="94.21991729736328" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">init var from config</text></g><g transform="translate(113.7766920015813 449.9690418210912) rotate(0 38.66798400878906 17.5)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="28px" fill="#e03131" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">main.c</text></g><g stroke-linecap="round" transform="translate(506.77668437218665 114.01751937120207) rotate(0 257.9999923706055 236.45753887329056)"><path d="M32 0 M32 0 C137.73 -1.53, 243.03 -1.95, 484 0 M32 0 C126.02 0.73, 220.97 0.02, 484 0 M484 0 C505.22 0.18, 515.32 12.5, 516 32 M484 0 C503.87 -1.22, 515.67 9.47, 516 32 M516 32 C515.7 155.38, 516.78 279.77, 516 440.92 M516 32 C516.11 151.67, 516.24 272, 516 440.92 M516 440.92 C517.15 463.12, 504.7 472.5, 484 472.92 M516 440.92 C514.06 460.85, 504.73 474.86, 484 472.92 M484 472.92 C346.44 473.6, 207.33 473.43, 32 472.92 M484 472.92 C315.24 474.78, 147.27 474.89, 32 472.92 M32 472.92 C10.34 472.85, 0.78 462.04, 0 440.92 M32 472.92 C12.38 474.72, -1.1 460.45, 0 440.92 M0 440.92 C0.6 358, 1.28 272.38, 0 32 M0 440.92 C1.29 289.07, 1.02 137.27, 0 32 M0 32 C0.99 11.31, 12.29 -0.65, 32 0 M0 32 C1.71 12.1, 12.35 -0.54, 32 0" stroke="#1e1e1e" stroke-width="4" fill="none"></path></g><g transform="translate(547.9743099350771 134.48631409327209) rotate(0 100.12794494628906 17.5)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="28px" fill="#e03131" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">isulad_config.h</text></g><g stroke-linecap="round" transform="translate(515.5099319172167 611.941277606682) rotate(0 256.8000488281252 230.80539402817294)"><path d="M32 0 M32 0 C157.51 2.09, 280.94 1.97, 481.6 0 M32 0 C170.65 -2.25, 309.85 -2.24, 481.6 0 M481.6 0 C502.97 0.2, 514.09 9.71, 513.6 32 M481.6 0 C502.67 1.15, 515.18 10.5, 513.6 32 M513.6 32 C514.29 159.38, 515.41 288.46, 513.6 429.61 M513.6 32 C512.4 168.29, 512.73 305.76, 513.6 429.61 M513.6 429.61 C513.71 451.76, 503.09 462.7, 481.6 461.61 M513.6 429.61 C514.83 449.72, 503.78 463.29, 481.6 461.61 M481.6 461.61 C340.4 461.33, 199 460.47, 32 461.61 M481.6 461.61 C348.46 461.76, 214.36 461, 32 461.61 M32 461.61 C8.77 460.81, 1.6 450.4, 0 429.61 M32 461.61 C12.56 462.4, -1.88 452.57, 0 429.61 M0 429.61 C-0.74 333.25, -1.03 238.74, 0 32 M0 429.61 C-1.42 297.81, -1.42 165.17, 0 32 M0 32 C1.47 12.11, 10 -0.62, 32 0 M0 32 C1.91 8.38, 9.21 1.6, 32 0" stroke="#1e1e1e" stroke-width="4" fill="none"></path></g><g transform="translate(543.598839794821 638.47456211189) rotate(0 106.91793060302734 17.5)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="28px" fill="#e03131" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">nri_plugin_ops.h</text></g><g stroke-linecap="round"><g transform="translate(1205.402898670562 211.38975913473496) rotate(0 -106.62272231449916 -4.240394101712809)"><path d="M-0.07 0.11 C-35.42 -0.9, -176.63 -5.56, -212.01 -7.04 M-1.56 -0.88 C-37.11 -2.11, -178.46 -7.44, -213.18 -8.59" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1205.402898670562 211.38975913473496) rotate(0 -106.62272231449916 -4.240394101712809)"><path d="M-184.46 -18.52 C-191.63 -16.29, -203.26 -12.47, -214.22 -8.55 M-184.75 -17.34 C-192.78 -15.18, -202.1 -11.83, -213.86 -8.79" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1205.402898670562 211.38975913473496) rotate(0 -106.62272231449916 -4.240394101712809)"><path d="M-185.19 1.99 C-191.97 -2.07, -203.38 -4.53, -214.22 -8.55 M-185.48 3.17 C-193.34 -1.02, -202.43 -4.04, -213.86 -8.79" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask></mask><g stroke-linecap="round"><g transform="translate(1339.6891897432415 228.46001580359916) rotate(0 0.6583335483969961 25.812424113767975)"><path d="M-0.41 1.1 C-0.5 9.38, -0.66 41.68, -0.6 49.79 M1.57 0.63 C1.88 9.05, 2.12 42.64, 1.71 51" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1339.6891897432415 228.46001580359916) rotate(0 0.6583335483969961 25.812424113767975)"><path d="M-4.7 25.9 C-5.14 33.93, -2.7 41.27, 1.92 51.49 M-5.85 27.1 C-4.26 33.55, -3.08 38.95, 1.26 51.78" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1339.6891897432415 228.46001580359916) rotate(0 0.6583335483969961 25.812424113767975)"><path d="M12.54 26.17 C7.03 34.06, 4.39 41.32, 1.92 51.49 M11.4 27.37 C8.63 33.84, 5.47 39.17, 1.26 51.78" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1218.390625029773 279.4939658833182) rotate(0 122.5 60)"><path d="M153.75 15.25 M153.75 15.25 C169.05 23.46, 185.14 31.89, 214.25 45.75 M153.75 15.25 C172.48 25.5, 192.04 34.39, 214.25 45.75 M214.25 45.75 C245.43 61.81, 243.64 60.59, 214.25 76.25 M214.25 45.75 C243.95 62.79, 246.32 62, 214.25 76.25 M214.25 76.25 C196.73 82.87, 183.39 91.15, 153.75 104.75 M214.25 76.25 C194.18 85.77, 173.38 96.74, 153.75 104.75 M153.75 104.75 C123.3 121.52, 122.09 120.89, 92.25 104.75 M153.75 104.75 C124.68 121.59, 122.63 119.93, 92.25 104.75 M92.25 104.75 C73 94.59, 49.02 83.99, 30.75 76.25 M92.25 104.75 C76.85 97.51, 59.21 90.47, 30.75 76.25 M30.75 76.25 C-1.19 61.32, -0.18 60.33, 30.75 45.75 M30.75 76.25 C-1.3 59.64, 1.14 61.74, 30.75 45.75 M30.75 45.75 C55.54 35.13, 77.27 23.51, 92.25 15.25 M30.75 45.75 C54.58 33.59, 76.01 23.93, 92.25 15.25 M92.25 15.25 C122.36 0.13, 124.74 -1.52, 153.75 15.25 M92.25 15.25 C123.46 1.67, 124.14 2.28, 153.75 15.25" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1285.4306869804566 326.9939658833182) rotate(0 55.709938049316406 12.5)"><text x="55.709938049316406" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">m_support?</text></g><g mask="url(#mask-7WfavuBOsJZAKEtDHZh9-)" stroke-linecap="round"><g transform="translate(1309.925252077095 387.21845170835877) rotate(0 -39.570858004917795 54.842982454373214)"><path d="M0.02 0.12 C-13.01 18.44, -65.24 90.92, -78.37 109.31 M-1.42 -0.86 C-14.57 17.67, -66.13 92.33, -79.17 110.55" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1309.925252077095 387.21845170835877) rotate(0 -39.570858004917795 54.842982454373214)"><path d="M-71.18 82.07 C-74.71 89.63, -74.98 99.51, -79.31 110.98 M-71.83 82.35 C-73.65 91.87, -77.2 101.09, -80.01 109.94" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1309.925252077095 387.21845170835877) rotate(0 -39.570858004917795 54.842982454373214)"><path d="M-54.38 93.85 C-62.65 98.05, -67.72 104.56, -79.31 110.98 M-55.03 94.14 C-62.5 99.6, -71.77 104.81, -80.01 109.94" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask id="mask-7WfavuBOsJZAKEtDHZh9-"><rect x="0" y="0" fill="#fff" width="1488.5943577474734" height="597.0998401502584"></rect><rect x="1245.5307245714957" y="429.6591459293086" fill="#000" width="50.11994934082031" height="25" opacity="1"></rect></mask><g transform="translate(1245.5307245714955 429.6591459293086) rotate(0 24.823669500681547 12.402288233423405)"><text x="25.059974670410156" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">false</text></g><g mask="url(#mask-7FSlzWEgw_RQAurWqiYCk)" stroke-linecap="round"><g transform="translate(1332.7565469271947 398.9505970234086) rotate(0 -0.7703164724059945 120.66137078634256)"><path d="M-0.14 0.6 C-0.16 40.69, 0.18 199.97, 0.13 239.96 M-1.67 -0.13 C-1.81 40.21, -0.49 201.69, -0.39 241.45" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1332.7565469271947 398.9505970234086) rotate(0 -0.7703164724059945 120.66137078634256)"><path d="M-9.8 214.69 C-7.7 222.9, -3.36 230.24, -0.8 240.54 M-11.12 213.11 C-7.55 223.3, -3.03 235.38, -0.33 241.86" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1332.7565469271947 398.9505970234086) rotate(0 -0.7703164724059945 120.66137078634256)"><path d="M10.72 214.58 C6.16 222.8, 3.84 230.18, -0.8 240.54 M9.4 213 C5.22 223.32, 1.98 235.44, -0.33 241.86" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask id="mask-7FSlzWEgw_RQAurWqiYCk"><rect x="0" y="0" fill="#fff" width="1433.4492353157402" height="738.9952097329497"></rect><rect x="1316.8202216538202" y="506.47290337817867" fill="#000" width="31.179962158203125" height="25" opacity="1"></rect></mask><g transform="translate(1316.8202216538202 506.47290337817867) rotate(0 15.166008800968399 13.139064431572478)"><text x="15.589981079101562" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">yes</text></g><g stroke-linecap="round" transform="translate(1143.6828540872134 492.14469477570583) rotate(0 54 58)"><path d="M36.79 3.89 C44.7 0.06, 55.63 -0.87, 64.31 0.66 C73 2.2, 82.23 6.79, 88.89 13.08 C95.56 19.37, 101.13 29.15, 104.31 38.41 C107.48 47.68, 109.41 59.1, 107.93 68.66 C106.46 78.22, 101.35 88.35, 95.45 95.78 C89.55 103.2, 81.18 109.95, 72.54 113.22 C63.89 116.49, 52.41 117.22, 43.59 115.39 C34.77 113.56, 26.25 108.65, 19.63 102.23 C13.01 95.81, 6.89 85.89, 3.84 76.87 C0.8 67.85, -0.28 57.58, 1.36 48.12 C2.99 38.67, 6.59 27.92, 13.65 20.14 C20.71 12.36, 37.39 4.47, 43.72 1.44 C50.05 -1.59, 51.35 1.15, 51.66 1.95 M34.62 3.97 C42.67 -0.12, 53.86 -0.5, 62.38 1.03 C70.89 2.55, 78.95 7.26, 85.72 13.11 C92.5 18.96, 99.43 27.46, 103.04 36.14 C106.65 44.81, 108.57 55.78, 107.39 65.17 C106.21 74.57, 101.28 84.68, 95.93 92.5 C90.59 100.33, 83.3 108.36, 75.32 112.11 C67.34 115.86, 56.93 116.21, 48.07 115.01 C39.22 113.8, 29.32 110.76, 22.17 104.89 C15.02 99.01, 8.6 88.66, 5.17 79.75 C1.74 70.84, 0.65 61.01, 1.58 51.41 C2.5 41.81, 5.2 30.24, 10.72 22.15 C16.23 14.06, 30.98 5.49, 34.67 2.88 C38.37 0.27, 32.83 5.33, 32.89 6.49" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1167.4291187258937 525.1325014668864) rotate(0 30.069969177246108 25)"><text x="30.069969177246094" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">return</text><text x="30.069969177246094" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">true</text></g><g stroke-linecap="round" transform="translate(1206.5773032663924 642.8232891267553) rotate(0 131.5 22.5)"><path d="M11.25 0 M11.25 0 C68.36 1.87, 123.6 0.39, 251.75 0 M11.25 0 C78.21 1.92, 145.65 1.56, 251.75 0 M251.75 0 C259.4 1.09, 264.07 2.69, 263 11.25 M251.75 0 C260.09 1.68, 263.35 5.5, 263 11.25 M263 11.25 C264.48 16.65, 263.04 23.49, 263 33.75 M263 11.25 C262.69 16.98, 262.22 23.88, 263 33.75 M263 33.75 C264.6 40.7, 260.89 45.69, 251.75 45 M263 33.75 C261.12 42.88, 257.89 45.36, 251.75 45 M251.75 45 C180.8 44.69, 112.36 45.22, 11.25 45 M251.75 45 C187.98 46.05, 123.6 46.85, 11.25 45 M11.25 45 C3.08 44.38, 1.66 39.26, 0 33.75 M11.25 45 C2.3 46.6, -0.74 41.4, 0 33.75 M0 33.75 C1.62 26.03, 2.15 15.53, 0 11.25 M0 33.75 C-0.59 26.16, -0.2 21.11, 0 11.25 M0 11.25 C-1.42 3.21, 4.65 1.13, 11.25 0 M0 11.25 C-1.41 2.18, 2.21 0.7, 11.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1283.5873588083846 652.8232891267553) rotate(0 54.48994445800781 12.5)"><text x="54.48994445800781" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">startPlugin</text></g><g stroke-linecap="round"><g transform="translate(1328.2659023502301 696.3303563857535) rotate(0 -0.6977216760748775 38.298936821813186)"><path d="M-0.38 -0.25 C-1 12.48, -2.7 63.73, -3.02 76.87 M1.62 -1.42 C1.31 11.39, -0.02 64.98, -0.84 78.02" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1328.2659023502301 696.3303563857535) rotate(0 -0.6977216760748775 38.298936821813186)"><path d="M-10.36 47.75 C-7.79 59.16, -1.62 70.02, -0.02 78.18 M-9.58 50.17 C-6.42 60.38, -3.79 70.2, -0.11 78.71" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1328.2659023502301 696.3303563857535) rotate(0 -0.6977216760748775 38.298936821813186)"><path d="M10.14 48.59 C5.38 59.63, 4.22 70.19, -0.02 78.18 M10.92 51 C6.82 60.81, 2.17 70.33, -0.11 78.71" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1195.1185884614952 778.7419902315023) rotate(0 125 24)"><path d="M12 0 M12 0 C71.63 1.18, 134.81 1.72, 238 0 M12 0 C89.52 -1.51, 166.44 -0.43, 238 0 M238 0 C246.3 1.52, 249.09 4.89, 250 12 M238 0 C247.68 1.59, 249.63 3.93, 250 12 M250 12 C251.54 19.18, 248.41 26.37, 250 36 M250 12 C251.01 18.59, 249.38 25.19, 250 36 M250 36 C248.81 44.32, 245.82 47.33, 238 48 M250 36 C248.7 42.64, 247.14 48.74, 238 48 M238 48 C152.91 50.69, 65.29 50.7, 12 48 M238 48 C154.57 46.55, 69.18 48, 12 48 M12 48 C3.36 48.13, 1.74 42.48, 0 36 M12 48 C4.46 49.67, 1.14 46.28, 0 36 M0 36 C1.7 28.5, -1.62 23.03, 0 12 M0 36 C0.62 27.13, -0.44 16.44, 0 12 M0 12 C-1.34 4.61, 4.02 -0.27, 12 0 M0 12 C-2.01 3.81, 3.46 -1.13, 12 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1266.978650107003 790.2419902315023) rotate(0 53.13993835449219 12.5)"><text x="53.13993835449219" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">sortPlugins</text></g><g stroke-linecap="round"><g transform="translate(1425.6176340271184 633.4783390841649) rotate(0 52.01495586337421 -201.0896557931942)"><path d="M0.07 0.49 C17.68 -66.49, 87.74 -333.78, 105.39 -400.89 M-1.36 -0.3 C16.14 -67.11, 86.68 -335.85, 104.49 -402.67" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1425.6176340271184 633.4783390841649) rotate(0 52.01495586337421 -201.0896557931942)"><path d="M108.03 -372.64 C107.3 -379.9, 104.41 -388.41, 105.95 -402.37 M107.94 -372.1 C106.16 -384.04, 105.69 -395.35, 105.23 -401.89" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1425.6176340271184 633.4783390841649) rotate(0 52.01495586337421 -201.0896557931942)"><path d="M88.19 -377.88 C92.94 -383.72, 95.47 -390.8, 105.95 -402.37 M88.1 -377.34 C93.75 -387.4, 100.74 -396.74, 105.23 -401.89" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1550.9443081601885 186.65457953529494) rotate(0 146 26.5)"><path d="M13.25 0 M13.25 0 C95.22 0.68, 174.62 0.42, 278.75 0 M13.25 0 C111.89 -1.24, 211.83 -1.05, 278.75 0 M278.75 0 C287.26 -0.06, 292.78 4.21, 292 13.25 M278.75 0 C289.29 1.81, 290.9 2.62, 292 13.25 M292 13.25 C291.35 20.33, 292.58 22.48, 292 39.75 M292 13.25 C291.75 22.74, 291.28 32.32, 292 39.75 M292 39.75 C292.99 49.23, 289.21 52.35, 278.75 53 M292 39.75 C293.71 50.01, 289.27 52.46, 278.75 53 M278.75 53 C179.1 53.62, 78.85 51.3, 13.25 53 M278.75 53 C181.28 52.54, 82.59 53.71, 13.25 53 M13.25 53 C5.41 54.98, -0.79 48.61, 0 39.75 M13.25 53 C4.03 54.96, -1.43 46.73, 0 39.75 M0 39.75 C-0.79 33.32, 0.89 23.88, 0 13.25 M0 39.75 C0.19 33.48, -0.7 27.39, 0 13.25 M0 13.25 C-0.47 3.43, 4.1 0.79, 13.25 0 M0 13.25 C-1.05 5.78, 6.17 -0.13, 13.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1625.8543881162432 200.65457953529494) rotate(0 71.08992004394531 12.5)"><text x="71.08992004394531" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">discoverPlugins</text></g><g stroke-linecap="round"><g transform="translate(1689.3658289035263 243.46203756092837) rotate(0 0.2950533270954452 35.37944387934431)"><path d="M0.44 0.87 C0.56 12.73, 1.35 58.93, 1.38 70.47 M-0.79 0.29 C-0.86 11.87, 0.46 57.54, 0.53 68.83" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1689.3658289035263 243.46203756092837) rotate(0 0.2950533270954452 35.37944387934431)"><path d="M-8.8 41.13 C-5.07 49.78, -1.89 62.05, 1.91 68.5 M-9.51 41.62 C-7.42 48.37, -4.69 57.07, 1.33 68.55" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1689.3658289035263 243.46203756092837) rotate(0 0.2950533270954452 35.37944387934431)"><path d="M11.72 40.75 C8.19 49.61, 4.12 62.01, 1.91 68.5 M11.01 41.23 C7 48.08, 3.64 56.89, 1.33 68.55" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1537.5480187829976 312.2688975372166) rotate(0 156.5 85)"><path d="M196.25 21.5 M196.25 21.5 C224.17 34.95, 249.64 51.8, 273.75 64.5 M196.25 21.5 C225.18 37.29, 254.14 54.06, 273.75 64.5 M273.75 64.5 C312.04 84.44, 311.1 85.2, 273.75 107.5 M273.75 64.5 C314.84 85.37, 314.89 86.79, 273.75 107.5 M273.75 107.5 C257.44 115.81, 239.53 123.81, 196.25 148.5 M273.75 107.5 C256.1 117.3, 237.36 128.12, 196.25 148.5 M196.25 148.5 C158.46 169.53, 158.47 171.45, 117.75 148.5 M196.25 148.5 C156.23 169.29, 158.91 167.71, 117.75 148.5 M117.75 148.5 C99.01 140.28, 78.76 129.23, 39.25 107.5 M117.75 148.5 C87.91 134.02, 58.9 119.59, 39.25 107.5 M39.25 107.5 C-1.25 84.38, -0.47 87.86, 39.25 64.5 M39.25 107.5 C-1.64 85.38, 1.03 87.3, 39.25 64.5 M39.25 64.5 C58.9 54.77, 76.78 42.73, 117.75 21.5 M39.25 64.5 C62.1 52.73, 84.7 40.38, 117.75 21.5 M117.75 21.5 C158.52 -0.12, 158.05 -0.64, 196.25 21.5 M117.75 21.5 C158.04 -0.03, 157.39 -1.01, 196.25 21.5" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1623.4780877527242 372.2688975372166) rotate(0 70.81993103027344 25)"><text x="70.81993103027344" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">for pair in </text><text x="70.81993103027344" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">tmp_storeMap</text></g><g stroke-linecap="round"><g transform="translate(1696.2854099449696 476.3684316484678) rotate(0 -0.18362465050449828 44.33537683633085)"><path d="M0.88 0.83 C0.94 15.23, -0.13 72.6, -0.2 86.93 M-0.12 0.22 C-0.19 14.77, -1.26 73.8, -1.25 88.45" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1696.2854099449696 476.3684316484678) rotate(0 -0.18362465050449828 44.33537683633085)"><path d="M-9.85 59.83 C-8.62 68.07, -6.23 75.98, 0.32 87.49 M-10.43 59.89 C-8.46 66.82, -7.89 73.46, -1.84 88.61" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1696.2854099449696 476.3684316484678) rotate(0 -0.18362465050449828 44.33537683633085)"><path d="M10.68 60.04 C6.67 68.37, 3.84 76.23, 0.32 87.49 M10.09 60.09 C7.51 66.89, 3.53 73.49, -1.84 88.61" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1557.0988234654205 570.9428996897641) rotate(0 135.5 32.5)"><path d="M16.25 0 M16.25 0 C105.18 0.98, 194.03 1.57, 254.75 0 M16.25 0 C105.72 1.22, 197.46 1.33, 254.75 0 M254.75 0 C267.23 0.69, 269.37 6.83, 271 16.25 M254.75 0 C264.22 0.36, 270.79 4.65, 271 16.25 M271 16.25 C271.46 25.79, 270.16 33.56, 271 48.75 M271 16.25 C271.97 28.75, 270.9 39.49, 271 48.75 M271 48.75 C272.66 57.59, 264.32 66.39, 254.75 65 M271 48.75 C270.26 59.74, 267.59 63.25, 254.75 65 M254.75 65 C180.07 65.32, 102.11 65.71, 16.25 65 M254.75 65 C162.7 64.17, 70.45 63.16, 16.25 65 M16.25 65 C6.32 66.13, -1.22 58.22, 0 48.75 M16.25 65 C3.87 65.7, 0.02 59.28, 0 48.75 M0 48.75 C-0.96 41.53, 0.82 34.03, 0 16.25 M0 48.75 C0.28 36.76, 0.21 25.4, 0 16.25 M0 16.25 C0.34 4.54, 6.94 -1.65, 16.25 0 M0 16.25 C0.48 3.66, 7.3 -1.46, 16.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1605.448898233487 590.9428996897641) rotate(0 87.1499252319336 12.5)"><text x="87.1499252319336" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">newLaunchedPlugin</text></g><g stroke-linecap="round"><g transform="translate(1696.0645322015266 639.4276200776249) rotate(0 1.5871809078216756 45.32478699906687)"><path d="M0.89 0.94 C1.36 15.83, 2.97 74.43, 3.27 89.21 M-0.1 0.39 C0.12 15.41, 1.17 75.2, 2.02 90.26" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1696.0645322015266 639.4276200776249) rotate(0 1.5871809078216756 45.32478699906687)"><path d="M-7.62 61.48 C-6.98 69.88, -2.42 82.81, 1.48 91.9 M-9.79 62.59 C-5.76 71.32, -2.49 80.37, 2.52 90.58" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1696.0645322015266 639.4276200776249) rotate(0 1.5871809078216756 45.32478699906687)"><path d="M12.89 60.78 C6.57 69.57, 4.19 82.74, 1.48 91.9 M10.72 61.89 C7.89 71, 4.3 80.28, 2.52 90.58" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask></mask><g stroke-linecap="round"><g transform="translate(1833.7806681421462 635.3843978120531) rotate(0 52.52115313701643 9.627725107538481)"><path d="M-0.57 -0.94 C16.53 2.06, 85.81 15.57, 103.5 18.91 M1.32 1.18 C18.7 4.26, 88.28 17.02, 105.62 20.19" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1833.7806681421462 635.3843978120531) rotate(0 52.52115313701643 9.627725107538481)"><path d="M74.48 23.31 C86.27 25.07, 97.6 23.24, 106.3 18.56 M75.95 24.88 C86.47 22.71, 98.5 21.77, 106.43 19.87" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1833.7806681421462 635.3843978120531) rotate(0 52.52115313701643 9.627725107538481)"><path d="M78.17 3.12 C88.74 12.43, 98.69 18.15, 106.3 18.56 M79.64 4.69 C88.68 10.02, 99.34 16.56, 106.43 19.87" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1965.301107575477 536.3616552811945) rotate(0 104 26.5)"><path d="M13.25 0 M13.25 0 C58.21 3.32, 104.22 3.37, 194.75 0 M13.25 0 C73.34 -1.13, 134.53 -1.62, 194.75 0 M194.75 0 C202.45 -1.18, 208.99 5.06, 208 13.25 M194.75 0 C205.45 -0.74, 209.71 5.85, 208 13.25 M208 13.25 C209.38 22.47, 207.32 34.69, 208 39.75 M208 13.25 C207.51 18.63, 208.7 23.11, 208 39.75 M208 39.75 C208.4 50.03, 204.57 54.98, 194.75 53 M208 39.75 C207.09 48.62, 203.2 54.96, 194.75 53 M194.75 53 C153.47 51.15, 107.92 53.47, 13.25 53 M194.75 53 C129.36 52.06, 65.31 51.63, 13.25 53 M13.25 53 C2.67 52.84, -0.47 47.6, 0 39.75 M13.25 53 C4.05 53.91, -1.05 49.94, 0 39.75 M0 39.75 C-0.63 30.69, -0.01 20.15, 0 13.25 M0 39.75 C0.01 32.2, 0.62 25.55, 0 13.25 M0 13.25 C1.33 4.28, 2.47 -0.71, 13.25 0 M0 13.25 C-0.38 2.33, 2.32 0.73, 13.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1982.901189972938 550.3616552811945) rotate(0 86.39991760253906 12.5)"><text x="86.39991760253906" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">CreateSocketPair</text></g><g stroke-linecap="round"><g transform="translate(2064.228269031724 585.4587092194788) rotate(0 1.012532919851651 25.11980622121746)"><path d="M0.99 0.41 C0.81 8.91, 0.05 42.39, -0.09 50.66 M0.05 -0.42 C0.15 7.78, 1.67 40.34, 2.11 48.98" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2064.228269031724 585.4587092194788) rotate(0 1.012532919851651 25.11980622121746)"><path d="M-6.83 24.38 C-2.51 33.48, 0.05 43.21, 1.44 47.85 M-6.71 25.69 C-4.34 32.13, -2.03 36.81, 2.85 49.7" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2064.228269031724 585.4587092194788) rotate(0 1.012532919851651 25.11980622121746)"><path d="M10.18 23.56 C8.04 33.11, 4.14 43.15, 1.44 47.85 M10.31 24.87 C8.58 31.57, 6.79 36.44, 2.85 49.7" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1967.376404486732 639.2013710293404) rotate(0 104 29)"><path d="M14.5 0 M14.5 0 C84.33 0.28, 151.65 1.28, 193.5 0 M14.5 0 C62.73 1.38, 110.77 0.74, 193.5 0 M193.5 0 C204.65 1.24, 209.46 4.36, 208 14.5 M193.5 0 C204.86 1.66, 207.23 4.12, 208 14.5 M208 14.5 C209.11 24.97, 207.84 38.47, 208 43.5 M208 14.5 C208.55 21.98, 207.66 27.5, 208 43.5 M208 43.5 C207.67 54.87, 201.92 56.38, 193.5 58 M208 43.5 C207.46 55.31, 201.53 57.38, 193.5 58 M193.5 58 C130.39 55.56, 70.62 56.92, 14.5 58 M193.5 58 C141.94 57.36, 90.7 58.25, 14.5 58 M14.5 58 C3.92 59.18, 1.52 53.05, 0 43.5 M14.5 58 C6.05 57.26, 1.04 53.13, 0 43.5 M0 43.5 C-1.53 34.73, -1.4 26.75, 0 14.5 M0 43.5 C-1.16 36.29, -0.36 28.88, 0 14.5 M0 14.5 C-1.83 5.47, 5.44 0.75, 14.5 0 M0 14.5 C-0.08 6.9, 6.98 -0.79, 14.5 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1978.9264914618298 655.7013710293404) rotate(0 92.44991302490234 12.5)"><text x="92.44991302490234" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">fork to exec plugin</text></g><g stroke-linecap="round"><g transform="translate(2071.631249687138 698.3712459193457) rotate(0 0.43689848492533656 29.382497450808387)"><path d="M-0.11 -0.4 C0.03 9.44, -0.11 49.13, 0.14 59.26 M-1.63 -1.66 C-1.03 8.31, 2.07 50.44, 2.5 60.42" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2071.631249687138 698.3712459193457) rotate(0 0.43689848492533656 29.382497450808387)"><path d="M-10.11 31.79 C-7.71 40.54, -3 47.79, 1.85 61.91 M-8.71 33.64 C-5.25 42.6, 0.24 51.88, 1.87 61.12" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2071.631249687138 698.3712459193457) rotate(0 0.43689848492533656 29.382497450808387)"><path d="M10.36 30.54 C7.84 39.58, 7.61 47.13, 1.85 61.91 M11.76 32.4 C7.81 41.76, 5.89 51.48, 1.87 61.12" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1973.3738757510996 759.3371465831096) rotate(0 101 29)"><path d="M14.5 0 M14.5 0 C63.64 1.97, 115.8 2.6, 187.5 0 M14.5 0 C61.54 -1.31, 107.86 -0.35, 187.5 0 M187.5 0 C196.5 -0.62, 203.66 2.84, 202 14.5 M187.5 0 C195.71 1.6, 201.26 4.99, 202 14.5 M202 14.5 C203.23 26.72, 203.76 36.17, 202 43.5 M202 14.5 C201.33 22.44, 201.72 32.92, 202 43.5 M202 43.5 C200.58 52.62, 198.07 59.13, 187.5 58 M202 43.5 C200.59 51.6, 195.62 58.7, 187.5 58 M187.5 58 C135.27 56.01, 82.39 56.17, 14.5 58 M187.5 58 C129.54 58.96, 70.59 58.7, 14.5 58 M14.5 58 C5.74 57.97, 0.34 52.29, 0 43.5 M14.5 58 C6.59 56.1, 0.48 51.41, 0 43.5 M0 43.5 C0.05 30.49, -0.52 21.03, 0 14.5 M0 43.5 C0.57 37.81, 0.23 32.26, 0 14.5 M0 14.5 C1.87 4.15, 4.66 0.25, 14.5 0 M0 14.5 C1.99 2.62, 5.91 -0.09, 14.5 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1985.6939822574473 775.8371465831096) rotate(0 88.67989349365234 12.5)"><text x="88.67989349365234" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">plugin->SetPid(pid);</text></g><g stroke-linecap="round" transform="translate(1946.1701080204493 513.517041542279) rotate(0 127.28138227314548 161.40762111706317)"><path d="M32 0 M32 0 C97.97 -0.06, 166.48 -0.08, 222.56 0 M222.56 0 C243.43 1.47, 256.01 10, 254.56 32 M254.56 32 C255.28 102.79, 255.77 171.44, 254.56 290.82 M254.56 290.82 C256.31 310.63, 244.3 324.27, 222.56 322.82 M222.56 322.82 C156 321.73, 91.07 320.81, 32 322.82 M32 322.82 C9.05 322.34, 1.86 310.73, 0 290.82 M0 290.82 C-1.91 219.1, -2.01 150.06, 0 32 M0 32 C0.02 10.4, 8.92 -0.16, 32 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"></path></g><g stroke-linecap="round" transform="translate(1575.6400776245337 742.9495810241624) rotate(0 121.5 30)"><path d="M15 0 M15 0 C97.29 -0.11, 176.98 -0.1, 228 0 M15 0 C95.03 -0.3, 173.04 1.19, 228 0 M228 0 C237.36 0.13, 244.74 3.48, 243 15 M228 0 C238.46 1.67, 244.14 7.28, 243 15 M243 15 C244.71 21.57, 241.39 30.15, 243 45 M243 15 C243.24 27.36, 242.18 37.91, 243 45 M243 45 C241.66 55.61, 238.02 59.73, 228 60 M243 45 C240.99 54.81, 237.46 58.87, 228 60 M228 60 C168.52 60.49, 106.87 60.07, 15 60 M228 60 C171.09 59.59, 114.55 59.25, 15 60 M15 60 C5.42 58.47, 1.63 53.73, 0 45 M15 60 C6.53 59.85, -2.23 54.19, 0 45 M0 45 C0.91 37.1, 1.02 27.92, 0 15 M0 45 C-0.37 33.43, 0.59 20.66, 0 15 M0 15 C0.94 4.92, 4.06 1.68, 15 0 M0 15 C1.03 4.03, 6.45 -0.94, 15 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1634.7201404907446 760.4495810241624) rotate(0 62.41993713378906 12.5)"><text x="62.41993713378906" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">plugin->Start</text></g><g stroke-linecap="round" transform="translate(1053.9866462145635 117.93778384067173) rotate(0 604.4564308538065 479.83843844674675)"><path d="M32 0 M32 0 C459.87 -3.63, 886.33 -2.7, 1176.91 0 M32 0 C261.43 -3.17, 491.46 -3.5, 1176.91 0 M1176.91 0 C1198.65 1.45, 1209.9 12.65, 1208.91 32 M1176.91 0 C1197.34 0.04, 1208.53 12.62, 1208.91 32 M1208.91 32 C1206.76 244.34, 1205.8 458.17, 1208.91 927.68 M1208.91 32 C1206.45 351.52, 1206.73 670.74, 1208.91 927.68 M1208.91 927.68 C1207.16 948.85, 1197.77 958.69, 1176.91 959.68 M1208.91 927.68 C1208.55 949.92, 1197.2 961.04, 1176.91 959.68 M1176.91 959.68 C745.96 962.45, 315.51 962.23, 32 959.68 M1176.91 959.68 C883.96 955.58, 591.16 955.63, 32 959.68 M32 959.68 C11.99 959.54, -1.94 948.3, 0 927.68 M32 959.68 C10.29 957.59, -2.1 949.74, 0 927.68 M0 927.68 C-0.62 632.52, -1.61 335.8, 0 32 M0 927.68 C-2.92 647.69, -3.29 368.06, 0 32 M0 32 C0.89 9.82, 11.93 -0.82, 32 0 M0 32 C-0.12 9.22, 11.02 0.12, 32 0" stroke="#1e1e1e" stroke-width="4" fill="none"></path></g><g transform="translate(1079.3840524672423 134.27591534706335) rotate(0 96.75393676757812 17.5)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="28px" fill="#e03131" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">nri_adaption.h</text></g><g stroke-linecap="round"><g transform="translate(1840.29695052346 207.3103559318688) rotate(0 35.801098535335655 -0.17867812737813438)"><path d="M1 -1.19 C12.6 -0.99, 57.79 0.62, 69.22 0.84 M0.06 0.79 C12.09 0.69, 60.11 -0.95, 71.54 -0.85" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1840.29695052346 207.3103559318688) rotate(0 35.801098535335655 -0.17867812737813438)"><path d="M41.52 8.59 C52.41 6.13, 58.62 5.66, 70.02 -0.45 M43.12 9.87 C53.73 6.44, 63.72 0.8, 71.3 0.09" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1840.29695052346 207.3103559318688) rotate(0 35.801098535335655 -0.17867812737813438)"><path d="M41.2 -11.92 C52.11 -8.92, 58.41 -3.92, 70.02 -0.45 M42.79 -10.64 C53.63 -6.43, 63.73 -4.44, 71.3 0.09" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1954.156486068844 140.38544819111303) rotate(0 110.5 17.5)"><path d="M8.75 0 M8.75 0 C87.98 0.58, 164.83 2.71, 212.25 0 M8.75 0 C54.74 -0.43, 99.49 -1.31, 212.25 0 M212.25 0 C217.75 1.7, 219.75 1.3, 221 8.75 M212.25 0 C217.54 2.14, 219.36 2.29, 221 8.75 M221 8.75 C219.9 13.61, 221.63 20.84, 221 26.25 M221 8.75 C220.64 13.34, 220.71 19.13, 221 26.25 M221 26.25 C220.09 33.26, 219.61 34.88, 212.25 35 M221 26.25 C222.21 31.34, 219.13 34.97, 212.25 35 M212.25 35 C146.27 36.76, 81.88 37.85, 8.75 35 M212.25 35 C163.25 34.56, 115.84 34.07, 8.75 35 M8.75 35 C1.09 35.63, 0.61 32.83, 0 26.25 M8.75 35 C2.83 37.06, 2.15 31.29, 0 26.25 M0 26.25 C-1.7 21.97, -0.09 15.24, 0 8.75 M0 26.25 C0.47 19.17, -0.13 12.26, 0 8.75 M0 8.75 C0.31 3.02, 3.6 1.92, 8.75 0 M0 8.75 C0.85 1.05, 3.61 -1.92, 8.75 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1976.396575485348 145.38544819111303) rotate(0 88.2599105834961 12.5)"><text x="88.2599105834961" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">util_scan_subdirs</text></g><g stroke-linecap="round"><g transform="translate(2061.629698853203 176.38544819111303) rotate(0 1.327661016229058 11.378398495884994)"><path d="M1.05 -0.91 C1.53 3.02, 2.21 19.52, 2.52 23.67 M0.14 1.22 C0.52 4.82, 1.32 18.5, 1.7 22.01" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2061.629698853203 176.38544819111303) rotate(0 1.327661016229058 11.378398495884994)"><path d="M-3.95 11.82 C-0.96 14.9, 0.62 16.67, 1.71 21.81 M-3.21 12.12 C-1.71 15.48, 0.79 19.97, 1.35 21.61" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2061.629698853203 176.38544819111303) rotate(0 1.327661016229058 11.378398495884994)"><path d="M3.86 11.19 C4.76 14.51, 4.25 16.45, 1.71 21.81 M4.6 11.49 C3.09 15.03, 2.58 19.77, 1.35 21.61" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1957.6395932905596 200.18647806842273) rotate(0 110.5 17.5)"><path d="M8.75 0 M8.75 0 C88.85 0.52, 168.13 -1.07, 212.25 0 M8.75 0 C69.55 -2.27, 131.26 -0.72, 212.25 0 M212.25 0 C216.66 -0.54, 221.9 4.05, 221 8.75 M212.25 0 C216.68 -1.57, 219.46 3.62, 221 8.75 M221 8.75 C220.83 13.6, 220.12 19, 221 26.25 M221 8.75 C221.59 14.64, 221.38 20.35, 221 26.25 M221 26.25 C221.91 32.06, 218.42 34.12, 212.25 35 M221 26.25 C222.75 30.18, 218.56 33.24, 212.25 35 M212.25 35 C134.45 34.47, 56.25 35.91, 8.75 35 M212.25 35 C169.93 35.91, 127.04 36.38, 8.75 35 M8.75 35 C4.78 34.31, -0.18 32.34, 0 26.25 M8.75 35 C4.91 32.79, 1.08 31.99, 0 26.25 M0 26.25 C-0.89 22.92, -0.86 17.3, 0 8.75 M0 26.25 C0.27 22.94, 0.3 17.24, 0 8.75 M0 8.75 C0.6 1.25, 2.58 0.4, 8.75 0 M0 8.75 C-0.56 3.03, 1.46 1.14, 8.75 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1975.7096769087236 205.18647806842273) rotate(0 92.42991638183594 12.5)"><text x="92.42991638183594" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">walk_plugin_dir_cb</text></g><g stroke-linecap="round"><g transform="translate(2063.6974983108485 238.0683248469013) rotate(0 -0.19771349608413402 11.766539813040708)"><path d="M-0.48 0.02 C-0.71 3.91, -0.67 20.79, -0.82 24.55 M1.48 -1.02 C1.01 3.07, -1.82 18.9, -1.87 22.96" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2063.6974983108485 238.0683248469013) rotate(0 -0.19771349608413402 11.766539813040708)"><path d="M-4.68 11.34 C-2.6 14.81, -3.56 19.26, -0.77 22.13 M-5.06 11.14 C-4.27 14.82, -3.1 17.68, -2.39 22.92" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2063.6974983108485 238.0683248469013) rotate(0 -0.19771349608413402 11.766539813040708)"><path d="M3.32 12.2 C2.6 15.33, -1.16 19.48, -0.77 22.13 M2.94 11.99 C1.54 15.49, 0.53 18.11, -2.39 22.92" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1965.1609446185 264.71333088305073) rotate(0 107.5 41)"><path d="M20.5 0 M20.5 0 C67.51 -0.95, 109.51 -0.18, 194.5 0 M20.5 0 C89.37 1.04, 156.72 -0.21, 194.5 0 M194.5 0 C206.82 0.61, 215.02 6.57, 215 20.5 M194.5 0 C206.15 -0.19, 214.46 5.7, 215 20.5 M215 20.5 C216.34 33.67, 215.04 44.85, 215 61.5 M215 20.5 C215.36 31.04, 215.95 41.64, 215 61.5 M215 61.5 C215.42 73.64, 209.8 80.73, 194.5 82 M215 61.5 C216.53 75.01, 205.93 81.19, 194.5 82 M194.5 82 C145.81 81.02, 96.59 80.34, 20.5 82 M194.5 82 C126.6 81.53, 59.75 80.44, 20.5 82 M20.5 82 C7.77 81.92, -0.94 76.85, 0 61.5 M20.5 82 C7.86 81.03, 1.45 74.23, 0 61.5 M0 61.5 C0.36 50.11, 2.17 38.08, 0 20.5 M0 61.5 C-0.29 52.72, -0.37 43.6, 0 20.5 M0 20.5 C-1.26 7.82, 6.69 1.76, 20.5 0 M0 20.5 C0.21 6.14, 5.3 0.33, 20.5 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1974.4710489886172 280.71333088305073) rotate(0 98.18989562988281 25)"><text x="98.18989562988281" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">Verify plugin </text><text x="98.18989562988281" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">permissions for exec</text></g><g stroke-linecap="round"><g transform="translate(2065.190811563474 351.97966378001) rotate(0 0.5311339763076148 2.4318372477032426)"><path d="M-0.28 1.12 C-0.3 1.75, -0.99 2.38, -0.85 2.84 M1.77 0.66 C2.19 1.56, 1.7 4, 1.41 4.2" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2065.190811563474 351.97966378001) rotate(0 0.5311339763076148 2.4318372477032426)"><path d="M1.63 2.51 C1.41 3.07, 1.52 3.28, 1.3 4.1 M1.41 2.62 C1.44 3.11, 1.42 3.7, 1.37 4.25" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2065.190811563474 351.97966378001) rotate(0 0.5311339763076148 2.4318372477032426)"><path d="M2.63 2.93 C2.17 3.38, 2.05 3.49, 1.3 4.1 M2.41 3.03 C2.11 3.38, 1.76 3.84, 1.37 4.25" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1966.5540195264566 355.053366307573) rotate(0 106.5 22.5)"><path d="M11.25 0 M11.25 0 C63.37 2.25, 117.7 0.13, 201.75 0 M11.25 0 C55.39 -0.71, 98.64 -0.66, 201.75 0 M201.75 0 C208.78 -0.98, 212.68 4.54, 213 11.25 M201.75 0 C208.2 1.36, 214.75 3.62, 213 11.25 M213 11.25 C212.9 19.53, 212.05 28.65, 213 33.75 M213 11.25 C213.86 15.5, 213.71 20.96, 213 33.75 M213 33.75 C211.06 40.54, 208.92 43.19, 201.75 45 M213 33.75 C210.9 41.98, 209.95 45.86, 201.75 45 M201.75 45 C144.62 46.6, 89.11 48.51, 11.25 45 M201.75 45 C162.42 45.76, 124.48 44.5, 11.25 45 M11.25 45 C5.01 44.18, -0.11 39.99, 0 33.75 M11.25 45 C4.1 45.12, 0.79 43.45, 0 33.75 M0 33.75 C-1.48 25.83, 0.58 18.09, 0 11.25 M0 33.75 C0.07 27.77, 0.23 19.72, 0 11.25 M0 11.25 C-1.34 4.03, 2.54 -0.67, 11.25 0 M0 11.25 C-0.59 2.71, 5.96 1.05, 11.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1986.3240924634683 365.053366307573) rotate(0 86.72992706298828 12.5)"><text x="86.72992706298828" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">Parse plugin name</text></g><g stroke-linecap="round"><g transform="translate(2068.336010758302 398.9811681924384) rotate(0 0.9929506080603687 12.65370072052201)"><path d="M-0.73 -0.82 C-0.75 3.42, 0.1 21.72, 0.43 26.13 M1.08 1.36 C1.4 5.17, 2.81 19.95, 2.71 24.2" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2068.336010758302 398.9811681924384) rotate(0 0.9929506080603687 12.65370072052201)"><path d="M-3.07 11.4 C-0.04 16.43, 1.09 19.4, 2.61 23.89 M-2.48 12.65 C-0.49 15.2, 0.59 18.03, 3.01 24.19" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2068.336010758302 398.9811681924384) rotate(0 0.9929506080603687 12.65370072052201)"><path d="M5.75 11.04 C5.74 16.14, 3.83 19.23, 2.61 23.89 M6.33 12.29 C6.12 14.97, 4.98 17.88, 3.01 24.19" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1973.699518686874 429.9865646911694) rotate(0 102.5 20.5)"><path d="M10.25 0 M10.25 0 C54.93 1.26, 98.13 -0.51, 194.75 0 M10.25 0 C63.92 1.3, 117.46 1.5, 194.75 0 M194.75 0 C203.11 -0.12, 206.05 2.77, 205 10.25 M194.75 0 C202.63 -0.03, 205.39 2.4, 205 10.25 M205 10.25 C203.3 19.6, 203.56 27, 205 30.75 M205 10.25 C204.64 15.36, 203.89 22.6, 205 30.75 M205 30.75 C205.61 38.33, 201.51 42.79, 194.75 41 M205 30.75 C207.15 36.79, 201.38 41.29, 194.75 41 M194.75 41 C123.33 40.92, 53.75 42.76, 10.25 41 M194.75 41 C147.13 39.62, 99.78 40.3, 10.25 41 M10.25 41 C4.1 42.92, 0.74 35.96, 0 30.75 M10.25 41 C4.11 39.08, -0.39 38.04, 0 30.75 M0 30.75 C0.98 24.96, 1.75 19.64, 0 10.25 M0 30.75 C-0.47 24.88, -0.12 19.23, 0 10.25 M0 10.25 C1.92 4.33, 3.87 0.03, 10.25 0 M0 10.25 C1.69 4.25, 1.43 -0.69, 10.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2031.3895669046474 437.9865646911694) rotate(0 44.80995178222656 12.5)"><text x="44.80995178222656" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">init plugin</text></g><g stroke-linecap="round" transform="translate(1936.841194201655 132.52731516328004) rotate(0 131.27185259755356 177.91174412612145)"><path d="M32 0 M32 0 C91.58 1.26, 150.52 1.42, 230.54 0 M230.54 0 C252.67 -0.91, 263.72 12.19, 262.54 32 M262.54 32 C264.37 117.82, 263.95 203.02, 262.54 323.82 M262.54 323.82 C264.07 343.5, 252.29 354.3, 230.54 355.82 M230.54 355.82 C154.62 355.2, 78.25 356.81, 32 355.82 M32 355.82 C8.85 354, 0.63 345.76, 0 323.82 M0 323.82 C1.47 224.85, -0.05 127.09, 0 32 M0 32 C1.73 8.75, 11.6 -0.08, 32 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"></path></g><g stroke-linecap="round" transform="translate(1526.366699932505 167.27708930521658) rotate(0 164.63797607565243 420.39281295420096)"><path d="M32 0 M32 0 C87.59 0.25, 145.36 -0.22, 297.28 0 M297.28 0 C319.79 1.52, 329.16 11.72, 329.28 32 M329.28 32 C330.82 239.74, 331.29 447.53, 329.28 808.79 M329.28 808.79 C329.69 828.59, 320.24 839.52, 297.28 840.79 M297.28 840.79 C199.52 838.49, 101.46 837.3, 32 840.79 M32 840.79 C11.3 841.39, 0.75 830.05, 0 808.79 M0 808.79 C-2.97 506.17, -2.21 202.59, 0 32 M0 32 C0.94 10.59, 9.73 1.68, 32 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"></path></g><g stroke-linecap="round" transform="translate(2375.512207168123 180.04629825078246) rotate(0 108 32)"><path d="M16 0 M16 0 C67.63 -0.62, 118.86 1.82, 200 0 M16 0 C70.58 0.58, 124.42 0.15, 200 0 M200 0 C212.19 -1.65, 216.42 3.81, 216 16 M200 0 C212.55 -1.46, 217.53 5.18, 216 16 M216 16 C214.07 20.66, 216.52 29.59, 216 48 M216 16 C216.92 26.45, 215.9 37.72, 216 48 M216 48 C217.73 56.75, 211.6 63.92, 200 64 M216 48 C214.92 60.6, 211.7 63.03, 200 64 M200 64 C131.95 64.21, 66.51 64.59, 16 64 M200 64 C127.48 61.68, 54.18 62.71, 16 64 M16 64 C4.84 64.1, -1.26 59.65, 0 48 M16 64 C5.17 66.02, 0.21 57.97, 0 48 M0 48 C-0.72 40.02, -0.95 35, 0 16 M0 48 C0.66 37.3, -0.94 25.58, 0 16 M0 16 C1.93 5.1, 5.52 -0.08, 16 0 M0 16 C0.37 4.75, 5.36 -1.43, 16 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2430.5222550807207 199.54629825078246) rotate(0 52.989952087402344 12.5)"><text x="52.989952087402344" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">NRIPlugin()</text></g><g stroke-linecap="round"><g transform="translate(2181.778519054026 441.23512397615013) rotate(0 92.7467369867993 -108.700449113162)"><path d="M0.91 -0.07 C31.93 -36.41, 154.8 -181.18, 185.56 -217.33 M-0.07 -1.15 C30.9 -37.38, 154.1 -180.25, 184.94 -216" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2181.778519054026 441.23512397615013) rotate(0 92.7467369867993 -108.700449113162)"><path d="M174.19 -186.89 C176.2 -194.26, 179.52 -201.96, 184.06 -214.47 M175.12 -188.58 C177.22 -194.02, 177.9 -200.41, 184.78 -216.9" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2181.778519054026 441.23512397615013) rotate(0 92.7467369867993 -108.700449113162)"><path d="M158.64 -200.29 C164.69 -204.37, 171.97 -208.66, 184.06 -214.47 M159.58 -201.98 C165.08 -204.4, 169.14 -207.87, 184.78 -216.9" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(2336.391890521107 900.3209595988292) rotate(0 157.5 27.5)"><path d="M13.75 0 M13.75 0 C121.42 -0.46, 230.03 -0.89, 301.25 0 M13.75 0 C87.49 1.22, 161.54 1.32, 301.25 0 M301.25 0 C311.74 -0.13, 313.06 3.88, 315 13.75 M301.25 0 C310.04 -2.09, 312.9 5.31, 315 13.75 M315 13.75 C316.9 24.7, 314.42 31.74, 315 41.25 M315 13.75 C315.71 22.31, 314.77 31.78, 315 41.25 M315 41.25 C315.89 49.57, 311.68 54.18, 301.25 55 M315 41.25 C314.88 48.97, 310.77 55.12, 301.25 55 M301.25 55 C203.89 56.26, 107.72 55.55, 13.75 55 M301.25 55 C208.77 55.48, 117.18 55.77, 13.75 55 M13.75 55 C4.77 54.4, -1.34 50.7, 0 41.25 M13.75 55 C3.2 54.23, -0.59 49.37, 0 41.25 M0 41.25 C-0.09 31.83, 0.6 17.74, 0 13.75 M0 41.25 C-0.17 33.78, -0.1 26.14, 0 13.75 M0 13.75 C0.03 3.34, 2.85 1.78, 13.75 0 M0 13.75 C-2.11 4.03, 6.77 1.1, 13.75 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2370.47200679308 915.3209595988292) rotate(0 123.41988372802734 12.5)"><text x="123.41988372802734" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">Start(uint32_t timeout)</text></g><g stroke-linecap="round"><g transform="translate(1826.5126721262648 801.3905374028327) rotate(0 253.93790454727468 50.75553872723867)"><path d="M0.2 -0.53 C34.45 10.55, 121.21 49.48, 206.01 66.43 C290.82 83.39, 458.4 95.38, 509.02 101.2 M-1.15 1.81 C32.97 13, 120.62 50.74, 205.55 67.45 C290.47 84.15, 457.56 96.27, 508.39 102.04" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1826.5126721262648 801.3905374028327) rotate(0 253.93790454727468 50.75553872723867)"><path d="M478.47 111.1 C487.73 107.34, 497.88 105.96, 507.12 103.37 M479.19 108.67 C487.03 107.78, 496.77 105.63, 508.35 102.94" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1826.5126721262648 801.3905374028327) rotate(0 253.93790454727468 50.75553872723867)"><path d="M480.42 90.67 C489 93.97, 498.48 99.64, 507.12 103.37 M481.14 88.24 C488.21 93.34, 497.38 97.18, 508.35 102.94" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask></mask><g stroke-linecap="round"><g transform="translate(2461.2847948561116 891.1061373014109) rotate(0 1.708079074100965 -60.309881883255)"><path d="M0.91 -0.99 C1.46 -21.19, 2.93 -101.62, 3.48 -121.72 M-0.06 1.1 C0.33 -18.93, 1.99 -100.37, 2.66 -120.66" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2461.2847948561116 891.1061373014109) rotate(0 1.708079074100965 -60.309881883255)"><path d="M10.54 -91.8 C7.77 -99.53, 5.01 -108.85, 2.53 -122.61 M11.28 -91.9 C10.02 -99.46, 7.16 -106.46, 3.6 -121.01" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2461.2847948561116 891.1061373014109) rotate(0 1.708079074100965 -60.309881883255)"><path d="M-9.97 -92.33 C-6.24 -100.04, -2.5 -109.2, 2.53 -122.61 M-9.23 -92.43 C-5.06 -99.66, -2.5 -106.52, 3.6 -121.01" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(2356.1017156294083 567.9370110758537) rotate(0 120 25)"><path d="M3.33 2.89 C3.33 2.89, 3.33 2.89, 3.33 2.89 M3.33 2.89 C3.33 2.89, 3.33 2.89, 3.33 2.89 M1.56 7.98 C3.99 6.73, 4.73 3.92, 6.8 1.94 M1.56 7.98 C3.33 6.13, 4.6 4.59, 6.8 1.94 M0.44 12.31 C3.27 8.26, 6.33 4.67, 10.28 0.99 M0.44 12.31 C3.68 8.15, 6.63 4.68, 10.28 0.99 M-0.02 15.89 C3.77 10.38, 8.37 7.23, 13.1 0.79 M-0.02 15.89 C4.39 10.21, 9.55 3.88, 13.1 0.79 M0.18 18.71 C5.49 14.04, 8.93 10.1, 15.93 0.59 M0.18 18.71 C6.59 12.55, 11.02 6.54, 15.93 0.59 M0.38 21.53 C5 13.11, 13.83 6.33, 18.75 0.4 M0.38 21.53 C6.02 15.73, 11.3 9.67, 18.75 0.4 M0.57 24.35 C7.63 18.17, 13.89 11.97, 20.91 0.95 M0.57 24.35 C8.61 15.72, 16.14 7.16, 20.91 0.95 M0.77 27.17 C9.53 17.84, 15.77 9.16, 23.73 0.76 M0.77 27.17 C9.84 17.66, 17.93 9.22, 23.73 0.76 M0.31 30.75 C9.52 19.52, 17.19 9.43, 26.56 0.56 M0.31 30.75 C9.44 20.32, 18.51 10.31, 26.56 0.56 M0.51 33.57 C11.5 20.87, 22.43 6.16, 29.38 0.36 M0.51 33.57 C6.28 27.23, 13.48 19.08, 29.38 0.36 M0.71 36.39 C10.21 23.19, 22.33 7.77, 31.54 0.92 M0.71 36.39 C9.02 25.82, 18.05 17.39, 31.54 0.92 M0.91 39.21 C10.24 31.26, 15.32 21.99, 34.36 0.72 M0.91 39.21 C10.95 27.76, 20.18 16.18, 34.36 0.72 M1.1 42.03 C11.03 33.42, 15.87 23.45, 37.19 0.52 M1.1 42.03 C11.86 29.03, 22.93 16.41, 37.19 0.52 M1.96 44.1 C16.32 28.44, 29.88 13.23, 40.01 0.33 M1.96 44.1 C11.16 33.64, 18.61 24.92, 40.01 0.33 M3.47 45.41 C16.69 31.51, 29.88 16.83, 42.17 0.88 M3.47 45.41 C16.35 30.3, 30.22 15.45, 42.17 0.88 M4.32 47.48 C19.38 27.78, 37.35 12.12, 44.99 0.69 M4.32 47.48 C14.62 34.41, 25.67 22.09, 44.99 0.69 M5.83 48.79 C18.23 31.61, 36.04 15.55, 47.82 0.49 M5.83 48.79 C21.87 31.53, 38.23 13.1, 47.82 0.49 M7.99 49.35 C20.41 37.07, 32.45 21.64, 49.98 1.05 M7.99 49.35 C21.82 34.13, 35.54 18.39, 49.98 1.05 M10.16 49.91 C25.4 33.96, 39.7 16.47, 52.8 0.85 M10.16 49.91 C25.14 34.38, 38.06 17.07, 52.8 0.85 M12.32 50.46 C25.1 32.63, 39.33 18.34, 55.62 0.65 M12.32 50.46 C24.26 35.46, 36.52 21.96, 55.62 0.65 M15.15 50.27 C25.46 37.09, 36.91 26.2, 58.45 0.46 M15.15 50.27 C25.1 38.26, 35.3 25.46, 58.45 0.46 M17.97 50.07 C29.66 34.91, 46.06 20.62, 60.61 1.01 M17.97 50.07 C30.95 36.8, 44.05 21.82, 60.61 1.01 M20.79 49.87 C31.93 40.04, 40.65 27.97, 63.43 0.82 M20.79 49.87 C33.29 35.06, 45.35 23.06, 63.43 0.82 M22.95 50.43 C40.09 31.68, 51.95 14.42, 66.25 0.62 M22.95 50.43 C35.03 36.45, 48.64 20.76, 66.25 0.62 M25.78 50.23 C36.9 37.93, 46.05 26.4, 69.08 0.42 M25.78 50.23 C40.12 34.19, 54.53 15.68, 69.08 0.42 M28.6 50.03 C42.86 32.45, 57.23 16.99, 71.24 0.98 M28.6 50.03 C39.67 36.36, 51.11 23.16, 71.24 0.98 M31.42 49.84 C41.39 39.19, 55.14 25.5, 74.06 0.78 M31.42 49.84 C40.97 38.74, 50.28 27.47, 74.06 0.78 M33.58 50.39 C44.9 35.83, 55.85 24.38, 76.88 0.58 M33.58 50.39 C43.52 41.05, 51.12 30.16, 76.88 0.58 M36.41 50.2 C50.61 32.15, 66.58 16.36, 79.71 0.39 M36.41 50.2 C48.62 36.96, 60.38 23.19, 79.71 0.39 M39.23 50 C49.44 36.87, 59.87 26.65, 81.87 0.94 M39.23 50 C53.65 33.71, 65.93 19.24, 81.87 0.94 M42.05 49.8 C55.85 33.44, 74.31 14.51, 84.69 0.75 M42.05 49.8 C55.64 34.01, 71.36 16.78, 84.69 0.75 M44.21 50.36 C56.34 36.49, 70.42 19.53, 87.51 0.55 M44.21 50.36 C55.63 37.03, 66.81 23.32, 87.51 0.55 M47.04 50.16 C63.62 30.94, 80.97 11.5, 90.34 0.35 M47.04 50.16 C62.85 30.21, 79.82 10.64, 90.34 0.35 M49.86 49.97 C59.79 38.11, 71.84 26.37, 92.5 0.91 M49.86 49.97 C61.61 35.82, 75.9 21.43, 92.5 0.91 M52.02 50.52 C63.71 33.49, 77.45 20.23, 95.32 0.71 M52.02 50.52 C65.34 34.09, 80.71 19.9, 95.32 0.71 M54.84 50.33 C69.33 29.93, 85.97 13.48, 98.14 0.52 M54.84 50.33 C69.01 36.02, 80.84 21.65, 98.14 0.52 M57.67 50.13 C68.83 36.35, 82.18 21.8, 100.31 1.07 M57.67 50.13 C73.92 33.07, 88.34 14.84, 100.31 1.07 M60.49 49.93 C74.57 35.47, 86.31 19.51, 103.13 0.88 M60.49 49.93 C77.47 30.77, 92.51 13.08, 103.13 0.88 M62.65 50.49 C77.22 35.2, 90.26 15.64, 105.95 0.68 M62.65 50.49 C73.59 38.38, 84.17 25.52, 105.95 0.68 M65.47 50.29 C82.22 33.81, 98.4 14.73, 108.77 0.48 M65.47 50.29 C76.07 38.45, 85.57 25.9, 108.77 0.48 M68.3 50.09 C77.24 39.4, 84.11 30.99, 110.94 1.04 M68.3 50.09 C79.13 38.33, 89.52 25.92, 110.94 1.04 M71.12 49.9 C85.86 32.9, 98.05 19.55, 113.76 0.84 M71.12 49.9 C82.19 37.54, 95.13 22.5, 113.76 0.84 M73.28 50.45 C86.53 34.83, 101.99 18.73, 116.58 0.64 M73.28 50.45 C89.4 32.51, 104.31 16.3, 116.58 0.64 M76.1 50.26 C88.75 34.37, 105.19 17.87, 119.4 0.45 M76.1 50.26 C84.95 40.69, 95.62 29.3, 119.4 0.45 M78.93 50.06 C95.02 33.9, 109.77 14.17, 121.57 1 M78.93 50.06 C89.6 37.81, 100.37 26.64, 121.57 1 M81.75 49.86 C98.13 34.37, 112.31 17.61, 124.39 0.81 M81.75 49.86 C93.45 37.03, 105.28 22.92, 124.39 0.81 M83.91 50.42 C98.29 30.82, 116.97 13.4, 127.21 0.61 M83.91 50.42 C99.01 31.2, 115.19 12.58, 127.21 0.61 M86.74 50.22 C95.09 38.73, 106.39 23.9, 130.03 0.41 M86.74 50.22 C98.88 36.47, 110.5 22.93, 130.03 0.41 M89.56 50.03 C103.8 33.55, 113.84 18.84, 132.2 0.97 M89.56 50.03 C102.3 37.31, 113.31 21.68, 132.2 0.97 M91.72 50.58 C102.16 38.88, 110.45 26.96, 135.02 0.77 M91.72 50.58 C100.43 38.49, 110.97 27.86, 135.02 0.77 M94.54 50.39 C106.23 38.36, 119.45 21.82, 137.84 0.57 M94.54 50.39 C105.36 39.08, 116.12 25.23, 137.84 0.57 M97.37 50.19 C105.82 40.33, 118.63 25.6, 140.67 0.38 M97.37 50.19 C106.53 40.57, 116.35 29.36, 140.67 0.38 M100.19 49.99 C113.22 35.2, 122.77 23.78, 142.83 0.93 M100.19 49.99 C112.3 37.65, 122.74 23.24, 142.83 0.93 M102.35 50.55 C115.74 36.04, 127.57 21.95, 145.65 0.74 M102.35 50.55 C116.11 36.77, 128.51 21.71, 145.65 0.74 M105.17 50.35 C115.82 37.34, 128.92 24.36, 148.47 0.54 M105.17 50.35 C115.84 37.67, 127.96 24.48, 148.47 0.54 M108 50.15 C119.12 40.12, 129.51 26.97, 150.64 1.1 M108 50.15 C123.38 31.49, 139.68 11.77, 150.64 1.1 M110.82 49.96 C123.8 33.6, 135.47 21.01, 153.46 0.9 M110.82 49.96 C121.33 39.14, 132.11 27.61, 153.46 0.9 M112.98 50.51 C125.83 36.28, 140.85 20.27, 156.28 0.7 M112.98 50.51 C122.82 39.9, 130.62 29.82, 156.28 0.7 M115.8 50.32 C131.29 30.45, 149.11 11.74, 159.1 0.51 M115.8 50.32 C133.24 30.06, 150.2 10.66, 159.1 0.51 M118.63 50.12 C129.06 40.89, 137.15 30.77, 161.27 1.06 M118.63 50.12 C131.11 36, 145.2 19.04, 161.27 1.06 M121.45 49.92 C132.84 34.76, 145.26 25.19, 164.09 0.87 M121.45 49.92 C137.76 30.46, 153.91 12.12, 164.09 0.87 M123.61 50.48 C134.89 36.44, 149.55 21.9, 166.91 0.67 M123.61 50.48 C133.4 38.28, 146.19 24.71, 166.91 0.67 M126.43 50.28 C134.31 38, 145.39 27.64, 169.73 0.47 M126.43 50.28 C137.13 38.91, 147.56 26.05, 169.73 0.47 M129.26 50.08 C137.63 38.79, 149.22 27.23, 171.9 1.03 M129.26 50.08 C139.37 38.23, 149.21 27.03, 171.9 1.03 M132.08 49.89 C147.62 34.07, 162.06 16.07, 174.72 0.83 M132.08 49.89 C143.49 37.89, 154.45 23.56, 174.72 0.83 M134.24 50.44 C143.8 36.19, 157.53 23.11, 177.54 0.63 M134.24 50.44 C146.46 36.69, 156.37 23.74, 177.54 0.63 M137.06 50.25 C148.24 35.35, 162.3 22.52, 180.36 0.44 M137.06 50.25 C150.45 35.01, 163.33 19.73, 180.36 0.44 M139.89 50.05 C156.35 34.46, 169.26 16.43, 182.53 0.99 M139.89 50.05 C154.96 30.3, 171.17 11.89, 182.53 0.99 M142.05 50.61 C157.27 33.39, 170.99 16.78, 185.35 0.8 M142.05 50.61 C156.75 33.29, 173.1 15.47, 185.35 0.8 M144.87 50.41 C162.83 32.97, 176.19 11.47, 188.17 0.6 M144.87 50.41 C155.36 40.2, 163.58 29.02, 188.17 0.6 M147.69 50.21 C156.45 40.42, 167.08 28.86, 190.34 1.16 M147.69 50.21 C158.42 36.77, 170.05 24.99, 190.34 1.16 M150.52 50.02 C165.05 33.61, 178.16 16.69, 193.16 0.96 M150.52 50.02 C162.24 36.81, 174.38 22.12, 193.16 0.96 M152.68 50.57 C165.11 36.76, 177.26 21.22, 195.98 0.76 M152.68 50.57 C165.06 35.67, 177.55 21.04, 195.98 0.76 M155.5 50.38 C170.38 32.69, 189.93 12.46, 198.8 0.56 M155.5 50.38 C167.21 37.2, 176.46 26.57, 198.8 0.56 M158.32 50.18 C172.03 32.83, 185.22 20.08, 200.97 1.12 M158.32 50.18 C173.78 33.31, 188.11 15.81, 200.97 1.12 M161.15 49.98 C177.04 32.12, 193.5 14.41, 203.79 0.92 M161.15 49.98 C170.87 37.55, 183.41 24.64, 203.79 0.92 M163.31 50.54 C178.65 33.68, 193.22 17.36, 206.61 0.73 M163.31 50.54 C175.41 35.24, 189.2 22.77, 206.61 0.73 M166.13 50.34 C176.9 35.33, 189.98 21.16, 209.43 0.53 M166.13 50.34 C175.54 38.3, 187.26 27.06, 209.43 0.53 M168.95 50.14 C181.32 36.64, 195.43 21.64, 211.6 1.09 M168.95 50.14 C181.74 35.06, 194.49 20.38, 211.6 1.09 M171.78 49.95 C184.93 36.36, 197.24 23.68, 214.42 0.89 M171.78 49.95 C188 32.3, 203.66 14.61, 214.42 0.89 M173.94 50.5 C189.58 34.26, 204.66 13.53, 217.24 0.69 M173.94 50.5 C185.73 35.57, 197.42 23.19, 217.24 0.69 M176.76 50.31 C193.92 31.47, 207.54 14.81, 220.06 0.5 M176.76 50.31 C185.63 40.05, 194.47 29.63, 220.06 0.5 M179.58 50.11 C194.25 32.7, 207.35 17.57, 222.23 1.05 M179.58 50.11 C189.82 37.11, 201.65 26.07, 222.23 1.05 M181.75 50.67 C197.35 31.34, 210.74 14.86, 225.05 0.86 M181.75 50.67 C199.05 31.26, 216.42 11.6, 225.05 0.86 M184.57 50.47 C200 33.16, 217.94 12.14, 227.87 0.66 M184.57 50.47 C200.69 32.73, 214.79 13.83, 227.87 0.66 M187.39 50.27 C202.43 33.61, 215.38 15.94, 230.69 0.46 M187.39 50.27 C202.49 32.67, 216.81 15.61, 230.69 0.46 M190.21 50.07 C200.08 39.25, 211.07 26.39, 232.86 1.02 M190.21 50.07 C200.71 37.54, 213.07 23.75, 232.86 1.02 M192.38 50.63 C206.52 35.21, 222.39 19.76, 234.37 2.33 M192.38 50.63 C205.77 35.04, 218.43 19.61, 234.37 2.33 M195.2 50.43 C206.2 37.99, 218.57 20.64, 236.53 2.89 M195.2 50.43 C211.64 32.68, 227.01 15.43, 236.53 2.89 M198.02 50.24 C213.02 33.85, 226.32 21.57, 238.04 4.2 M198.02 50.24 C212.66 32.78, 227.79 16.58, 238.04 4.2 M200.84 50.04 C212.2 36.22, 222.62 23.84, 238.9 6.27 M200.84 50.04 C215.45 32.89, 228.71 17.18, 238.9 6.27 M203.01 50.6 C211.83 41.18, 219.56 31.67, 240.41 7.58 M203.01 50.6 C211.74 40.96, 219.86 30.71, 240.41 7.58 M205.83 50.4 C220.17 34.45, 230.23 23.29, 239.95 11.16 M205.83 50.4 C215.83 37.81, 226.26 25.62, 239.95 11.16 M208.65 50.2 C220.73 36.12, 234.84 20.32, 240.14 13.98 M208.65 50.2 C220.59 35.35, 232.33 20.66, 240.14 13.98 M211.47 50.01 C219.78 43.29, 224.45 33.94, 240.34 16.8 M211.47 50.01 C220.82 38.71, 231.91 28.75, 240.34 16.8 M213.64 50.56 C219.93 45.33, 226.25 35.93, 240.54 19.62 M213.64 50.56 C218.77 44.75, 224.4 38.44, 240.54 19.62 M216.46 50.37 C225.99 39.42, 232.29 31.05, 240.74 22.44 M216.46 50.37 C224.15 40.6, 231.81 31.48, 240.74 22.44 M219.28 50.17 C226.05 42.53, 234.06 30.18, 240.28 26.02 M219.28 50.17 C223.72 45.11, 227.79 38.94, 240.28 26.02 M222.1 49.97 C227.17 45.69, 232.66 41.07, 240.47 28.84 M222.1 49.97 C226.23 43.65, 231.24 38.86, 240.47 28.84 M224.27 50.53 C226.4 46.72, 234.26 39.98, 240.67 31.66 M224.27 50.53 C227.33 47.06, 230.75 41.13, 240.67 31.66 M227.09 50.33 C229.42 47.66, 233.98 42.72, 240.87 34.48 M227.09 50.33 C231.05 45.69, 235.21 41, 240.87 34.48 M229.26 50.89 C232.14 45.45, 239.5 41.73, 240.41 38.06 M229.26 50.89 C231.33 48.59, 234.19 45.56, 240.41 38.06 M232.73 49.94 C235.1 46.24, 238.49 44.42, 240.61 40.88 M232.73 49.94 C234.55 48.35, 236.98 46.12, 240.61 40.88" stroke="#ffec99" stroke-width="0.25" fill="none"></path><path d="M12.5 0 M12.5 0 C64.07 -2.23, 113.9 -1.37, 227.5 0 M12.5 0 C68.04 -0.04, 123.7 -0.8, 227.5 0 M227.5 0 C235.03 1.01, 241.91 3.47, 240 12.5 M227.5 0 C234.54 -0.73, 238.1 3.43, 240 12.5 M240 12.5 C241.31 18.62, 238.2 29.36, 240 37.5 M240 12.5 C240.9 19.5, 241.18 26.39, 240 37.5 M240 37.5 C239.26 46.82, 235.55 49.42, 227.5 50 M240 37.5 C238.92 45.69, 237.18 50.66, 227.5 50 M227.5 50 C153.26 50.82, 77.08 50.68, 12.5 50 M227.5 50 C178.09 51.21, 131.01 51.33, 12.5 50 M12.5 50 C5.57 50.69, 0.67 46.57, 0 37.5 M12.5 50 C2.81 48, -0.84 47.71, 0 37.5 M0 37.5 C0.67 31.35, -0.43 23.64, 0 12.5 M0 37.5 C0.5 28.06, -0.91 20.55, 0 12.5 M0 12.5 C-1.5 2.35, 4.08 1.24, 12.5 0 M0 12.5 C-2.02 3.89, 4.45 -1.8, 12.5 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2401.721771781752 580.4370110758537) rotate(0 74.37994384765625 12.5)"><text x="74.37994384765625" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">WaitForReady()</text></g><g stroke-linecap="round" transform="translate(2360.605443261682 387.87648566907774) rotate(0 116.5 20.5)"><path d="M10.25 0 M10.25 0 C91.25 -2.95, 171.87 -1.53, 222.75 0 M10.25 0 C54.99 -0.26, 99.16 0.2, 222.75 0 M222.75 0 C231.45 -0.69, 232.82 3.67, 233 10.25 M222.75 0 C231.58 -2.21, 234.08 3.33, 233 10.25 M233 10.25 C232.33 16.7, 232.36 20.52, 233 30.75 M233 10.25 C233.38 16.07, 233.4 19.17, 233 30.75 M233 30.75 C233.6 35.92, 229.25 41.4, 222.75 41 M233 30.75 C232.44 37.7, 228.13 42.14, 222.75 41 M222.75 41 C159.95 40, 98.45 40.11, 10.25 41 M222.75 41 C166.98 40.55, 109.72 40.17, 10.25 41 M10.25 41 C4.89 41.72, -1.73 36.98, 0 30.75 M10.25 41 C5.63 40.73, 0.21 37.49, 0 30.75 M0 30.75 C-1.19 22.53, 1.83 15.95, 0 10.25 M0 30.75 C0.58 25.21, 0.79 19.33, 0 10.25 M0 10.25 C-0.3 4.8, 5.19 1.43, 10.25 0 M0 10.25 C0.76 5.59, 4.72 1.79, 10.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2432.115483544885 395.87648566907774) rotate(0 44.989959716796875 12.5)"><text x="44.989959716796875" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">Connect()</text></g><g stroke-linecap="round"><g transform="translate(2481.879219806936 422.8590814145782) rotate(0 0.24542602784003975 61.82937405903249)"><path d="M-0.2 -1.09 C0.02 19.77, -0.18 103.76, 0.12 124.75 M-1.76 0.95 C-1.09 21.48, 1.79 102.53, 2.25 122.82" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2481.879219806936 422.8590814145782) rotate(0 0.24542602784003975 61.82937405903249)"><path d="M-10.66 93.13 C-6.06 101.23, -3.7 106.21, 4.05 124.69 M-7.98 93.99 C-5.46 102.31, -3.44 110.59, 2.7 122.4" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2481.879219806936 422.8590814145782) rotate(0 0.24542602784003975 61.82937405903249)"><path d="M9.85 92.51 C10.29 100.71, 8.49 105.81, 4.05 124.69 M12.53 93.38 C9.6 101.88, 6.17 110.33, 2.7 122.4" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask></mask><g stroke-linecap="round"><g transform="translate(2476.315329432925 628.8339278917038) rotate(0 -0.1367275121659759 33.65393252462809)"><path d="M-1.1 0.38 C-0.93 11.86, 0.57 56.44, 0.82 67.77 M0.53 -0.47 C0.59 10.74, 0.22 54.71, 0.07 65.89" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2476.315329432925 628.8339278917038) rotate(0 -0.1367275121659759 33.65393252462809)"><path d="M-9.27 38.2 C-6.58 45.63, -2.71 55.59, -0.62 65.71 M-9.43 37.55 C-7.01 47.37, -2.98 54.95, 0.7 65.48" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2476.315329432925 628.8339278917038) rotate(0 -0.1367275121659759 33.65393252462809)"><path d="M11.25 38.41 C8.12 45.6, 6.17 55.5, -0.62 65.71 M11.09 37.76 C7.09 47.32, 4.71 54.84, 0.7 65.48" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(2363.786681990579 689.1901479212938) rotate(0 116.5 20.5)"><path d="M10.25 0 M10.25 0 C55.24 -0.63, 100.5 1.07, 222.75 0 M10.25 0 C67.47 -1.08, 123.58 -0.27, 222.75 0 M222.75 0 C228.64 1.68, 233.89 2.57, 233 10.25 M222.75 0 C231.03 -0.94, 232.88 1.97, 233 10.25 M233 10.25 C234.93 17.46, 231.39 23.78, 233 30.75 M233 10.25 C232.72 14.74, 232.33 19.63, 233 30.75 M233 30.75 C232.86 39.34, 229.77 40.4, 222.75 41 M233 30.75 C231.46 37.91, 228.2 40.23, 222.75 41 M222.75 41 C165.3 43.19, 106.27 44.09, 10.25 41 M222.75 41 C152.17 40.34, 79.97 40.41, 10.25 41 M10.25 41 C3.74 40.49, 0.03 36.34, 0 30.75 M10.25 41 C1.42 43.05, -2.11 37.03, 0 30.75 M0 30.75 C0.13 24.63, -0.33 14.24, 0 10.25 M0 30.75 C0.15 24.13, 0.38 16.39, 0 10.25 M0 10.25 C-0.92 2.52, 3.89 0.24, 10.25 0 M0 10.25 C0.18 4.35, 4.31 2.27, 10.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2428.596740584329 697.1901479212938) rotate(0 51.68994140625 12.5)"><text x="51.68994140625" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">Configure()</text></g><g stroke-linecap="round" transform="translate(2339.6331084344206 327.0159682781823) rotate(0 149.58805398767788 214.8219191126218)"><path d="M32 0 M32 0 C101.11 -1.99, 171.58 -0.39, 267.18 0 M267.18 0 C286.59 0.94, 299.1 9.73, 299.18 32 M299.18 32 C300.98 171.88, 300.13 311.99, 299.18 397.64 M299.18 397.64 C299.48 419.08, 289.2 431.56, 267.18 429.64 M267.18 429.64 C186.48 429.95, 108.93 429.82, 32 429.64 M32 429.64 C10.77 428.38, 0.99 418.84, 0 397.64 M0 397.64 C-1.04 255.96, -0.96 113.51, 0 32 M0 32 C-0.52 9.76, 12.59 0.91, 32 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"></path></g><g stroke-linecap="round" transform="translate(2309.4927758771514 119.86548781130386) rotate(0 190.43722219901656 479.37661568433987)"><path d="M32 0 M32 0 C156.8 1.42, 279.9 0.86, 348.87 0 M32 0 C126.13 -2.17, 220.08 -2.16, 348.87 0 M348.87 0 C370.1 -1.26, 381.18 10.77, 380.87 32 M348.87 0 C371 2.2, 381.73 8.8, 380.87 32 M380.87 32 C378.05 327.03, 377.93 621.96, 380.87 926.75 M380.87 32 C380.66 344.47, 380.36 657.24, 380.87 926.75 M380.87 926.75 C379.67 947.42, 369.69 957.84, 348.87 958.75 M380.87 926.75 C383.08 949.14, 370.73 958.79, 348.87 958.75 M348.87 958.75 C230.07 958.18, 111.95 956.95, 32 958.75 M348.87 958.75 C255.08 957.75, 160.66 958.81, 32 958.75 M32 958.75 C8.83 958.27, 1.9 949.04, 0 926.75 M32 958.75 C12.45 959.02, 2.27 947.82, 0 926.75 M0 926.75 C-1.91 672.13, -1.72 417.44, 0 32 M0 926.75 C1.73 588.57, 1.67 250.46, 0 32 M0 32 C0.77 12.64, 10.3 1.24, 32 0 M0 32 C-0.5 10.33, 10.48 1.22, 32 0" stroke="#1e1e1e" stroke-width="4" fill="none"></path></g><g transform="translate(2336.854232661256 131.28246764997766) rotate(0 45.905967712402344 17.5)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="28px" fill="#e03131" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">plugin.h</text></g><g mask="url(#mask-tNa8LJUAE4h13JL4UE0aU)" stroke-linecap="round"><g transform="translate(2583.3291895067046 429.87648566907774) rotate(0 27.177049285369435 460.0288721262573)"><path d="M0.56 -0.05 C14.76 97.95, 91.63 433.48, 86.25 586.85 C80.88 740.21, -12.06 864.57, -31.68 920.12 M-0.6 -1.12 C13.3 96.67, 90.27 431.54, 85.01 585.25 C79.75 738.97, -12.86 865.22, -32.17 921.18" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2583.3291895067046 429.87648566907774) rotate(0 27.177049285369435 460.0288721262573)"><path d="M-29.26 890.38 C-28.94 903.78, -32.62 915.68, -32.98 921.07 M-28.84 892.28 C-29.47 897.6, -30.25 904.66, -32.33 921.38" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2583.3291895067046 429.87648566907774) rotate(0 27.177049285369435 460.0288721262573)"><path d="M-10.77 899.29 C-17.52 909.32, -28.34 917.79, -32.98 921.07 M-10.35 901.18 C-15.38 904.4, -20.55 909.34, -32.33 921.38" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask id="mask-tNa8LJUAE4h13JL4UE0aU"><rect x="0" y="0" fill="#fff" width="2802.3640622970233" height="1450.5067696917304"></rect><rect x="2637.564846609109" y="1003.2152901995905" fill="#000" width="65.15994262695312" height="25" opacity="1"></rect></mask><g transform="translate(2637.564846609109 1003.2152901995905) rotate(0 -27.058607817034613 -113.3099324042555)"><text x="32.57997131347656" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">create</text></g><g mask="url(#mask-LrOHeB0UlwLSTQErYV_xH)" stroke-linecap="round"><g transform="translate(2598.4981580659055 715.4945154647676) rotate(0 50.4703276388185 317.9066741016918)"><path d="M0.54 -0.51 C22.31 53.13, 134.94 216.64, 130.1 322.84 C125.26 429.04, -2.12 584.26, -28.51 636.71 M-0.64 -1.82 C21.09 52.5, 134.33 217.55, 129.55 324.12 C124.77 430.7, -3.07 585.21, -29.31 637.63" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2598.4981580659055 715.4945154647676) rotate(0 50.4703276388185 317.9066741016918)"><path d="M-23.68 609.6 C-25.04 615.67, -27.12 623.5, -29.21 638.32 M-22.53 607.51 C-25.46 620.08, -28.1 631.52, -29.94 638.13" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2598.4981580659055 715.4945154647676) rotate(0 50.4703276388185 317.9066741016918)"><path d="M-6.36 620.62 C-12.2 623.98, -18.66 629.02, -29.21 638.32 M-5.22 618.52 C-15.04 626.6, -24.53 633.68, -29.94 638.13" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask id="mask-LrOHeB0UlwLSTQErYV_xH"><rect x="0" y="0" fill="#fff" width="2856.288281578487" height="1452.9572027189315"></rect><rect x="2710.891670065227" y="1026.326511050681" fill="#000" width="33.89997863769531" height="25" opacity="1"></rect></mask><g transform="translate(2710.891670065227 1026.326511050681) rotate(0 -61.923184360503114 7.074678515778487)"><text x="16.949989318847656" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">call</text></g><g stroke-linecap="round"><g transform="translate(1700.8565399888764 804.6401852767158) rotate(0 -0.07100263651312844 43.01698527064764)"><path d="M-0.06 -0.75 C0.05 13.53, 1.13 70.71, 1.42 85.22 M-1.56 1.47 C-1.65 16, 0.13 72.94, 0.57 86.79" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1700.8565399888764 804.6401852767158) rotate(0 -0.07100263651312844 43.01698527064764)"><path d="M-11.81 59.24 C-6.53 69.73, -0.68 79.9, -1.05 87.39 M-10.8 58.98 C-8.82 65.44, -5.85 71.86, 0.66 86.49" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1700.8565399888764 804.6401852767158) rotate(0 -0.07100263651312844 43.01698527064764)"><path d="M8.7 58.6 C6.53 69.32, 4.94 79.72, -1.05 87.39 M9.71 58.34 C7.24 64.98, 5.77 71.54, 0.66 86.49" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1581.0590702593133 905.6958114124654) rotate(0 123.5 31)"><path d="M15.5 0 M15.5 0 C79.23 0.31, 144.45 0.36, 231.5 0 M15.5 0 C62.65 1.75, 109.87 1.62, 231.5 0 M231.5 0 C240.57 0.99, 246.86 6.93, 247 15.5 M231.5 0 C242.04 -0.69, 245.46 5.49, 247 15.5 M247 15.5 C245.99 24.85, 247.81 30.82, 247 46.5 M247 15.5 C246.36 24.55, 248.19 34.09, 247 46.5 M247 46.5 C247.19 56.75, 242.16 61.49, 231.5 62 M247 46.5 C247.03 55.4, 239.83 64.05, 231.5 62 M231.5 62 C187.34 65.15, 141.71 65.52, 15.5 62 M231.5 62 C169.96 63.93, 107.16 64.13, 15.5 62 M15.5 62 C6.3 63.56, -0.92 55.94, 0 46.5 M15.5 62 C5.71 62.27, 0.18 57.77, 0 46.5 M0 46.5 C0.94 35.57, -0.6 25.34, 0 15.5 M0 46.5 C0.98 35.24, 0.97 25.18, 0 15.5 M0 15.5 C0.51 6.88, 3.92 0.93, 15.5 0 M0 15.5 C-1.81 5.23, 3.05 1.01, 15.5 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1613.4391514360711 924.1958114124654) rotate(0 91.11991882324219 12.5)"><text x="91.11991882324219" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">AddPluginByIndex()</text></g><g transform="translate(617.8636750995107 195.36737834316955) rotate(0 108.61988830566406 12.5)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">conf_get_nri_support</text></g><g transform="translate(569.5221941231623 239.07094340847016) rotate(0 163.26983642578125 12.5)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">conf_get_nri_plugin_config_path</text></g><g transform="translate(593.6036565060843 287.2287876306327) rotate(0 127.71987915039062 12.5)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">conf_get_nri_plugin_path</text></g><g transform="translate(562.2084664154596 334.6783857792989) rotate(0 207.36976623535156 12.5)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">conf_get_nri_plugin_registration_timeout</text></g><g transform="translate(570.7707810878355 380.70097684745997) rotate(0 182.28981018066406 12.5)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">conf_get_nri_plugin_requst_timeout</text></g><g stroke-linecap="round" transform="translate(537.9484478220331 181.62664355496145) rotate(0 223.69108324185004 131.6459283244626)"><path d="M32 0 M32 0 C123.83 0.27, 213.22 -0.64, 415.38 0 M415.38 0 C437 -1.21, 446.71 10.15, 447.38 32 M447.38 32 C449.75 82.77, 450.77 134.2, 447.38 231.29 M447.38 231.29 C445.65 252.03, 438.64 263.06, 415.38 263.29 M415.38 263.29 C296.49 263.68, 177.46 262.63, 32 263.29 M32 263.29 C12.45 261.46, -0.49 254.53, 0 231.29 M0 231.29 C0.43 161.79, -1.85 94.13, 0 32 M0 32 C1.77 12.09, 11.33 1.89, 32 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"></path></g><g transform="translate(615.8540934831792 416.43051850539996) rotate(0 115.16989135742188 12.5)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">conf_get_socket_path</text></g><g stroke-linecap="round" transform="translate(1857.45363715511 1708.06763419055) rotate(0 108.5 39.5)"><path d="M5.26 4.57 C5.26 4.57, 5.26 4.57, 5.26 4.57 M5.26 4.57 C5.26 4.57, 5.26 4.57, 5.26 4.57 M2.17 11.16 C4.09 6.93, 8.75 3.77, 10.7 1.35 M2.17 11.16 C4.94 8.49, 7.43 5.64, 10.7 1.35 M1.06 15.5 C5.36 12.31, 8.93 9.05, 12.21 2.67 M1.06 15.5 C5.54 10.8, 9.62 6.19, 12.21 2.67 M0.6 19.07 C6.78 12.79, 10.35 7.18, 15.69 1.71 M0.6 19.07 C6.84 12.72, 12.16 7.47, 15.69 1.71 M0.8 21.89 C7.73 13.32, 13.05 5.93, 19.82 0.01 M0.8 21.89 C7.44 14.35, 14.03 7.2, 19.82 0.01 M1 24.72 C9.1 15.28, 17.24 3.79, 22.65 -0.19 M1 24.72 C5.26 20.18, 10.88 13.85, 22.65 -0.19 M1.19 27.54 C8.02 17.48, 17.45 5.15, 24.81 0.37 M1.19 27.54 C7.5 19.23, 14.58 13.03, 24.81 0.37 M0.73 31.11 C8.6 24.89, 12.16 17.36, 27.63 0.17 M0.73 31.11 C8.84 21.89, 16.14 12.54, 27.63 0.17 M0.93 33.93 C9.4 27.08, 12.72 18.85, 30.45 -0.03 M0.93 33.93 C9.67 23.26, 18.69 12.98, 30.45 -0.03 M1.13 36.76 C13.43 23.51, 24.87 10.74, 33.28 -0.22 M1.13 36.76 C9.05 27.81, 15.17 20.62, 33.28 -0.22 M0.67 40.33 C12.54 27.9, 24.42 14.72, 35.44 0.33 M0.67 40.33 C12.19 26.76, 24.7 13.48, 35.44 0.33 M0.87 43.15 C14.59 24.95, 31.26 10.79, 38.26 0.14 M0.87 43.15 C10.27 31.11, 20.43 19.81, 38.26 0.14 M1.06 45.98 C12.8 29.55, 29.95 14.25, 41.08 -0.06 M1.06 45.98 C16.37 29.57, 31.99 11.98, 41.08 -0.06 M0.61 49.55 C13.42 36.84, 25.84 20.97, 43.91 -0.26 M0.61 49.55 C14.86 33.86, 28.99 17.63, 43.91 -0.26 M0.8 52.37 C16.93 35.4, 32.12 16.88, 46.07 0.3 M0.8 52.37 C16.66 35.87, 30.44 17.56, 46.07 0.3 M1 55.19 C15.26 35.6, 30.98 19.59, 48.89 0.1 M1 55.19 C14.19 38.68, 27.77 23.67, 48.89 0.1 M1.2 58.02 C13.38 42.74, 26.65 29.76, 51.71 -0.1 M1.2 58.02 C12.84 43.97, 24.78 29.17, 51.71 -0.1 M0.74 61.59 C15.91 42.65, 35.65 24.5, 54.54 -0.29 M0.74 61.59 C17.04 44.63, 33.39 25.9, 54.54 -0.29 M0.94 64.41 C14.91 51.35, 26.54 35.93, 56.7 0.26 M0.94 64.41 C17.03 45.33, 32.8 29.06, 56.7 0.26 M1.79 66.48 C24.16 41.85, 41.11 18.72, 59.52 0.07 M1.79 66.48 C18.05 47.7, 35.92 27.12, 59.52 0.07 M2.64 68.55 C17.56 51.95, 30.33 36.24, 62.34 -0.13 M2.64 68.55 C22.51 46.13, 42.54 21.16, 62.34 -0.13 M2.84 71.37 C23.97 45.79, 45.16 22.5, 65.17 -0.33 M2.84 71.37 C19 51.56, 35.68 32.32, 65.17 -0.33 M4.35 72.68 C19.66 56.26, 38.54 36.68, 67.33 0.23 M4.35 72.68 C18.41 56.02, 32.38 39.4, 67.33 0.23 M5.86 73.99 C22.66 53.07, 39.28 35.11, 70.15 0.03 M5.86 73.99 C20.19 59.86, 32.08 44.04, 70.15 0.03 M7.37 75.31 C29.13 48.26, 52.79 23.63, 72.97 -0.17 M7.37 75.31 C25.57 54.91, 43.47 34.06, 72.97 -0.17 M8.88 76.62 C24.93 56.97, 41.09 40.16, 75.14 0.39 M8.88 76.62 C30.74 51.7, 50.71 28.37, 75.14 0.39 M11.04 77.18 C33.63 50.9, 60.75 22.01, 77.96 0.19 M11.04 77.18 C32.82 51.98, 56.68 25.39, 77.96 0.19 M12.55 78.49 C32.19 55.54, 54.05 29.64, 80.78 0 M12.55 78.49 C30.62 57.27, 48.55 35.79, 80.78 0 M14.72 79.04 C40.68 49.06, 67.59 18.61, 83.6 -0.2 M14.72 79.04 C40.35 47.59, 67.13 16.71, 83.6 -0.2 M16.88 79.6 C33.27 59.98, 51.95 40.61, 85.77 0.36 M16.88 79.6 C36.41 56.51, 58.59 33.04, 85.77 0.36 M19.7 79.4 C39.17 53.22, 60.86 30.81, 88.59 0.16 M19.7 79.4 C41.37 53.83, 64.91 30.24, 88.59 0.16 M22.53 79.21 C46.57 47.79, 72.63 20.52, 91.41 -0.04 M22.53 79.21 C44.61 56.3, 64.11 33.11, 91.41 -0.04 M25.35 79.01 C44.58 55.97, 66.04 32.09, 94.23 -0.23 M25.35 79.01 C51 51.12, 74.88 22, 94.23 -0.23 M28.17 78.81 C49.63 55.75, 68.77 31.27, 96.4 0.32 M28.17 78.81 C54.76 48.73, 79.33 20.08, 96.4 0.32 M30.33 79.37 C53.43 54.01, 75.16 24.47, 99.22 0.13 M30.33 79.37 C47.9 59.84, 64.87 39.64, 99.22 0.13 M33.16 79.17 C59.61 51.86, 85.4 21.73, 102.04 -0.07 M33.16 79.17 C49.83 60.32, 65.65 40.51, 102.04 -0.07 M35.98 78.98 C50.47 62.46, 62.62 47.98, 104.86 -0.27 M35.98 78.98 C53.59 59.71, 70.53 39.77, 104.86 -0.27 M38.8 78.78 C61.88 52.33, 82.13 29.71, 107.03 0.29 M38.8 78.78 C56.81 58.57, 76.69 35.54, 107.03 0.29 M40.97 79.34 C62.59 54.57, 86.22 29.08, 109.85 0.09 M40.97 79.34 C66.07 51.18, 89.81 24.81, 109.85 0.09 M43.79 79.14 C64.67 53.9, 89.37 27.9, 112.67 -0.11 M43.79 79.14 C58.01 63.36, 74.26 45.57, 112.67 -0.11 M46.61 78.94 C71.58 52.23, 95.5 21.95, 115.49 -0.3 M46.61 78.94 C63.83 59.56, 81 41.03, 115.49 -0.3 M48.77 79.5 C74.19 53.76, 97.37 26.65, 117.66 0.25 M48.77 79.5 C67.5 58.42, 86.65 35.9, 117.66 0.25 M51.6 79.3 C75.53 48.81, 103.79 20.38, 120.48 0.06 M51.6 79.3 C75.96 48.9, 101.71 19.28, 120.48 0.06 M54.42 79.1 C68.84 60.35, 86.47 38.24, 123.3 -0.14 M54.42 79.1 C73.26 57.34, 91.76 35.89, 123.3 -0.14 M57.24 78.91 C79.23 53.23, 97.28 29.32, 126.12 -0.34 M57.24 78.91 C77.49 57.77, 96.01 33.5, 126.12 -0.34 M59.4 79.46 C75.71 60.96, 90.03 42.1, 128.29 0.22 M59.4 79.46 C73.69 60.76, 89.99 43.51, 128.29 0.22 M62.23 79.27 C81.18 59.17, 101.58 34.36, 131.11 0.02 M62.23 79.27 C79.26 60.74, 96.51 39.42, 131.11 0.02 M65.05 79.07 C79.62 62.39, 98.5 40.68, 133.93 -0.18 M65.05 79.07 C79.75 63.18, 94.82 45.91, 133.93 -0.18 M67.87 78.87 C87.97 56.01, 104.38 36.7, 136.1 0.38 M67.87 78.87 C86.81 58.81, 104.18 36.43, 136.1 0.38 M70.03 79.43 C91.12 56.06, 110.81 32.93, 138.92 0.19 M70.03 79.43 C91.37 57.29, 111.22 33.66, 138.92 0.19 M72.86 79.23 C90.09 58.32, 109.92 37.59, 141.74 -0.01 M72.86 79.23 C90.26 59.08, 108.88 38.42, 141.74 -0.01 M75.68 79.04 C92.84 62.31, 109.27 42.21, 144.56 -0.21 M75.68 79.04 C100.86 48.73, 127.24 17.41, 144.56 -0.21 M78.5 78.84 C98.76 53.8, 117.85 32.69, 146.73 0.35 M78.5 78.84 C95.03 61.14, 111.97 42.52, 146.73 0.35 M80.66 79.4 C101.47 55.6, 124.66 30.2, 149.55 0.15 M80.66 79.4 C96.1 62.54, 109.23 46.32, 149.55 0.15 M83.49 79.2 C109.07 47.63, 137.07 17.2, 152.37 -0.05 M83.49 79.2 C111 47.42, 137.99 16.48, 152.37 -0.05 M86.31 79 C102.3 63.13, 116.01 46.54, 155.19 -0.24 M86.31 79 C107.08 55.63, 129.31 29.3, 155.19 -0.24 M89.13 78.8 C107.48 55.54, 126.79 38.05, 157.36 0.31 M89.13 78.8 C115.13 47.93, 141.25 18.13, 157.36 0.31 M91.29 79.36 C110.01 57.05, 131.96 34.12, 160.18 0.12 M91.29 79.36 C107.44 59.42, 126.82 38.28, 160.18 0.12 M94.12 79.16 C107.25 60.56, 123.71 44.01, 163 -0.08 M94.12 79.16 C110.94 60.66, 127.43 40.83, 163 -0.08 M96.94 78.97 C111.1 60.69, 128.66 42.26, 165.82 -0.28 M96.94 78.97 C113.12 59.94, 129.25 41.51, 165.82 -0.28 M99.1 79.52 C123.93 52.86, 147.76 24.06, 167.99 0.28 M99.1 79.52 C117.14 59.57, 134.93 37.39, 167.99 0.28 M101.92 79.33 C118.07 57.08, 138.64 36.13, 170.81 0.08 M101.92 79.33 C120.82 57.59, 137.61 36.72, 170.81 0.08 M104.75 79.13 C123.08 56.16, 144.18 35.22, 173.63 -0.12 M104.75 79.13 C125.92 55.02, 146.65 30.72, 173.63 -0.12 M107.57 78.93 C133.17 52.76, 155.45 23.94, 176.45 -0.31 M107.57 78.93 C132.27 47.54, 158.41 17.7, 176.45 -0.31 M109.73 79.49 C133.83 52.06, 156.64 24.99, 178.62 0.24 M109.73 79.49 C133.53 51.98, 158.73 23.97, 178.62 0.24 M112.55 79.29 C140.15 50.58, 163.22 17.92, 181.44 0.05 M112.55 79.29 C128.93 62.68, 142.87 44.92, 181.44 0.05 M115.38 79.09 C129.8 62.64, 146.14 44.5, 184.26 -0.15 M115.38 79.09 C132.92 57.77, 151.64 37.84, 184.26 -0.15 M118.2 78.9 C141.47 52.38, 163.34 25.38, 186.43 0.41 M118.2 78.9 C137.4 57.23, 156.76 34.24, 186.43 0.41 M120.36 79.45 C140.5 56.55, 160.65 31.82, 189.25 0.21 M120.36 79.45 C140.03 56.18, 159.95 32.98, 189.25 0.21 M123.18 79.26 C147.73 50.06, 177.1 18.53, 192.07 0.01 M123.18 79.26 C141.29 58.88, 157.01 40.81, 192.07 0.01 M126.01 79.06 C147.96 52.14, 169.41 29.89, 194.89 -0.18 M126.01 79.06 C150.51 51.83, 173.87 23.93, 194.89 -0.18 M128.83 78.86 C154.46 50.35, 180.39 21.76, 197.06 0.37 M128.83 78.86 C144.98 58.83, 164.1 38.36, 197.06 0.37 M130.99 79.42 C154.57 52.66, 177.57 26.65, 199.88 0.18 M130.99 79.42 C150.69 55.62, 171.84 34.68, 199.88 0.18 M133.81 79.22 C151.85 55.83, 172.25 33.24, 202.04 0.73 M133.81 79.22 C148.94 60.24, 166.57 42.2, 202.04 0.73 M136.64 79.03 C156.13 57.45, 177.46 34.15, 204.21 1.29 M136.64 79.03 C156.77 55.44, 176.98 32.16, 204.21 1.29 M139.46 78.83 C159.39 57.43, 178.43 36.99, 206.38 1.85 M139.46 78.83 C164.66 51.12, 189.02 23.43, 206.38 1.85 M141.62 79.39 C165.23 53.71, 188.51 23.57, 207.88 3.16 M141.62 79.39 C159.8 57, 177.92 37.21, 207.88 3.16 M144.44 79.19 C170.11 50.86, 192.02 24.65, 210.05 3.72 M144.44 79.19 C157.71 63.56, 171.18 47.82, 210.05 3.72 M147.27 78.99 C168.84 53.61, 188.86 30.51, 211.56 5.03 M147.27 78.99 C163.13 59.74, 180.44 42.4, 211.56 5.03 M150.09 78.79 C172.58 51.38, 192.94 26.88, 213.07 6.34 M150.09 78.79 C175.33 50.31, 200.43 21.76, 213.07 6.34 M152.25 79.35 C174.92 53.87, 200.12 24.5, 213.92 8.41 M152.25 79.35 C174.81 53.87, 195.55 27.32, 213.92 8.41 M155.07 79.15 C175.58 56.02, 194.18 31.86, 215.43 9.72 M155.07 79.15 C176.15 54.8, 196.34 30.98, 215.43 9.72 M157.9 78.96 C171.48 64.04, 186.08 47.04, 216.29 11.79 M157.9 78.96 C172.59 61.64, 189.25 42.9, 216.29 11.79 M160.06 79.51 C179.36 58.14, 200.32 36.84, 217.14 13.85 M160.06 79.51 C178.29 58.44, 195.72 37.53, 217.14 13.85 M162.88 79.32 C177.87 62.17, 194.29 40.16, 217.99 15.92 M162.88 79.32 C184.55 55.65, 205.08 32.46, 217.99 15.92 M165.7 79.12 C184.61 58.25, 201.86 41.44, 217.53 19.5 M165.7 79.12 C184.87 56.6, 204.48 35.24, 217.53 19.5 M168.53 78.92 C183.14 61.29, 196.82 45.16, 217.73 22.32 M168.53 78.92 C187.32 56.95, 204.79 36.39, 217.73 22.32 M170.69 79.48 C181.59 67.74, 191.43 55.8, 217.93 25.14 M170.69 79.48 C181.74 67.14, 192.15 54.26, 217.93 25.14 M173.51 79.28 C191.65 58.97, 205.44 43.52, 218.13 27.96 M173.51 79.28 C186.74 62.91, 200.46 46.93, 218.13 27.96 M176.33 79.08 C192.32 60.48, 210.31 40.21, 217.67 31.54 M176.33 79.08 C192.03 59.7, 207.64 40.56, 217.67 31.54 M179.16 78.89 C189.84 69.54, 196.85 57.51, 217.86 34.36 M179.16 78.89 C191.9 63.82, 206.31 50.04, 217.86 34.36 M181.32 79.44 C189.75 71.73, 198.26 59.81, 218.06 37.18 M181.32 79.44 C188.46 71.42, 196.06 62.84, 218.06 37.18 M184.14 79.25 C196.88 64.63, 206.41 52.55, 218.26 40 M184.14 79.25 C194.9 65.79, 205.73 53.03, 218.26 40 M186.96 79.05 C197.5 67.2, 209.2 50.61, 217.8 43.58 M186.96 79.05 C193.57 71.6, 199.78 62.98, 217.8 43.58 M189.79 78.85 C197.12 71.86, 204.96 64.54, 218 46.4 M189.79 78.85 C196.46 69.51, 204.1 61.69, 218 46.4 M191.95 79.41 C196.55 72.83, 206.89 63.24, 218.19 49.22 M191.95 79.41 C197.26 73.33, 202.88 64.86, 218.19 49.22 M194.77 79.21 C199.29 74.05, 205.94 66.7, 218.39 52.04 M194.77 79.21 C201.61 71.22, 208.64 63.22, 218.39 52.04 M197.59 79.02 C203.76 69.41, 215.21 61.81, 217.93 55.62 M197.59 79.02 C201.68 74.35, 206.7 68.88, 217.93 55.62 M201.07 78.06 C206.08 70.73, 212.77 66.49, 217.47 59.2 M201.07 78.06 C204.85 74.52, 209.69 69.91, 217.47 59.2 M203.89 77.87 C207.22 73.56, 208.58 70.94, 217.02 62.77 M203.89 77.87 C206.92 74.23, 210.15 69.43, 217.02 62.77 M207.37 76.91 C209.08 73.55, 211.34 70.66, 215.9 67.1 M207.37 76.91 C210.21 74.62, 211.5 70.98, 215.9 67.1" stroke="#b2f2bb" stroke-width="0.25" fill="none"></path><path d="M19.75 0 M19.75 0 C90.46 0.03, 160.83 0.59, 197.25 0 M19.75 0 C84.18 -1.17, 148.43 -0.94, 197.25 0 M197.25 0 C211.59 0.57, 217.97 8.32, 217 19.75 M197.25 0 C209.59 0.89, 218.97 5.83, 217 19.75 M217 19.75 C218.59 32.71, 215.41 51, 217 59.25 M217 19.75 C217.61 29.98, 216.69 38.61, 217 59.25 M217 59.25 C216.27 74.05, 210.47 78.65, 197.25 79 M217 59.25 C217.88 73.13, 212.04 78.45, 197.25 79 M197.25 79 C131.69 82.03, 67.04 79.21, 19.75 79 M197.25 79 C130.13 78.01, 62.22 77.27, 19.75 79 M19.75 79 C6.83 77.44, -0.99 73.15, 0 59.25 M19.75 79 C7.44 79.01, -0.32 70.87, 0 59.25 M0 59.25 C-0.3 44.8, 0 30.63, 0 19.75 M0 59.25 C-0.65 43.96, 0.17 28.36, 0 19.75 M0 19.75 C-0.05 5.28, 7.3 -0.16, 19.75 0 M0 19.75 C-0.27 6.94, 8.29 0.04, 19.75 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1940.8436670623366 1735.06763419055) rotate(0 25.109970092773438 12.5)"><text x="25.109970092773438" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">plugin</text></g><g mask="url(#mask-iZyFQP6igOtELXCKu_6Fw)" stroke-linecap="round"><g transform="translate(2172.4697410192366 701.0457387953638) rotate(0 -59.54647337216102 502.95415536226983)"><path d="M0.88 0.43 C5.38 77.5, 53.37 294.97, 27.27 462.41 C1.17 629.85, -125.06 914.59, -155.73 1005.05 M-0.11 -0.38 C4.72 76.85, 55.4 295.98, 29.44 463.76 C3.47 631.54, -124.91 915.88, -155.89 1006.29" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2172.4697410192366 701.0457387953638) rotate(0 -59.54647337216102 502.95415536226983)"><path d="M-154.5 974.57 C-156.04 987.91, -155.88 995.85, -155.97 1006.62 M-156.1 977.19 C-156.32 984.29, -154.64 993.24, -155.11 1006.41" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2172.4697410192366 701.0457387953638) rotate(0 -59.54647337216102 502.95415536226983)"><path d="M-135.38 982.01 C-143.08 993.09, -149.1 998.63, -155.97 1006.62 M-136.97 984.63 C-142.45 989.57, -146.01 996.48, -155.11 1006.41" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask id="mask-iZyFQP6igOtELXCKu_6Fw"><rect x="0" y="0" fill="#fff" width="2457.6660261027505" height="1806.2343916274494"></rect><rect x="2186.127484619935" y="1151.3149977346666" fill="#000" width="29.299972534179688" height="25" opacity="1"></rect></mask><g transform="translate(2186.127484619935 1151.3149977346666) rotate(0 -73.20421697285929 52.684896422967086)"><text x="14.649986267089844" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">run</text></g><g stroke-linecap="round" transform="translate(1536.1493435076768 1350.0358194070259) rotate(0 128 34.5)"><path d="M4.59 3.99 C4.59 3.99, 4.59 3.99, 4.59 3.99 M4.59 3.99 C4.59 3.99, 4.59 3.99, 4.59 3.99 M1.51 10.59 C4.98 7.93, 8.05 2.57, 10.04 0.78 M1.51 10.59 C4.79 7.52, 8.09 3.79, 10.04 0.78 M1.05 14.16 C4.33 10.82, 7.85 7.92, 12.86 0.58 M1.05 14.16 C5.59 8.54, 10.41 3.74, 12.86 0.58 M1.25 16.98 C4.3 14.36, 9.65 9.13, 15.68 0.38 M1.25 16.98 C6.13 10.54, 11.56 4.55, 15.68 0.38 M1.44 19.81 C6.39 14.44, 12.04 10.28, 17.85 0.94 M1.44 19.81 C4.99 15.46, 7.54 11.06, 17.85 0.94 M0.99 23.38 C10.64 15.66, 16.77 6.81, 20.67 0.74 M0.99 23.38 C7.02 15.93, 11.98 9.29, 20.67 0.74 M1.18 26.2 C4.12 21.6, 11.19 13.7, 23.49 0.54 M1.18 26.2 C9.57 16.5, 17.29 7.75, 23.49 0.54 M1.38 29.03 C8.9 19.22, 16.14 9.95, 26.31 0.35 M1.38 29.03 C10.94 17.5, 19.86 7.9, 26.31 0.35 M0.92 32.6 C9.11 21.88, 18.13 14.28, 28.48 0.9 M0.92 32.6 C11.81 20.53, 22.86 6.88, 28.48 0.9 M1.12 35.42 C10.67 22.29, 23.77 10.73, 31.3 0.71 M1.12 35.42 C7.93 28.39, 14.23 21.54, 31.3 0.71 M1.32 38.24 C11.5 26.99, 22.48 17.67, 34.12 0.51 M1.32 38.24 C7.55 30.85, 15.79 22.58, 34.12 0.51 M1.51 41.07 C10.36 28.74, 20.64 20.14, 36.28 1.07 M1.51 41.07 C11.76 30.18, 22.41 18.02, 36.28 1.07 M1.05 44.64 C16.02 31.14, 28.79 14.09, 39.11 0.87 M1.05 44.64 C10.02 35.32, 17.28 26.7, 39.11 0.87 M1.25 47.46 C13.5 34.58, 25.06 20.67, 41.93 0.67 M1.25 47.46 C15.65 31.53, 29.25 15.34, 41.93 0.67 M1.45 50.29 C12.29 37.81, 19.69 27.83, 44.75 0.47 M1.45 50.29 C14.38 33.91, 29.34 18.15, 44.75 0.47 M0.99 53.86 C17.29 33.16, 36.57 11.83, 46.91 1.03 M0.99 53.86 C12.15 40.73, 24.43 27.19, 46.91 1.03 M1.19 56.68 C18.71 35.97, 34.48 16.83, 49.74 0.83 M1.19 56.68 C11.12 46.65, 21.27 34.15, 49.74 0.83 M2.04 58.75 C17.21 45.37, 29.07 26.76, 52.56 0.64 M2.04 58.75 C20.68 38.4, 37.87 17.17, 52.56 0.64 M2.89 60.82 C20.71 39.37, 44.11 16.21, 55.38 0.44 M2.89 60.82 C13.99 45.97, 25.86 33.44, 55.38 0.44 M3.75 62.88 C23.08 43.02, 39.17 19.91, 57.54 1 M3.75 62.88 C14.24 49.76, 25.43 36.67, 57.54 1 M5.26 64.2 C26.24 41.08, 42.39 21.37, 60.37 0.8 M5.26 64.2 C19.73 46.48, 35.6 29.29, 60.37 0.8 M6.77 65.51 C26.78 41.46, 47.64 15.99, 63.19 0.6 M6.77 65.51 C19.82 53.03, 31.08 37.72, 63.19 0.6 M8.28 66.82 C29.93 41.68, 51.07 17.38, 66.01 0.41 M8.28 66.82 C29.63 41.64, 52.2 16.22, 66.01 0.41 M9.79 68.13 C26.29 49.61, 39.87 34.08, 68.17 0.96 M9.79 68.13 C29.45 45.07, 50.56 21.04, 68.17 0.96 M11.95 68.69 C34.34 44.64, 55.55 18.11, 71 0.77 M11.95 68.69 C26.86 50.99, 43.07 33.69, 71 0.77 M14.77 68.49 C37.44 39.69, 63.91 15.12, 73.82 0.57 M14.77 68.49 C30.34 52.64, 44.52 35.03, 73.82 0.57 M16.94 69.05 C34.99 51.55, 50.14 31.28, 76.64 0.37 M16.94 69.05 C41.04 42.11, 63.85 14.94, 76.64 0.37 M18.45 70.36 C39.13 45.74, 58.98 25.26, 78.8 0.93 M18.45 70.36 C40.66 45.71, 62.31 20.23, 78.8 0.93 M21.27 70.16 C38.07 48.68, 55.48 27.53, 81.63 0.73 M21.27 70.16 C42.79 45.94, 64.63 20.85, 81.63 0.73 M24.09 69.97 C45.93 47.87, 67.35 23.14, 84.45 0.53 M24.09 69.97 C44.82 45.05, 67.45 21.07, 84.45 0.53 M26.26 70.52 C39.64 55.7, 57.66 38.78, 87.27 0.34 M26.26 70.52 C40.23 52.88, 56.54 35.69, 87.27 0.34 M29.08 70.33 C43.06 54.28, 57.04 40.42, 89.44 0.89 M29.08 70.33 C48.97 48.09, 68.34 27.65, 89.44 0.89 M31.9 70.13 C52.66 47.11, 71.11 24.4, 92.26 0.7 M31.9 70.13 C50.03 48.96, 67.59 27.85, 92.26 0.7 M34.72 69.93 C49.42 50.6, 67.1 32.15, 95.08 0.5 M34.72 69.93 C56.82 44.49, 76.92 20.13, 95.08 0.5 M36.89 70.49 C50.87 54.36, 63.32 41.63, 97.24 1.06 M36.89 70.49 C54.48 52.04, 70.35 32.92, 97.24 1.06 M39.71 70.29 C63.14 42.23, 85.32 15.87, 100.07 0.86 M39.71 70.29 C52.2 55.08, 65.4 41.1, 100.07 0.86 M42.53 70.1 C56.8 53.96, 69.82 39.81, 102.89 0.66 M42.53 70.1 C53.58 55.62, 66.81 41.52, 102.89 0.66 M45.35 69.9 C65.46 50.92, 80.3 28.5, 105.71 0.46 M45.35 69.9 C59.68 54.74, 73.3 39.04, 105.71 0.46 M47.52 70.46 C60.99 54.47, 77.88 36.01, 107.87 1.02 M47.52 70.46 C66.27 50.27, 84.46 27.7, 107.87 1.02 M50.34 70.26 C74.65 47.18, 95.47 21.02, 110.7 0.82 M50.34 70.26 C74.58 43.16, 97.07 14.48, 110.7 0.82 M53.16 70.06 C67.34 53.7, 83.81 36.58, 113.52 0.63 M53.16 70.06 C71.4 50.19, 88.8 29.35, 113.52 0.63 M55.98 69.86 C81.3 42.84, 104.18 16.53, 116.34 0.43 M55.98 69.86 C74.19 49.52, 93.26 28.69, 116.34 0.43 M58.15 70.42 C81.01 42.01, 104.98 14.59, 118.5 0.99 M58.15 70.42 C81.92 43.5, 103.99 17.02, 118.5 0.99 M60.97 70.22 C82.09 46.28, 103.56 19.74, 121.33 0.79 M60.97 70.22 C83.38 42.44, 106.65 16.33, 121.33 0.79 M63.79 70.03 C84.53 47.44, 99.58 26.5, 124.15 0.59 M63.79 70.03 C85.65 44.35, 107.14 19.49, 124.15 0.59 M66.61 69.83 C86.44 45.53, 109.22 20.14, 126.97 0.4 M66.61 69.83 C84.24 51.47, 99.27 32.01, 126.97 0.4 M68.78 70.39 C95.1 42.96, 116.47 14.18, 129.13 0.95 M68.78 70.39 C86.91 51.87, 102.33 31.2, 129.13 0.95 M71.6 70.19 C85.54 57.31, 96.15 40.8, 131.96 0.76 M71.6 70.19 C84.4 53.45, 99.21 37.44, 131.96 0.76 M74.42 69.99 C96.41 45.5, 114.42 23.07, 134.78 0.56 M74.42 69.99 C96.49 44.78, 118.43 20.69, 134.78 0.56 M76.58 70.55 C93.76 53.21, 108.52 32.75, 137.6 0.36 M76.58 70.55 C97.86 46.57, 119.42 20.79, 137.6 0.36 M79.41 70.35 C92.28 56.68, 104.14 39.59, 139.76 0.92 M79.41 70.35 C98.74 49.92, 116.05 28.83, 139.76 0.92 M82.23 70.15 C106.3 41.99, 128.22 15.79, 142.59 0.72 M82.23 70.15 C101.52 46.2, 123 21.74, 142.59 0.72 M85.05 69.96 C107.98 47.43, 129.6 21.59, 145.41 0.52 M85.05 69.96 C107.88 44.76, 130.15 17.87, 145.41 0.52 M87.22 70.51 C99.72 53.75, 113.22 42.57, 148.23 0.33 M87.22 70.51 C107.51 47.17, 128.35 25.28, 148.23 0.33 M90.04 70.32 C105.67 51.14, 123.04 29, 150.39 0.88 M90.04 70.32 C109.86 47.65, 129.55 25.15, 150.39 0.88 M92.86 70.12 C110.18 53, 121.89 35.13, 153.22 0.69 M92.86 70.12 C108.08 51.54, 126.42 33.06, 153.22 0.69 M95.68 69.92 C118.6 42.02, 143.78 15.08, 156.04 0.49 M95.68 69.92 C111.03 53.5, 125.92 35.23, 156.04 0.49 M97.85 70.48 C121.86 46.32, 143.05 17.09, 158.2 1.05 M97.85 70.48 C117.31 47.57, 137.96 22.77, 158.2 1.05 M100.67 70.28 C118.29 51.92, 132.83 30.3, 161.02 0.85 M100.67 70.28 C122.12 46.03, 143.65 20, 161.02 0.85 M103.49 70.09 C125.83 43.72, 149.52 19.12, 163.85 0.65 M103.49 70.09 C124.22 46.81, 143.96 23.62, 163.85 0.65 M106.31 69.89 C129.95 45.16, 148.86 18.1, 166.67 0.46 M106.31 69.89 C128.92 43.84, 152.35 16.44, 166.67 0.46 M108.48 70.45 C129.88 42.86, 152.81 18.61, 168.83 1.01 M108.48 70.45 C123.62 53.27, 139.76 34.67, 168.83 1.01 M111.3 70.25 C130.44 47.66, 149.63 27.14, 171.65 0.82 M111.3 70.25 C135.71 42.89, 158.51 17.52, 171.65 0.82 M114.12 70.05 C135.86 41.8, 160.14 16.31, 174.48 0.62 M114.12 70.05 C137.09 43.62, 159.92 19.19, 174.48 0.62 M116.94 69.85 C130.03 56.38, 142.12 41.38, 177.3 0.42 M116.94 69.85 C136.97 46.23, 155.15 23.76, 177.3 0.42 M119.11 70.41 C135.58 51.25, 150.25 34.53, 179.46 0.98 M119.11 70.41 C138.71 48.79, 157.57 25.86, 179.46 0.98 M121.93 70.21 C144.56 43.28, 170.65 17.92, 182.28 0.78 M121.93 70.21 C140.57 48.17, 159.01 27.45, 182.28 0.78 M124.75 70.02 C138.75 54.64, 149.69 41.62, 185.11 0.58 M124.75 70.02 C149.64 41.78, 173.49 15.26, 185.11 0.58 M127.57 69.82 C148.01 45.05, 169.13 22.23, 187.93 0.39 M127.57 69.82 C149.2 46.47, 169.95 20.77, 187.93 0.39 M129.74 70.38 C147.99 45.69, 171.5 25.1, 190.09 0.94 M129.74 70.38 C146.79 49.44, 166.27 28.97, 190.09 0.94 M132.56 70.18 C150.74 49.63, 167.71 29.37, 192.91 0.75 M132.56 70.18 C145.4 56.36, 157.87 41.65, 192.91 0.75 M135.38 69.98 C156.75 47.59, 177.15 22.82, 195.74 0.55 M135.38 69.98 C156.74 45.13, 178.76 18.88, 195.74 0.55 M137.54 70.54 C149.13 56.62, 166.36 39.85, 198.56 0.35 M137.54 70.54 C156.98 47.89, 176.74 25.53, 198.56 0.35 M140.37 70.34 C162.25 46.92, 185.66 19.26, 200.72 0.91 M140.37 70.34 C161.37 48.04, 181.15 23.87, 200.72 0.91 M143.19 70.15 C163.02 46.64, 183.69 22.03, 203.54 0.71 M143.19 70.15 C160.56 50.54, 178.49 30.18, 203.54 0.71 M146.01 69.95 C158.06 57.35, 170.02 42.85, 206.37 0.51 M146.01 69.95 C169.84 43.34, 191.44 16.49, 206.37 0.51 M148.17 70.51 C165.54 47.09, 187.97 27.75, 208.53 1.07 M148.17 70.51 C161.24 54.29, 175.29 39.18, 208.53 1.07 M151 70.31 C174.73 43.39, 198.12 16.31, 211.35 0.87 M151 70.31 C168.24 49.36, 186.68 27.72, 211.35 0.87 M153.82 70.11 C166.77 54.69, 183.66 39.11, 214.17 0.68 M153.82 70.11 C172.95 46.06, 193.67 24.57, 214.17 0.68 M156.64 69.91 C169.61 56.03, 181.9 43.17, 217 0.48 M156.64 69.91 C176.71 48.81, 194.46 28.62, 217 0.48 M158.8 70.47 C183.58 40.94, 205.93 14.99, 219.16 1.04 M158.8 70.47 C182.23 43.03, 205.09 16.63, 219.16 1.04 M161.63 70.27 C175.95 51.47, 191.37 33.91, 221.98 0.84 M161.63 70.27 C175.96 54.75, 189.14 37.86, 221.98 0.84 M164.45 70.08 C187.34 45.76, 207.86 22.27, 224.8 0.64 M164.45 70.08 C179.9 53.22, 195.68 34.95, 224.8 0.64 M167.27 69.88 C181.83 51.02, 196.73 34.7, 227.63 0.45 M167.27 69.88 C188.36 46.23, 207.16 23.81, 227.63 0.45 M169.43 70.44 C185.43 52.99, 202.46 34.15, 229.79 1 M169.43 70.44 C188.91 48.08, 210.17 25.38, 229.79 1 M172.26 70.24 C188.28 50.61, 206.93 33.52, 232.61 0.81 M172.26 70.24 C190.13 48.55, 208.94 28.2, 232.61 0.81 M175.08 70.04 C193.39 45.77, 214.6 23.19, 235.43 0.61 M175.08 70.04 C194.81 47.84, 213.98 24.61, 235.43 0.61 M177.9 69.84 C195.41 49.34, 211.67 32.52, 238.26 0.41 M177.9 69.84 C190.99 52.56, 206.85 36.07, 238.26 0.41 M180.06 70.4 C201.2 44.23, 223.13 22.73, 240.42 0.97 M180.06 70.4 C196.43 52.09, 212.26 33.32, 240.42 0.97 M182.89 70.2 C205.33 41.52, 228.13 14.47, 242.59 1.53 M182.89 70.2 C199.32 51.54, 214.09 34.53, 242.59 1.53 M185.71 70.01 C198.55 56.86, 214.02 38.86, 244.75 2.08 M185.71 70.01 C207.4 46.52, 226.78 21.41, 244.75 2.08 M188.53 69.81 C205.16 51.62, 220.72 34.27, 247.57 1.89 M188.53 69.81 C206.31 48.06, 225.18 27.03, 247.57 1.89 M190.69 70.37 C211.74 44.13, 234.17 19.6, 249.08 3.2 M190.69 70.37 C204.77 54.96, 216.34 39.46, 249.08 3.2 M193.52 70.17 C208.6 52.11, 221.33 35.38, 250.59 4.51 M193.52 70.17 C214.8 46.74, 236.01 21.28, 250.59 4.51 M196.34 69.97 C215.63 45.51, 237.98 21.88, 252.1 5.82 M196.34 69.97 C214.59 51.51, 230.89 29.98, 252.1 5.82 M198.5 70.53 C214.18 52.45, 225.19 36.01, 253.61 7.13 M198.5 70.53 C210.16 57.59, 221.19 43.46, 253.61 7.13 M201.32 70.33 C217.19 52.07, 233.1 35.22, 254.47 9.2 M201.32 70.33 C218.36 51.83, 236.39 30.26, 254.47 9.2 M204.15 70.14 C221.12 51.9, 236.65 34.63, 255.32 11.27 M204.15 70.14 C217.51 55.24, 231.92 38.51, 255.32 11.27 M206.97 69.94 C225.99 52.09, 242.83 29.42, 256.17 13.33 M206.97 69.94 C221.94 50.95, 238.21 33.17, 256.17 13.33 M209.13 70.5 C225.84 51.06, 240.27 32.6, 255.71 16.91 M209.13 70.5 C224.38 51.57, 240.43 32.8, 255.71 16.91 M211.95 70.3 C225.99 53.44, 236.03 41.58, 255.91 19.73 M211.95 70.3 C223.91 57.16, 235.84 44.02, 255.91 19.73 M214.78 70.1 C224.83 56.71, 236.16 46.32, 256.11 22.55 M214.78 70.1 C227.76 54.51, 242.56 38.18, 256.11 22.55 M217.6 69.9 C234.17 51.7, 245.35 38.56, 255.65 26.13 M217.6 69.9 C227.12 59.21, 235.7 47.04, 255.65 26.13 M219.76 70.46 C228.58 60.29, 239.05 50, 255.85 28.95 M219.76 70.46 C230.36 58.29, 240.15 46.77, 255.85 28.95 M222.58 70.26 C228.2 61.54, 238.98 50.97, 256.04 31.77 M222.58 70.26 C231.68 59.8, 242.19 48.35, 256.04 31.77 M225.41 70.07 C233.5 59.82, 242.46 49.62, 255.58 35.35 M225.41 70.07 C234.82 59.64, 242.69 49.73, 255.58 35.35 M228.23 69.87 C236.14 59.94, 246.55 48.32, 255.78 38.17 M228.23 69.87 C237.09 60.61, 244.41 50.28, 255.78 38.17 M230.39 70.43 C240.96 57.59, 251.89 47.54, 255.32 41.75 M230.39 70.43 C238.04 61.18, 244.83 54.06, 255.32 41.75 M233.21 70.23 C240.43 61.5, 251.91 49.81, 255.52 44.57 M233.21 70.23 C242.24 59.36, 251.27 49.98, 255.52 44.57 M236.04 70.03 C242.05 63.02, 253.36 51.63, 255.72 47.39 M236.04 70.03 C239.95 64.97, 244.86 59.4, 255.72 47.39 M239.51 69.08 C244.18 62.7, 250.66 56.1, 255.26 50.97 M239.51 69.08 C244.6 64.03, 248.16 58.04, 255.26 50.97 M242.34 68.88 C247.19 63.73, 250.92 55.87, 255.46 53.79 M242.34 68.88 C246.36 64.45, 250.13 60.92, 255.46 53.79 M245.81 67.93 C249.47 65.48, 252.55 61.5, 255 57.36 M245.81 67.93 C249.16 64.84, 251.06 61.03, 255 57.36" stroke="#ffc9c9" stroke-width="0.25" fill="none"></path><path d="M17.25 0 M17.25 0 C70.2 2.59, 121.35 1.68, 238.75 0 M17.25 0 C104.04 1.48, 191.28 1.52, 238.75 0 M238.75 0 C252.06 -1.78, 256.83 6.83, 256 17.25 M238.75 0 C248.32 1.94, 255.39 5.29, 256 17.25 M256 17.25 C256.96 24.57, 255.92 31.96, 256 51.75 M256 17.25 C256.49 30.06, 255.64 42.58, 256 51.75 M256 51.75 C255.58 62.93, 251.95 70.89, 238.75 69 M256 51.75 C253.96 64.29, 249.97 67.38, 238.75 69 M238.75 69 C159.73 71.08, 80.59 67.88, 17.25 69 M238.75 69 C190.21 67.98, 140.78 67.96, 17.25 69 M17.25 69 C7.4 70.68, 0.69 62.57, 0 51.75 M17.25 69 C6.59 69.87, 1.84 63.59, 0 51.75 M0 51.75 C1.86 42.72, 0.64 34.06, 0 17.25 M0 51.75 C-0.36 39.68, 0.34 25.54, 0 17.25 M0 17.25 C0.13 5.89, 5.73 1.45, 17.25 0 M0 17.25 C0.74 6.74, 4.74 -0.9, 17.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1591.7494182757432 1372.0358194070259) rotate(0 72.3999252319336 12.5)"><text x="72.3999252319336" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">runtime service</text></g><g mask="url(#mask-55XNUYyrHcmWaj7sqrS-g)" stroke-linecap="round"><g transform="translate(1863.2975344788365 1700.856714752801) rotate(0 -46.79148766456939 -136.6762116912064)"><path d="M-1.04 1.07 C-16.74 -44.55, -79.04 -228.65, -94.19 -274.42 M0.61 0.58 C-14.74 -44.79, -76.6 -227, -92.06 -273.04" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1863.2975344788365 1700.856714752801) rotate(0 -46.79148766456939 -136.6762116912064)"><path d="M-71.54 -251.45 C-79.3 -254.93, -83.51 -262.33, -91.83 -271.06 M-72.43 -248.9 C-78.36 -255.29, -83.53 -262.22, -92.52 -273.48" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1863.2975344788365 1700.856714752801) rotate(0 -46.79148766456939 -136.6762116912064)"><path d="M-90.98 -244.88 C-92.85 -250.17, -91.2 -259.55, -91.83 -271.06 M-91.87 -242.33 C-92.18 -250.75, -91.74 -259.57, -92.52 -273.48" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask id="mask-55XNUYyrHcmWaj7sqrS-g"><rect x="0" y="0" fill="#fff" width="2056.3909212490685" height="2074.9851080294447"></rect><rect x="1779.7108859545606" y="1551.2925181144792" fill="#000" width="74.07991027832031" height="25" opacity="1"></rect></mask><g transform="translate(1779.7108859545601 1551.2925181144792) rotate(0 36.79516085970687 12.887984947115456)"><text x="37.039955139160156" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">registry</text></g><g stroke-linecap="round" transform="translate(2404.4809298968185 1354.993794984509) rotate(0 108.5 39.5)"><path d="M5.26 4.57 C5.26 4.57, 5.26 4.57, 5.26 4.57 M5.26 4.57 C5.26 4.57, 5.26 4.57, 5.26 4.57 M2.17 11.16 C5.77 7.58, 7.66 4.43, 10.7 1.35 M2.17 11.16 C5.77 7.55, 8.77 4.66, 10.7 1.35 M1.06 15.5 C5.37 10.06, 8.26 5.65, 12.21 2.67 M1.06 15.5 C4.98 11.09, 8.85 7, 12.21 2.67 M0.6 19.07 C6.1 12.6, 11.68 4.04, 15.69 1.71 M0.6 19.07 C3.5 16.18, 7.68 11.5, 15.69 1.71 M0.8 21.89 C5.92 13.84, 13.64 3.47, 19.82 0.01 M0.8 21.89 C5.83 15.02, 11.67 10.24, 19.82 0.01 M1 24.72 C7.68 19.87, 10.04 13.73, 22.65 -0.19 M1 24.72 C7.56 17.28, 13.3 9.71, 22.65 -0.19 M1.19 27.54 C8.34 22.27, 10.31 15.6, 24.81 0.37 M1.19 27.54 C8.1 18.96, 15.3 10.78, 24.81 0.37 M0.73 31.11 C11.21 20.01, 20.76 9.41, 27.63 0.17 M0.73 31.11 C7.52 23.51, 12.46 17.67, 27.63 0.17 M0.93 33.93 C11 23.47, 21.14 12.29, 30.45 -0.03 M0.93 33.93 C10.64 22.42, 21.32 11.22, 30.45 -0.03 M1.13 36.76 C12.71 20.94, 27.31 9.17, 33.28 -0.22 M1.13 36.76 C9.09 26.36, 17.83 16.69, 33.28 -0.22 M0.67 40.33 C10.62 25.91, 25.99 12.67, 35.44 0.33 M0.67 40.33 C14.02 26.18, 27.65 10.88, 35.44 0.33 M0.87 43.15 C11.89 32.39, 22.59 18.51, 38.26 0.14 M0.87 43.15 C13.21 29.59, 25.45 15.55, 38.26 0.14 M1.06 45.98 C15.43 31.06, 28.83 14.59, 41.08 -0.06 M1.06 45.98 C15.16 31.43, 27.22 15.1, 41.08 -0.06 M0.61 49.55 C13.38 31.72, 27.61 17.43, 43.91 -0.26 M0.61 49.55 C12.54 34.55, 24.8 21.05, 43.91 -0.26 M0.8 52.37 C11.63 38.62, 23.57 27.17, 46.07 0.3 M0.8 52.37 C11.22 39.81, 21.9 26.47, 46.07 0.3 M1 55.19 C14.33 38.25, 32.3 22.15, 48.89 0.1 M1 55.19 C15.54 40.19, 30.17 23.44, 48.89 0.1 M1.2 58.02 C14.04 46.25, 24.5 32.17, 51.71 -0.1 M1.2 58.02 C15.85 40.65, 30.14 26.08, 51.71 -0.1 M0.74 61.59 C21.68 38.57, 37.24 17.04, 54.54 -0.29 M0.74 61.59 C15.86 44.12, 32.56 24.88, 54.54 -0.29 M0.94 64.41 C14.94 48.85, 26.85 34.14, 56.7 0.26 M0.94 64.41 C19.48 43.53, 38.16 20.11, 56.7 0.26 M1.79 66.48 C21.32 42.77, 40.91 21.3, 59.52 0.07 M1.79 66.48 C16.76 48.1, 32.22 30.27, 59.52 0.07 M2.64 68.55 C17.09 53.06, 35.14 34.42, 62.34 -0.13 M2.64 68.55 C15.97 52.79, 29.19 37.02, 62.34 -0.13 M2.84 71.37 C19.13 51.04, 35.21 33.69, 65.17 -0.33 M2.84 71.37 C16.76 57.68, 28.25 42.33, 65.17 -0.33 M4.35 72.68 C25.22 46.69, 47.98 23.1, 67.33 0.23 M4.35 72.68 C21.85 53.13, 39.02 33.11, 67.33 0.23 M5.86 73.99 C21.42 54.89, 37.11 38.62, 70.15 0.03 M5.86 73.99 C27.1 49.8, 46.43 27.2, 70.15 0.03 M7.37 75.31 C29.48 49.57, 56.13 21.22, 72.97 -0.17 M7.37 75.31 C28.71 50.62, 52.12 24.53, 72.97 -0.17 M8.88 76.62 C27.92 54.39, 49.17 29.19, 75.14 0.39 M8.88 76.62 C26.42 56.02, 43.82 35.15, 75.14 0.39 M11.04 77.18 C36.28 48.02, 62.46 18.42, 77.96 0.19 M11.04 77.18 C35.92 46.6, 61.95 16.6, 77.96 0.19 M12.55 78.49 C28.78 59.06, 47.29 39.88, 80.78 0 M12.55 78.49 C31.88 55.62, 53.86 32.37, 80.78 0 M14.72 79.04 C34.18 52.86, 55.87 30.45, 83.6 -0.2 M14.72 79.04 C36.38 53.47, 59.92 29.88, 83.6 -0.2 M16.88 79.6 C40.93 48.19, 66.98 20.91, 85.77 0.36 M16.88 79.6 C38.97 56.7, 58.46 33.5, 85.77 0.36 M19.7 79.4 C38.94 56.37, 60.4 32.49, 88.59 0.16 M19.7 79.4 C45.36 51.51, 69.24 22.39, 88.59 0.16 M22.53 79.21 C44.18 55.92, 63.51 31.22, 91.41 -0.04 M22.53 79.21 C49.36 48.84, 74.17 19.91, 91.41 -0.04 M25.35 79.01 C48.44 53.65, 70.18 24.11, 94.23 -0.23 M25.35 79.01 C42.91 59.48, 59.88 39.28, 94.23 -0.23 M28.17 78.81 C54.37 51.78, 79.92 21.93, 96.4 0.32 M28.17 78.81 C44.69 60.14, 60.35 40.52, 96.4 0.32 M30.33 79.37 C44.83 62.85, 56.97 48.38, 99.22 0.13 M30.33 79.37 C47.95 60.1, 64.89 40.17, 99.22 0.13 M33.16 79.17 C56.46 52.48, 76.91 29.62, 102.04 -0.07 M33.16 79.17 C51.34 58.76, 71.4 35.53, 102.04 -0.07 M35.98 78.98 C57.6 54.21, 81.23 28.72, 104.86 -0.27 M35.98 78.98 C61.08 50.82, 84.82 24.45, 104.86 -0.27 M38.8 78.78 C59.47 53.78, 83.96 28.03, 107.03 0.29 M38.8 78.78 C52.89 63.16, 68.99 45.53, 107.03 0.29 M40.97 79.34 C65.94 52.62, 89.86 22.34, 109.85 0.09 M40.97 79.34 C58.19 59.95, 75.35 41.43, 109.85 0.09 M43.79 79.14 C69.2 53.4, 92.38 26.29, 112.67 -0.11 M43.79 79.14 C62.51 58.06, 81.66 35.54, 112.67 -0.11 M46.61 78.94 C70.54 48.45, 98.8 20.02, 115.49 -0.3 M46.61 78.94 C70.98 48.54, 96.72 18.92, 115.49 -0.3 M48.77 79.5 C63.2 60.75, 80.82 38.64, 117.66 0.25 M48.77 79.5 C67.62 57.73, 86.12 36.28, 117.66 0.25 M51.6 79.3 C73.59 53.63, 91.64 29.71, 120.48 0.06 M51.6 79.3 C71.84 58.16, 90.37 33.9, 120.48 0.06 M54.42 79.1 C70.72 60.6, 85.05 41.74, 123.3 -0.14 M54.42 79.1 C68.7 60.4, 85 43.15, 123.3 -0.14 M57.24 78.91 C76.19 58.81, 96.59 34, 126.12 -0.34 M57.24 78.91 C74.27 60.38, 91.52 39.06, 126.12 -0.34 M59.4 79.46 C73.98 62.78, 92.86 41.08, 128.29 0.22 M59.4 79.46 C74.1 63.58, 89.18 46.31, 128.29 0.22 M62.23 79.27 C82.5 56.2, 99.1 36.68, 131.11 0.02 M62.23 79.27 C81.34 59.01, 98.89 36.42, 131.11 0.02 M65.05 79.07 C86.13 55.7, 105.82 32.57, 133.93 -0.18 M65.05 79.07 C86.38 56.93, 106.23 33.3, 133.93 -0.18 M67.87 78.87 C84.93 58.17, 104.6 37.63, 136.1 0.38 M67.87 78.87 C85.1 58.91, 103.56 38.44, 136.1 0.38 M70.03 79.43 C87.19 62.7, 103.63 42.6, 138.92 0.19 M70.03 79.43 C95.22 49.12, 121.6 17.8, 138.92 0.19 M72.86 79.23 C93.3 53.97, 112.58 32.64, 141.74 -0.01 M72.86 79.23 C89.54 61.36, 106.64 42.56, 141.74 -0.01 M75.68 79.04 C96.49 55.24, 119.67 29.84, 144.56 -0.21 M75.68 79.04 C91.11 62.18, 104.24 45.96, 144.56 -0.21 M78.5 78.84 C103.82 47.57, 131.57 17.44, 146.73 0.35 M78.5 78.84 C105.75 47.35, 132.49 16.71, 146.73 0.35 M80.66 79.4 C96.66 63.53, 110.37 46.94, 149.55 0.15 M80.66 79.4 C101.44 56.02, 123.67 29.7, 149.55 0.15 M83.49 79.2 C102.02 55.73, 121.5 38.03, 152.37 -0.05 M83.49 79.2 C109.74 48.04, 136.11 17.94, 152.37 -0.05 M86.31 79 C105.02 56.69, 126.97 33.76, 155.19 -0.24 M86.31 79 C102.46 59.06, 121.83 37.92, 155.19 -0.24 M89.13 78.8 C102.13 60.37, 118.45 43.97, 157.36 0.31 M89.13 78.8 C105.79 60.49, 122.14 40.83, 157.36 0.31 M91.29 79.36 C105.46 61.09, 123.02 42.66, 160.18 0.12 M91.29 79.36 C107.47 60.33, 123.6 41.9, 160.18 0.12 M94.12 79.16 C118.95 52.5, 142.77 23.7, 163 -0.08 M94.12 79.16 C112.15 59.21, 129.94 37.03, 163 -0.08 M96.94 78.97 C113.08 56.72, 133.65 35.77, 165.82 -0.28 M96.94 78.97 C115.84 57.23, 132.63 36.36, 165.82 -0.28 M99.1 79.52 C117.44 56.55, 138.53 35.62, 167.99 0.28 M99.1 79.52 C120.28 55.42, 141.01 31.12, 167.99 0.28 M101.92 79.33 C127.52 53.15, 149.81 24.33, 170.81 0.08 M101.92 79.33 C126.63 47.93, 152.77 18.1, 170.81 0.08 M104.75 79.13 C128.85 51.7, 151.66 24.63, 173.63 -0.12 M104.75 79.13 C128.54 51.62, 153.74 23.61, 173.63 -0.12 M107.57 78.93 C135.17 50.22, 158.24 17.56, 176.45 -0.31 M107.57 78.93 C123.94 62.32, 137.88 44.56, 176.45 -0.31 M109.73 79.49 C124.15 63.03, 140.5 44.89, 178.62 0.24 M109.73 79.49 C127.28 58.16, 145.99 38.24, 178.62 0.24 M112.55 79.29 C136.05 52.52, 158.15 25.26, 181.44 0.05 M112.55 79.29 C131.95 57.41, 151.5 34.2, 181.44 0.05 M115.38 79.09 C135.52 56.19, 155.66 31.46, 184.26 -0.15 M115.38 79.09 C135.04 55.82, 154.97 32.62, 184.26 -0.15 M118.2 78.9 C142.49 49.99, 171.61 18.75, 186.43 0.41 M118.2 78.9 C136.14 58.71, 151.69 40.83, 186.43 0.41 M120.36 79.45 C142.32 52.53, 163.77 30.29, 189.25 0.21 M120.36 79.45 C144.86 52.22, 168.23 24.32, 189.25 0.21 M123.18 79.26 C149.07 50.47, 175.24 21.6, 192.07 0.01 M123.18 79.26 C139.51 59.03, 158.79 38.37, 192.07 0.01 M126.01 79.06 C149.58 52.3, 172.58 26.29, 194.89 -0.18 M126.01 79.06 C145.7 55.26, 166.86 34.32, 194.89 -0.18 M128.83 78.86 C146.86 55.47, 167.27 32.88, 197.06 0.37 M128.83 78.86 C143.96 59.88, 161.58 41.84, 197.06 0.37 M130.99 79.42 C150.86 57.42, 172.58 33.68, 199.88 0.18 M130.99 79.42 C151.51 55.39, 172.12 31.66, 199.88 0.18 M133.81 79.22 C154.11 57.4, 173.52 36.54, 202.04 0.73 M133.81 79.22 C159.5 50.97, 184.33 22.74, 202.04 0.73 M136.64 79.03 C160.7 52.82, 184.44 22.13, 204.21 1.29 M136.64 79.03 C155.18 56.21, 173.67 36, 204.21 1.29 M139.46 78.83 C165.62 49.94, 188.02 23.17, 206.38 1.85 M139.46 78.83 C152.98 62.88, 166.72 46.83, 206.38 1.85 M141.62 79.39 C163.83 53.27, 184.48 29.46, 207.88 3.16 M141.62 79.39 C158 59.57, 175.8 41.66, 207.88 3.16 M144.44 79.19 C167.85 50.7, 189.15 25.13, 210.05 3.72 M144.44 79.19 C170.75 49.5, 196.88 19.76, 210.05 3.72 M147.27 78.99 C170.97 52.35, 197.21 21.78, 211.56 5.03 M147.27 78.99 C170.74 52.4, 192.43 24.77, 211.56 5.03 M150.09 78.79 C171.44 54.67, 190.91 29.5, 213.07 6.34 M150.09 78.79 C172.08 53.4, 193.18 28.54, 213.07 6.34 M152.25 79.35 C166.62 63.58, 181.96 45.71, 213.92 8.41 M152.25 79.35 C167.82 61.03, 185.38 41.27, 213.92 8.41 M155.07 79.15 C175.5 56.49, 197.56 33.91, 215.43 9.72 M155.07 79.15 C174.36 56.89, 192.82 34.79, 215.43 9.72 M157.9 78.96 C173.84 60.69, 191.22 37.57, 216.29 11.79 M157.9 78.96 C180.8 53.88, 202.57 29.27, 216.29 11.79 M160.06 79.51 C180.71 56.66, 199.71 37.82, 217.14 13.85 M160.06 79.51 C181.24 54.74, 202.84 31.1, 217.14 13.85 M162.88 79.32 C179.22 59.68, 194.63 41.55, 217.99 15.92 M162.88 79.32 C183.89 54.8, 203.6 31.67, 217.99 15.92 M165.7 79.12 C177.57 66.3, 188.39 53.23, 217.53 19.5 M165.7 79.12 C177.83 65.52, 189.32 51.41, 217.53 19.5 M168.53 78.92 C188.32 56.71, 203.74 39.38, 217.73 22.32 M168.53 78.92 C183.16 60.9, 198.33 43.25, 217.73 22.32 M170.69 79.48 C189.02 58.16, 209.34 35.21, 217.93 25.14 M170.69 79.48 C188.65 57.37, 206.57 35.56, 217.93 25.14 M173.51 79.28 C185.63 68.36, 194.04 54.71, 218.13 27.96 M173.51 79.28 C188.3 61.95, 204.7 45.89, 218.13 27.96 M176.33 79.08 C185.76 70.21, 195.29 57.11, 217.67 31.54 M176.33 79.08 C184.41 70.02, 192.93 60.38, 217.67 31.54 M179.16 78.89 C193.39 62.55, 204.42 48.75, 217.86 34.36 M179.16 78.89 C191.34 63.71, 203.65 49.25, 217.86 34.36 M181.32 79.44 C194.11 65.07, 208.03 45.93, 218.06 37.18 M181.32 79.44 C189.24 70.56, 196.71 60.47, 218.06 37.18 M184.14 79.25 C192.84 70.62, 202.08 61.68, 218.26 40 M184.14 79.25 C192.35 68.08, 201.57 58.45, 218.26 40 M186.96 79.05 C192.71 71.18, 204.22 60.26, 217.8 43.58 M186.96 79.05 C193.32 71.75, 199.97 62.1, 217.8 43.58 M189.79 78.85 C195.32 72.52, 202.95 64.06, 218 46.4 M189.79 78.85 C197.96 69.3, 206.34 59.75, 218 46.4 M191.95 79.41 C200.38 67.17, 214.12 56.95, 218.19 49.22 M191.95 79.41 C197.39 73.13, 203.76 66.11, 218.19 49.22 M194.77 79.21 C202.12 69.3, 211.12 62.39, 218.39 52.04 M194.77 79.21 C200.23 73.71, 206.78 67.12, 218.39 52.04 M197.59 79.02 C202.63 72.66, 205.71 68.05, 217.93 55.62 M197.59 79.02 C202.45 73.21, 207.56 66.26, 217.93 55.62 M201.07 78.06 C204.54 71.88, 208.87 66.45, 217.47 59.2 M201.07 78.06 C206.33 73.5, 209.19 66.89, 217.47 59.2 M203.89 77.87 C209.68 73.21, 215.08 67.8, 217.02 62.77 M203.89 77.87 C208.58 72.02, 213.03 66.76, 217.02 62.77 M207.37 76.91 C211.53 72.89, 213.97 70.95, 215.9 67.1 M207.37 76.91 C209.12 74.97, 211.22 71.31, 215.9 67.1" stroke="#ffc9c9" stroke-width="0.25" fill="none"></path><path d="M19.75 0 M19.75 0 C86.41 -0.58, 152.44 -2.43, 197.25 0 M19.75 0 C57.59 -1.63, 95.76 -1.14, 197.25 0 M197.25 0 C211.83 -0.48, 218.27 7.67, 217 19.75 M197.25 0 C212.47 -1.18, 218.66 5.27, 217 19.75 M217 19.75 C215.49 28.79, 218.55 36.24, 217 59.25 M217 19.75 C216.55 31.5, 217.42 42.51, 217 59.25 M217 59.25 C216.72 71.07, 211.52 78.23, 197.25 79 M217 59.25 C216.83 71.9, 209.93 78.82, 197.25 79 M197.25 79 C148.98 78.97, 101.15 80.61, 19.75 79 M197.25 79 C141.43 77.77, 84.77 78, 19.75 79 M19.75 79 C8.06 79.03, 0.81 73.91, 0 59.25 M19.75 79 C4.38 79.92, 2.23 71.13, 0 59.25 M0 59.25 C1.45 46.99, -0.14 33.6, 0 19.75 M0 59.25 C-0.62 49.96, 0.97 40.16, 0 19.75 M0 19.75 C0 5.1, 8.07 1.13, 19.75 0 M0 19.75 C0 8.62, 7.82 0.16, 19.75 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2457.2409854388106 1381.993794984509) rotate(0 55.73994445800781 12.5)"><text x="55.73994445800781" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">plugin client</text></g><g mask="url(#mask-1L5e7UHL8LNV_1rXTN2IN)" stroke-linecap="round"><g transform="translate(2474.3351325421972 1435.6433489963056) rotate(0 -196.74806178414883 134.15632252373598)"><path d="M0.68 0.94 C-64.88 45.19, -327.47 222.27, -392.94 266.67 M-0.42 0.38 C-66.22 45.31, -328.79 223.2, -394.17 267.93" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2474.3351325421972 1435.6433489963056) rotate(0 -196.74806178414883 134.15632252373598)"><path d="M-375.08 242.67 C-383.14 252.27, -387.85 260.06, -393.36 268.7 M-376.85 243.44 C-383.75 253.93, -391.18 262.51, -393.35 267.62" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(2474.3351325421972 1435.6433489963056) rotate(0 -196.74806178414883 134.15632252373598)"><path d="M-363.53 259.64 C-375.61 263.65, -384.16 265.79, -393.36 268.7 M-365.31 260.41 C-376.78 264.21, -388.8 266.02, -393.35 267.62" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask id="mask-1L5e7UHL8LNV_1rXTN2IN"><rect x="0" y="0" fill="#fff" width="2966.7236017392984" height="1802.8542695864653"></rect><rect x="2261.1909086247992" y="1556.7488092913854" fill="#000" width="33.89997863769531" height="25" opacity="1"></rect></mask><g transform="translate(2261.1909086247997 1556.7488092913854) rotate(0 16.39616213324939 13.050862228656115)"><text x="16.949989318847656" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">call</text></g><g stroke-linecap="round"><g transform="translate(767.030290185769 807.4045609498135) rotate(0 0.44150554361590366 14.448143312257798)"><path d="M-0.91 0.33 C-0.44 5.19, 1.29 23.54, 1.79 28.19 M0.82 -0.54 C1.28 4.56, 1.11 24.46, 1.3 29.43" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(767.030290185769 807.4045609498135) rotate(0 0.44150554361590366 14.448143312257798)"><path d="M-3.39 17.02 C-3.46 20.37, -2.28 22.02, 0.29 29.96 M-3.46 15.53 C-2.41 19.16, -1.09 21.56, 1.5 29" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(767.030290185769 807.4045609498135) rotate(0 0.44150554361590366 14.448143312257798)"><path d="M6.44 16.9 C4.23 20.2, 3.26 21.87, 0.29 29.96 M6.38 15.4 C5.2 18.98, 4.31 21.41, 1.5 29" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask></mask><g transform="translate(608.2618975076747 916.0206572152019) rotate(0 143.83987426757812 12.5)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">init nri adaption for nri plugin</text></g><g stroke-linecap="round" transform="translate(1019.8775677041053 1221.2067954221857) rotate(0 849.0476335797993 161.42859322684126)"><path d="M32 0 M32 0 C455.08 5.96, 875.93 6.9, 1666.1 0 M32 0 C665.95 1.85, 1300.79 1.38, 1666.1 0 M1666.1 0 C1688.25 1.7, 1696.98 10.07, 1698.1 32 M1666.1 0 C1688.25 1.34, 1697.46 9.32, 1698.1 32 M1698.1 32 C1697.01 101.9, 1695.26 174.69, 1698.1 290.86 M1698.1 32 C1699.61 124.24, 1699.07 216.5, 1698.1 290.86 M1698.1 290.86 C1698.78 312.05, 1688.26 320.9, 1666.1 322.86 M1698.1 290.86 C1698.3 312.11, 1685.24 323.77, 1666.1 322.86 M1666.1 322.86 C1103.03 324.19, 539.22 325.56, 32 322.86 M1666.1 322.86 C1312.3 328.82, 959.14 328.77, 32 322.86 M32 322.86 C11.8 323.91, -0.47 312.35, 0 290.86 M32 322.86 C11.58 321.63, -1.78 310.12, 0 290.86 M0 290.86 C-0.37 206.29, -1.55 121.74, 0 32 M0 290.86 C-0.52 231.15, 0.36 173.69, 0 32 M0 32 C1.23 12.49, 12.62 -1.41, 32 0 M0 32 C0.1 9.12, 9.72 -1.7, 32 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1064.7401857582518 1240.7305904138175) rotate(0 183.63589477539062 22.5)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="36px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">isula-rust-extensions</text></g><g stroke-linecap="round" transform="translate(10 10) rotate(0 1401.6161924419978 560)"><path d="M32 0 M32 0 C1019.51 0.86, 2006.72 0.78, 2771.23 0 M32 0 C783.18 -9.02, 1534.07 -8.88, 2771.23 0 M2771.23 0 C2791.15 0.09, 2801.88 9.85, 2803.23 32 M2771.23 0 C2790.86 1.06, 2802.76 10.72, 2803.23 32 M2803.23 32 C2801.42 310.8, 2801.87 589.65, 2803.23 1088 M2803.23 32 C2804.63 250.65, 2804.76 469.38, 2803.23 1088 M2803.23 1088 C2803.1 1110.13, 2790.96 1121.08, 2771.23 1120 M2803.23 1088 C2804.65 1108.72, 2794.81 1119.8, 2771.23 1120 M2771.23 1120 C1877.99 1112.23, 986.63 1111.79, 32 1120 M2771.23 1120 C2164.59 1114.43, 1558.6 1114.75, 32 1120 M32 1120 C11.3 1120.51, 0.7 1109.99, 0 1088 M32 1120 C8.52 1117.97, 0.92 1107.19, 0 1088 M0 1088 C-1.2 699.7, -0.35 309.55, 0 32 M0 1088 C-3.84 693.41, -4.07 298.94, 0 32 M0 32 C0 11.96, 10.35 -0.25, 32 0 M0 32 C1.99 12.38, 8.87 0.31, 32 0" stroke="#1e1e1e" stroke-width="4" fill="none"></path></g><g transform="translate(62.12113813920496 44.44447835286246) rotate(0 52.09197998046875 22.5)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="36px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">iSulad</text></g><g stroke-linecap="round" transform="translate(574.8221902100627 738.1266876146383) rotate(0 187.5 29.5)"><path d="M14.75 0 M14.75 0 C124.95 -0.1, 235.73 0.22, 360.25 0 M14.75 0 C95.36 0, 176.9 -0.51, 360.25 0 M360.25 0 C369.83 -0.71, 375.43 6.71, 375 14.75 M360.25 0 C369.12 -1.36, 374.08 5.01, 375 14.75 M375 14.75 C374.32 22.41, 373.72 33.71, 375 44.25 M375 14.75 C374.13 22.63, 374.95 28.41, 375 44.25 M375 44.25 C376.55 53.7, 371.36 60.94, 360.25 59 M375 44.25 C377.1 55.89, 371.7 57.42, 360.25 59 M360.25 59 C242.52 56.77, 122.48 57.75, 14.75 59 M360.25 59 C275.29 59.01, 189.55 59.24, 14.75 59 M14.75 59 C5.68 60.61, -0.99 53.95, 0 44.25 M14.75 59 C4.57 58.83, -2.24 54.02, 0 44.25 M0 44.25 C-0.79 39.58, 1.42 30.04, 0 14.75 M0 44.25 C0.14 37.02, 0.58 28.32, 0 14.75 M0 14.75 C0.94 4.39, 5.91 0.82, 14.75 0 M0 14.75 C-0.85 5.57, 2.8 -0.88, 14.75 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(644.2823113648478 755.1266876146383) rotate(0 118.03987884521484 12.5)"><text x="118.03987884521484" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">nri_runtime_service_init</text></g><g transform="translate(590.8070756859149 699.1417466522807) rotate(0 177.9464569091797 14.008811488808078)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="22.41409838209323px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">register runtime service callback</text></g><g mask="url(#mask-ZCSyk_rtiJuADQW74jI3M)" stroke-linecap="round"><g transform="translate(1319.8398044369849 831.2629957644976) rotate(0 -0.002643446790841608 59.67985069917631)"><path d="M0.52 0.41 C0.61 20.32, 0.61 98.96, 0.7 118.6 M-0.67 -0.42 C-0.78 19.7, -0.61 99.87, -0.46 119.78" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1319.8398044369849 831.2629957644976) rotate(0 -0.002643446790841608 59.67985069917631)"><path d="M-10.16 90.98 C-8.95 100.26, -6.25 107.29, 0.52 119.95 M-9.98 92.03 C-6.74 100.61, -4.63 108.69, -0.46 119.45" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1319.8398044369849 831.2629957644976) rotate(0 -0.002643446790841608 59.67985069917631)"><path d="M10.36 90.89 C5.19 100.29, 1.52 107.34, 0.52 119.95 M10.55 91.94 C7.45 100.52, 3.22 108.63, -0.46 119.45" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask id="mask-ZCSyk_rtiJuADQW74jI3M"><rect x="0" y="0" fill="#fff" width="1420.9360972289537" height="1050.5500333334312"></rect><rect x="1215.0980414701762" y="865.906514548964" fill="#000" width="210.57981872558594" height="50" opacity="1"></rect></mask><g transform="translate(1215.0980414701762 865.906514548964) rotate(0 104.73911952001771 25.03633191470999)"><text x="105.28990936279297" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">if </text><text x="105.28990936279297" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">！disable_connections</text></g><g stroke-linecap="round" transform="translate(1191.921244720006 960.2469660393972) rotate(0 135 27.5)"><path d="M13.75 0 M13.75 0 C103.79 0.52, 195.27 -0.35, 256.25 0 M13.75 0 C108.75 -1.2, 202.53 -2.65, 256.25 0 M256.25 0 C266.96 1.06, 270.14 4.37, 270 13.75 M256.25 0 C264.34 -1.89, 272.21 3.16, 270 13.75 M270 13.75 C271.33 22.37, 271.1 29.53, 270 41.25 M270 13.75 C269.1 21.53, 269.3 30.58, 270 41.25 M270 41.25 C268.33 52.37, 266.59 56.28, 256.25 55 M270 41.25 C270.29 48.68, 267.37 53.55, 256.25 55 M256.25 55 C193.3 57.51, 127.74 57.78, 13.75 55 M256.25 55 C204.87 54.79, 152.32 55.54, 13.75 55 M13.75 55 C4.34 56.09, -1.84 52.35, 0 41.25 M13.75 55 C4.27 55.29, 1.1 48.7, 0 41.25 M0 41.25 C0.6 28.9, -1.12 20.11, 0 13.75 M0 41.25 C0.91 32.17, 0.43 24.73, 0 13.75 M0 13.75 C-1.08 5.44, 2.96 -1.85, 13.75 0 M0 13.75 C1.77 6.24, 3.64 0.71, 13.75 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1260.0513106379747 975.2469660393972) rotate(0 66.86993408203125 12.5)"><text x="66.86993408203125" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">startListener</text></g><g stroke-linecap="round" transform="translate(1417.5546275536744 1720.7162934137905) rotate(0 155 31.5)"><path d="M4.19 3.64 C4.19 3.64, 4.19 3.64, 4.19 3.64 M4.19 3.64 C4.19 3.64, 4.19 3.64, 4.19 3.64 M1.77 9.48 C5.36 7.54, 6.41 3.47, 8.98 1.18 M1.77 9.48 C3.8 8.07, 4.75 6.51, 8.98 1.18 M1.31 13.06 C4.48 9.23, 7.68 5, 11.8 0.99 M1.31 13.06 C5.34 9.34, 7.77 4.89, 11.8 0.99 M0.85 16.64 C3.94 14.63, 6.5 11.41, 15.28 0.03 M0.85 16.64 C4.75 11.17, 10.92 6.25, 15.28 0.03 M1.05 19.46 C8.25 14.68, 11.88 5.45, 16.79 1.35 M1.05 19.46 C5.18 13.57, 10.17 10.01, 16.79 1.35 M1.24 22.28 C5.51 16.53, 13.49 9.74, 19.61 1.15 M1.24 22.28 C8.34 14.11, 16.46 5.7, 19.61 1.15 M1.44 25.1 C5.88 21.14, 11.03 14.06, 22.43 0.95 M1.44 25.1 C9.63 15.54, 18.73 6.58, 22.43 0.95 M0.98 28.68 C10.56 19.78, 16.27 10.59, 25.26 0.75 M0.98 28.68 C8.1 19.89, 16.75 10.1, 25.26 0.75 M1.18 31.5 C5.96 25.76, 12.62 19.54, 27.42 1.31 M1.18 31.5 C7.27 23.78, 12.5 17.9, 27.42 1.31 M1.38 34.32 C7.03 26.59, 14.45 16.03, 30.24 1.11 M1.38 34.32 C7.48 26.9, 13.85 21.73, 30.24 1.11 M1.57 37.14 C12.4 25.99, 21.15 14.04, 33.06 0.92 M1.57 37.14 C11.74 24.28, 24.03 11.17, 33.06 0.92 M1.77 39.96 C12.95 23.71, 27.36 9.94, 35.23 1.47 M1.77 39.96 C13.16 25.34, 25.82 11.83, 35.23 1.47 M1.31 43.54 C15.82 29.76, 26.98 14, 38.05 1.28 M1.31 43.54 C9.77 34.73, 17.24 23.64, 38.05 1.28 M1.51 46.36 C10.75 38.18, 18.8 27.67, 40.87 1.08 M1.51 46.36 C15.37 31.66, 26.95 15.19, 40.87 1.08 M1.71 49.18 C15.52 34.84, 28.45 17.66, 43.69 0.88 M1.71 49.18 C13.77 37.31, 23.69 24.43, 43.69 0.88 M1.9 52.01 C12.59 37.05, 22.45 27.91, 45.86 1.44 M1.9 52.01 C16.17 34.29, 33.03 15.77, 45.86 1.44 M2.1 54.83 C15.24 38.49, 31.82 23.02, 48.68 1.24 M2.1 54.83 C17.92 36.94, 33.8 18.96, 48.68 1.24 M2.95 56.89 C16.67 43.83, 29.32 25.93, 51.5 1.04 M2.95 56.89 C19.51 36.96, 37.52 16.17, 51.5 1.04 M4.46 58.21 C23.38 35.96, 40.09 17.67, 54.32 0.85 M4.46 58.21 C23.11 37.55, 40.49 16.16, 54.32 0.85 M5.97 59.52 C23.98 38.68, 42.97 18.13, 56.49 1.41 M5.97 59.52 C19.67 42.7, 35.45 25.86, 56.49 1.41 M6.83 61.58 C18.61 51.22, 29.83 37.48, 59.31 1.21 M6.83 61.58 C20.14 44.51, 34.2 28.57, 59.31 1.21 M8.99 62.14 C30.2 40.55, 47.67 15.5, 62.13 1.01 M8.99 62.14 C24.4 42.97, 40.86 24.53, 62.13 1.01 M11.16 62.7 C28.96 43.69, 46.55 19.87, 64.95 0.81 M11.16 62.7 C22.4 49.73, 32.79 36.76, 64.95 0.81 M13.32 63.26 C27.56 47.84, 41.71 30.44, 67.12 1.37 M13.32 63.26 C34.73 37.99, 54.91 12.9, 67.12 1.37 M15.49 63.81 C31.53 45.16, 48.41 26.7, 69.94 1.17 M15.49 63.81 C34.59 40.92, 53.33 18.7, 69.94 1.17 M18.31 63.62 C36.19 43.3, 52.65 23.44, 72.76 0.98 M18.31 63.62 C30.06 49.59, 42.29 36.39, 72.76 0.98 M21.13 63.42 C35.25 47.11, 51.82 28.16, 75.58 0.78 M21.13 63.42 C39.55 42.97, 58.56 20.78, 75.58 0.78 M23.3 63.98 C38.85 43.65, 54.18 25.48, 77.75 1.34 M23.3 63.98 C43.99 40.43, 64.32 15.27, 77.75 1.34 M26.12 63.78 C36.02 49.36, 48.98 35.37, 80.57 1.14 M26.12 63.78 C37.45 51.09, 48.84 37.91, 80.57 1.14 M28.94 63.58 C48.95 39.83, 66.36 19.34, 83.39 0.94 M28.94 63.58 C40.25 50.21, 50.84 36.82, 83.39 0.94 M31.76 63.39 C49.67 42.34, 69.65 18.69, 85.56 1.5 M31.76 63.39 C44.74 49.8, 55.97 37.31, 85.56 1.5 M33.93 63.94 C53.04 43.16, 69.06 23.82, 88.38 1.3 M33.93 63.94 C52.65 40.79, 72.83 18.59, 88.38 1.3 M36.75 63.75 C53.36 45.32, 68.37 26.47, 91.2 1.1 M36.75 63.75 C51.42 46.51, 66.62 30.74, 91.2 1.1 M39.57 63.55 C58.91 41.95, 78.93 16.51, 94.02 0.91 M39.57 63.55 C51.78 48.91, 65.81 33.65, 94.02 0.91 M42.39 63.35 C60.16 43.4, 77.82 25.78, 96.19 1.46 M42.39 63.35 C57.65 44.97, 75.07 26.76, 96.19 1.46 M44.56 63.91 C65.51 41.41, 82.63 19.36, 99.01 1.27 M44.56 63.91 C64.53 41.88, 82 20, 99.01 1.27 M47.38 63.71 C58.67 48.33, 73.42 33.99, 101.83 1.07 M47.38 63.71 C61.42 47.14, 75.38 31.8, 101.83 1.07 M50.2 63.51 C66.15 45.33, 85.82 23.28, 104.65 0.87 M50.2 63.51 C67.62 42.79, 87 21.03, 104.65 0.87 M53.02 63.32 C64.89 49.28, 74.42 35.85, 106.82 1.43 M53.02 63.32 C70.36 43.2, 87.09 24.97, 106.82 1.43 M55.19 63.87 C76.56 41.42, 93.22 17.39, 109.64 1.23 M55.19 63.87 C75.89 39.48, 96.28 16.78, 109.64 1.23 M58.01 63.68 C75.98 43.62, 93.83 23.37, 112.46 1.04 M58.01 63.68 C74.14 45.33, 90.14 27.88, 112.46 1.04 M60.83 63.48 C79.59 42.79, 99.52 19.73, 115.28 0.84 M60.83 63.48 C79.67 41.44, 98.98 19.01, 115.28 0.84 M63.65 63.28 C81.45 45.49, 97.19 28.4, 117.45 1.4 M63.65 63.28 C75.99 48.78, 87.92 35.66, 117.45 1.4 M65.82 63.84 C80.48 46.47, 94.44 30.54, 120.27 1.2 M65.82 63.84 C82.13 44.84, 99.07 25.09, 120.27 1.2 M68.64 63.64 C86.98 46.04, 104.5 25.87, 123.09 1 M68.64 63.64 C89.95 38.69, 111.42 14.46, 123.09 1 M71.46 63.44 C92.26 40.9, 114.74 15.66, 125.26 1.56 M71.46 63.44 C82.46 49.89, 93.5 38.13, 125.26 1.56 M73.63 64 C94.23 43.33, 110.74 20.26, 128.08 1.36 M73.63 64 C89.2 46.09, 104.66 28.91, 128.08 1.36 M76.45 63.8 C94.04 41.17, 113.4 22.85, 130.9 1.16 M76.45 63.8 C95.09 43.76, 111.86 23.97, 130.9 1.16 M79.27 63.61 C99.86 39.44, 121.84 15.83, 133.72 0.97 M79.27 63.61 C91.21 50.26, 102.65 36.34, 133.72 0.97 M82.09 63.41 C97.2 48.02, 108.36 32.8, 135.89 1.52 M82.09 63.41 C96.95 47.87, 109.68 30.96, 135.89 1.52 M84.26 63.97 C97.65 44.46, 116.99 26.19, 138.71 1.33 M84.26 63.97 C103.55 42.04, 121.98 21.58, 138.71 1.33 M87.08 63.77 C107.44 38.7, 130.18 15.14, 141.53 1.13 M87.08 63.77 C105.97 41.86, 127.32 17.66, 141.53 1.13 M89.9 63.57 C102.76 45.31, 117.72 29.41, 144.35 0.93 M89.9 63.57 C103.12 48.98, 116.42 32.99, 144.35 0.93 M92.72 63.38 C110.57 43.34, 129.19 22.52, 146.52 1.49 M92.72 63.38 C109.9 44.02, 126.08 25.89, 146.52 1.49 M94.89 63.93 C104.14 51.05, 116.74 39.33, 149.34 1.29 M94.89 63.93 C114.35 42.28, 131.38 21.71, 149.34 1.29 M97.71 63.74 C117.01 41, 139.13 15.57, 152.16 1.09 M97.71 63.74 C112.65 45.1, 130.61 26.15, 152.16 1.09 M100.53 63.54 C120.09 41.42, 141.12 16.08, 154.98 0.9 M100.53 63.54 C113.36 48.06, 126.94 33.15, 154.98 0.9 M103.35 63.34 C124.93 38.33, 143.74 16.42, 157.15 1.45 M103.35 63.34 C116.03 48.93, 130.17 33.85, 157.15 1.45 M105.52 63.9 C124.93 41.52, 141.55 23.88, 159.97 1.26 M105.52 63.9 C119.43 48.96, 131.5 34.88, 159.97 1.26 M108.34 63.7 C121.02 47.35, 137.81 32.79, 162.79 1.06 M108.34 63.7 C118.67 51.46, 130.2 37.8, 162.79 1.06 M111.16 63.5 C129.39 40.99, 151.88 18, 165.61 0.86 M111.16 63.5 C121.54 50.4, 133.95 37.54, 165.61 0.86 M113.32 64.06 C127.89 47.02, 145.84 27.16, 167.78 1.42 M113.32 64.06 C132.9 42.06, 152.72 18.55, 167.78 1.42 M116.15 63.86 C131.08 50.28, 143.39 32.76, 170.6 1.22 M116.15 63.86 C130.56 46.99, 146.49 29.73, 170.6 1.22 M118.97 63.67 C132.24 47.45, 145.38 31.94, 173.42 1.03 M118.97 63.67 C131.35 49.67, 143.04 36.57, 173.42 1.03 M121.79 63.47 C138.4 41.2, 157.75 23.09, 175.59 1.58 M121.79 63.47 C142.44 40.46, 163.76 15.13, 175.59 1.58 M123.95 64.03 C139.74 46.33, 156.23 25.27, 178.41 1.39 M123.95 64.03 C136.24 49.41, 149.49 35.56, 178.41 1.39 M126.78 63.83 C141.91 46.94, 155.34 31.09, 181.23 1.19 M126.78 63.83 C145 43.43, 160.95 25, 181.23 1.19 M129.6 63.63 C142.14 48.28, 155.14 35.63, 184.05 0.99 M129.6 63.63 C148.43 42.24, 166.32 21.61, 184.05 0.99 M132.42 63.43 C144.14 50.43, 158.88 33.29, 186.22 1.55 M132.42 63.43 C146.89 45.34, 162.68 29.52, 186.22 1.55 M134.58 63.99 C146.76 49.99, 157.95 37.41, 189.04 1.35 M134.58 63.99 C149.06 48.31, 162.27 32.02, 189.04 1.35 M137.41 63.79 C154.15 45.67, 171.42 24.76, 191.86 1.15 M137.41 63.79 C153.32 46.25, 169.51 27.23, 191.86 1.15 M140.23 63.6 C156.66 44.36, 173.98 24.09, 194.68 0.96 M140.23 63.6 C161.26 39.32, 182.7 13.09, 194.68 0.96 M143.05 63.4 C158.5 43.43, 176.44 24.04, 196.85 1.51 M143.05 63.4 C154.62 48.02, 169.28 34.02, 196.85 1.51 M145.21 63.96 C162.03 40.12, 182.52 20.42, 199.67 1.32 M145.21 63.96 C161.25 44.7, 177.61 26.37, 199.67 1.32 M148.04 63.76 C166.68 42.49, 187.91 20.12, 202.49 1.12 M148.04 63.76 C169.98 40.16, 192.33 14.01, 202.49 1.12 M150.86 63.56 C161.95 48.58, 176.43 32.09, 205.31 0.92 M150.86 63.56 C167 46.33, 181.37 29.33, 205.31 0.92 M153.68 63.37 C173.77 39.67, 192.04 19.06, 207.48 1.48 M153.68 63.37 C167.98 47.36, 182.06 29.37, 207.48 1.48 M155.84 63.92 C166.28 50.24, 178.85 35.81, 210.3 1.28 M155.84 63.92 C171.98 45.3, 188.06 26.63, 210.3 1.28 M158.67 63.73 C170.92 50.19, 184.07 35.94, 213.12 1.08 M158.67 63.73 C170.33 49.66, 182.46 37.18, 213.12 1.08 M161.49 63.53 C177.09 44.68, 191.78 31.03, 215.28 1.64 M161.49 63.53 C177.91 44.61, 194.89 24.61, 215.28 1.64 M163.65 64.09 C182.87 43.25, 200.19 25.2, 218.11 1.45 M163.65 64.09 C176.57 50.2, 186.84 37.35, 218.11 1.45 M166.47 63.89 C182.17 46.72, 197.45 30.44, 220.93 1.25 M166.47 63.89 C186.6 39.55, 207.95 15.91, 220.93 1.25 M169.3 63.69 C183.45 47.49, 194.23 34.43, 223.75 1.05 M169.3 63.69 C183.55 46.05, 197.11 30.13, 223.75 1.05 M172.12 63.49 C183.44 50.46, 199.62 33.87, 225.91 1.61 M172.12 63.49 C188.98 44.91, 205.25 26.37, 225.91 1.61 M174.28 64.05 C195.27 43.48, 211.37 21.94, 228.74 1.41 M174.28 64.05 C196.38 38.9, 215.72 15.83, 228.74 1.41 M177.1 63.85 C192.12 46.86, 209.29 26.79, 231.56 1.21 M177.1 63.85 C197.19 39.04, 219.33 15.43, 231.56 1.21 M179.93 63.66 C197 47.45, 208.41 28.4, 234.38 1.02 M179.93 63.66 C192.57 50.17, 206.36 34.78, 234.38 1.02 M182.75 63.46 C199.93 45.48, 215.53 26.22, 236.54 1.57 M182.75 63.46 C199.03 47.06, 213.24 28.19, 236.54 1.57 M184.91 64.02 C198.29 49.3, 213.02 31.24, 239.37 1.38 M184.91 64.02 C202.23 43.95, 219.34 25.13, 239.37 1.38 M187.74 63.82 C200.94 48.7, 211.81 34.71, 242.19 1.18 M187.74 63.82 C208.55 39.94, 228.51 17.26, 242.19 1.18 M190.56 63.62 C207.95 42.11, 228.38 20.19, 245.01 0.98 M190.56 63.62 C207.37 43.48, 225.87 23.49, 245.01 0.98 M193.38 63.43 C212.88 38.05, 233.19 13.66, 247.17 1.54 M193.38 63.43 C211.53 42.77, 229.22 22.2, 247.17 1.54 M195.54 63.98 C208.64 48.34, 224.85 32.08, 250 1.34 M195.54 63.98 C207.37 49.35, 217.97 36.57, 250 1.34 M198.37 63.79 C216.91 40.77, 237.03 16.59, 252.82 1.14 M198.37 63.79 C211.33 49.13, 223.55 33.63, 252.82 1.14 M201.19 63.59 C215.48 45.35, 229.16 29.66, 255.64 0.95 M201.19 63.59 C220.66 43.03, 238.99 20.11, 255.64 0.95 M204.01 63.39 C221.39 43.89, 240.36 20.33, 257.81 1.5 M204.01 63.39 C215.46 49.81, 226.16 38.26, 257.81 1.5 M206.17 63.95 C223.65 42.05, 241.5 24.93, 260.63 1.31 M206.17 63.95 C224.82 42.73, 242.41 21.97, 260.63 1.31 M209 63.75 C228.38 42.46, 249.62 18.41, 263.45 1.11 M209 63.75 C228.28 41.52, 247.06 20.59, 263.45 1.11 M211.82 63.55 C230.66 43.04, 251.35 18.48, 265.61 1.67 M211.82 63.55 C232.36 39.46, 253.59 14.73, 265.61 1.67 M213.98 64.11 C227.27 49.01, 239.75 35.17, 268.44 1.47 M213.98 64.11 C229.59 45.53, 244.01 29.4, 268.44 1.47 M216.8 63.91 C238.32 42.32, 256.61 19.83, 271.26 1.27 M216.8 63.91 C236.99 40.96, 256.87 16.84, 271.26 1.27 M219.63 63.72 C231.43 51.61, 241.04 38.69, 274.08 1.08 M219.63 63.72 C237.34 42.16, 256.06 21.31, 274.08 1.08 M222.45 63.52 C239.87 44.22, 256.22 27.32, 276.24 1.63 M222.45 63.52 C242.79 40.33, 262.6 18.85, 276.24 1.63 M224.61 64.08 C243 43.46, 263.19 21.92, 279.07 1.44 M224.61 64.08 C246.4 39.73, 266.38 13.64, 279.07 1.44 M227.43 63.88 C250.03 41.06, 270.36 13.43, 281.89 1.24 M227.43 63.88 C242.94 45.81, 257.83 27.94, 281.89 1.24 M230.26 63.68 C241.87 48.14, 253.58 36.09, 284.71 1.04 M230.26 63.68 C243.3 48.29, 256.86 32.96, 284.71 1.04 M233.08 63.48 C250.04 45.83, 266.68 27.31, 286.87 1.6 M233.08 63.48 C246.84 48.2, 261.45 32.7, 286.87 1.6 M235.24 64.04 C249.89 46.97, 265.76 30.45, 289.7 1.4 M235.24 64.04 C247.15 50.94, 259.78 34.8, 289.7 1.4 M238.06 63.84 C252.48 45.86, 265.61 29.75, 292.52 1.2 M238.06 63.84 C250.66 46.57, 266.33 31.87, 292.52 1.2 M240.89 63.65 C258.23 43.79, 275.69 25.27, 295.34 1.01 M240.89 63.65 C263.11 39.74, 285.59 13.66, 295.34 1.01 M243.71 63.45 C262.24 39.85, 282.77 19.59, 298.16 0.81 M243.71 63.45 C259.05 44.7, 275.14 27.61, 298.16 0.81 M245.87 64.01 C261.97 49.24, 275.85 29.93, 300.33 1.37 M245.87 64.01 C264.2 43.73, 281.97 21.62, 300.33 1.37 M248.69 63.81 C264.12 46.7, 279.53 25.16, 301.83 2.68 M248.69 63.81 C267.09 40.66, 285.71 19.33, 301.83 2.68 M251.52 63.61 C268.56 44.02, 285.97 22.52, 304 3.24 M251.52 63.61 C269.48 42.17, 288.58 21.12, 304 3.24 M253.68 64.17 C265.64 52.28, 277.92 40.2, 306.17 3.79 M253.68 64.17 C271.14 43.72, 287.82 24.21, 306.17 3.79 M256.5 63.97 C274.02 43.04, 287.93 25.42, 307.68 5.11 M256.5 63.97 C272.49 46.96, 286.64 29.75, 307.68 5.11 M259.32 63.78 C269.3 49.09, 284.18 35.94, 308.53 7.17 M259.32 63.78 C272.51 48.22, 288 33.14, 308.53 7.17 M262.15 63.58 C275.47 50.64, 286.42 34.34, 309.38 9.24 M262.15 63.58 C277.32 46.53, 292.19 29.13, 309.38 9.24 M264.31 64.14 C278.62 45.97, 294.81 28.04, 309.58 12.06 M264.31 64.14 C278.59 46.74, 292.27 31.4, 309.58 12.06 M267.13 63.94 C275.97 55.21, 287.6 41.92, 309.78 14.88 M267.13 63.94 C282.81 47.98, 296.48 30.89, 309.78 14.88 M269.95 63.74 C280.66 48.74, 295.73 36.46, 309.32 18.46 M269.95 63.74 C285.89 47.81, 298.76 30.79, 309.32 18.46 M272.78 63.54 C283.05 50.42, 291.82 39.76, 309.52 21.28 M272.78 63.54 C286.46 47.86, 300.1 32.63, 309.52 21.28 M274.94 64.1 C289.05 50.73, 301.9 35.16, 309.71 24.1 M274.94 64.1 C282.17 56.72, 288.54 46.68, 309.71 24.1 M277.76 63.9 C285.26 53, 295.69 40.91, 309.91 26.92 M277.76 63.9 C289.16 49.98, 302.45 36.01, 309.91 26.92 M280.58 63.71 C290.24 51.52, 301.9 42.11, 310.11 29.74 M280.58 63.71 C287.15 55.36, 295.32 47.05, 310.11 29.74 M283.41 63.51 C287.28 58.28, 295.01 49.24, 309.65 33.32 M283.41 63.51 C288.47 56.35, 295.83 51.09, 309.65 33.32 M285.57 64.07 C292.17 58.41, 297.56 49.8, 309.85 36.14 M285.57 64.07 C291.42 57.29, 297.53 50.92, 309.85 36.14 M288.39 63.87 C294.78 55.4, 305.5 46.11, 310.04 38.96 M288.39 63.87 C296.17 55.82, 303.22 47.97, 310.04 38.96 M291.21 63.67 C295.77 56.57, 300.72 51.7, 310.24 41.79 M291.21 63.67 C296.73 57.73, 303.63 50.93, 310.24 41.79 M294.04 63.47 C301.03 56.56, 305.93 51.3, 309.78 45.36 M294.04 63.47 C298.26 60.19, 300.68 54.46, 309.78 45.36 M297.51 62.52 C303.99 56.8, 305.56 51.01, 309.98 48.18 M297.51 62.52 C300.37 57.67, 304.8 52.81, 309.98 48.18 M300.34 62.33 C302.57 57.97, 307 53.5, 310.18 51 M300.34 62.33 C304.39 57.51, 307.75 53.68, 310.18 51 M302.5 62.88 C304.52 60.84, 305.32 58.32, 307.09 57.6 M302.5 62.88 C303.97 60.88, 305.15 59.61, 307.09 57.6" stroke="#a5d8ff" stroke-width="0.25" fill="none"></path><path d="M15.75 0 M15.75 0 C100.56 -2.51, 188.44 -2.25, 294.25 0 M15.75 0 C121.89 -1.52, 228.86 -0.67, 294.25 0 M294.25 0 C306.56 1.48, 311.37 6.22, 310 15.75 M294.25 0 C306.95 -2.2, 309.56 5.33, 310 15.75 M310 15.75 C310.43 27.49, 312.19 37.98, 310 47.25 M310 15.75 C309 27.43, 310.51 40.42, 310 47.25 M310 47.25 C309.82 56.7, 305.39 61.64, 294.25 63 M310 47.25 C310.42 58.23, 306.48 63.61, 294.25 63 M294.25 63 C219.67 64.68, 144.45 63.28, 15.75 63 M294.25 63 C205.37 62.57, 115.91 62.29, 15.75 63 M15.75 63 C6.74 62.11, -0.72 59.58, 0 47.25 M15.75 63 C7.37 64.85, 0.13 57.26, 0 47.25 M0 47.25 C-0.52 37.97, 1.08 28.51, 0 15.75 M0 47.25 C-1.03 33.86, -0.71 22, 0 15.75 M0 15.75 C-1.36 3.48, 6.51 -0.51, 15.75 0 M0 15.75 C2.11 7.43, 7.36 -0.76, 15.75 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1501.96469225094 1739.7162934137905) rotate(0 70.58993530273438 12.5)"><text x="70.58993530273438" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">external plugin</text></g><g stroke-linecap="round"><g transform="translate(1584.288064523253 1714.3113411834584) rotate(0 35.721670925128535 -145.4241019509318)"><path d="M-0.25 0.93 C11.66 -47.5, 59.73 -243.12, 71.7 -291.78 M1.82 0.37 C13.55 -47.77, 59.26 -241.48, 70.74 -290.4" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1584.288064523253 1714.3113411834584) rotate(0 35.721670925128535 -145.4241019509318)"><path d="M75.81 -260.64 C72.93 -270.43, 73.03 -278.46, 72.47 -292.14 M73.35 -260.58 C73.93 -272.27, 72.09 -283, 71.18 -291.16" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1584.288064523253 1714.3113411834584) rotate(0 35.721670925128535 -145.4241019509318)"><path d="M55.84 -265.34 C59.29 -273.59, 65.77 -280.11, 72.47 -292.14 M53.38 -265.28 C61.53 -275.26, 67.29 -284.21, 71.18 -291.16" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(601.3402437471732 1000.2881143540808) rotate(0 178 27.5)"><path d="M13.75 0 M13.75 0 C98.88 2.63, 185.79 1.89, 342.25 0 M13.75 0 C129.26 0.46, 245.38 -0.11, 342.25 0 M342.25 0 C351.58 1.37, 357.15 3.63, 356 13.75 M342.25 0 C349.18 -0.79, 354.06 5.79, 356 13.75 M356 13.75 C355.48 22.23, 354.08 30.55, 356 41.25 M356 13.75 C355.4 20.92, 355.73 29.75, 356 41.25 M356 41.25 C357.24 51.27, 350.87 54.9, 342.25 55 M356 41.25 C357.07 50.65, 349.31 53.74, 342.25 55 M342.25 55 C233.42 52.59, 123.72 53.02, 13.75 55 M342.25 55 C266.05 56.61, 190.57 55.92, 13.75 55 M13.75 55 C5.77 54.7, -0.86 50.43, 0 41.25 M13.75 55 C6.77 55.58, 1.3 50.95, 0 41.25 M0 41.25 C-0.84 33.23, 0.86 25.82, 0 13.75 M0 41.25 C-1.17 31.92, -0.8 21.19, 0 13.75 M0 13.75 C-1.57 2.78, 4.13 -0.4, 13.75 0 M0 13.75 C0.63 3.43, 2.77 1.91, 13.75 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(641.7703584932669 1015.2881143540808) rotate(0 137.56988525390625 12.5)"><text x="137.56988525390625" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">nri_external_plugin_connect</text></g><g mask="url(#mask-DfSAYmOosbg_eZ8jxGRof)" stroke-linecap="round"><g transform="translate(1635.887790133892 1350.279062203228) rotate(0 -388.4316504155438 -146.43912801203805)"><path d="M1.14 0.3 C-54.69 -27.27, -205.79 -116.83, -335.51 -165.44 C-465.24 -214.05, -703.58 -270.15, -777.21 -291.37 M0.28 -0.58 C-55.61 -28.54, -206.39 -118.65, -336.11 -167.41 C-465.82 -216.18, -704.51 -272.45, -778 -293.18" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1635.887790133892 1350.279062203228) rotate(0 -388.4316504155438 -146.43912801203805)"><path d="M-747.63 -294.83 C-755.36 -295.06, -761.82 -294.45, -778.61 -293.1 M-747.22 -295.39 C-759.63 -293.87, -769.85 -293.38, -778.79 -294.08" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g><g transform="translate(1635.887790133892 1350.279062203228) rotate(0 -388.4316504155438 -146.43912801203805)"><path d="M-752.83 -274.98 C-759.22 -280.37, -764.34 -284.86, -778.61 -293.1 M-752.43 -275.54 C-763.07 -281.09, -771.44 -287.64, -778.79 -294.08" stroke="#1e1e1e" stroke-width="0.5" fill="none"></path></g></g><mask id="mask-DfSAYmOosbg_eZ8jxGRof"><rect x="0" y="0" fill="#fff" width="2513.030647276749" height="1742.6600436485405"></rect><rect x="1239.157293046141" y="1172.0647764889418" fill="#000" width="121.07992553710938" height="25" opacity="1"></rect></mask><g transform="translate(1239.157293046141 1172.0647764889418) rotate(0 8.298846672207219 31.77515770224818)"><text x="60.53996276855469" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">call callback</text></g></svg> +\ No newline at end of file +-- +2.25.1 + diff --git a/0118-NRI-add-nri-head-file-and-common-func.patch b/0118-NRI-add-nri-head-file-and-common-func.patch new file mode 100644 index 0000000..61c287e --- /dev/null +++ b/0118-NRI-add-nri-head-file-and-common-func.patch @@ -0,0 +1,1051 @@ +From d21dc88cb33049ec5d9ce8e16eb1a2f9c070219b Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Tue, 30 Jul 2024 20:17:08 +0800 +Subject: [PATCH 117/121] [NRI] add nri head file and common func + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + cmake/checker.cmake | 8 ++ + cmake/options.cmake | 13 ++- + src/CMakeLists.txt | 10 +++ + src/daemon/CMakeLists.txt | 9 ++ + src/daemon/common/CMakeLists.txt | 5 ++ + src/daemon/common/nri/CMakeLists.txt | 13 +++ + src/daemon/common/nri/nri_convert.h | 36 ++++++++ + src/daemon/common/nri/nri_spec.h | 24 ++++++ + src/daemon/common/nri/nri_utils.h | 71 ++++++++++++++++ + src/daemon/config/isulad_config.h | 16 ++++ + src/daemon/modules/api/specs_api.h | 10 ++- + src/daemon/modules/spec/specs.c | 67 ++++++++++++++- + src/daemon/nri/CMakeLists.txt | 6 ++ + src/daemon/nri/nri_adaption.h | 123 +++++++++++++++++++++++++++ + src/daemon/nri/nri_helpers.h | 63 ++++++++++++++ + src/daemon/nri/nri_plugin_ops.h | 38 +++++++++ + src/daemon/nri/nri_result.h | 114 +++++++++++++++++++++++++ + src/daemon/nri/plugin.h | 95 +++++++++++++++++++++ + src/utils/cpputils/transform.cc | 16 ++++ + src/utils/cpputils/transform.h | 2 + + src/utils/cutils/utils.c | 7 ++ + src/utils/cutils/utils.h | 2 + + src/utils/tar/util_archive.c | 7 -- + 23 files changed, 744 insertions(+), 11 deletions(-) + create mode 100644 src/daemon/common/nri/CMakeLists.txt + create mode 100644 src/daemon/common/nri/nri_convert.h + create mode 100644 src/daemon/common/nri/nri_spec.h + create mode 100644 src/daemon/common/nri/nri_utils.h + create mode 100644 src/daemon/nri/CMakeLists.txt + create mode 100644 src/daemon/nri/nri_adaption.h + create mode 100644 src/daemon/nri/nri_helpers.h + create mode 100644 src/daemon/nri/nri_plugin_ops.h + create mode 100644 src/daemon/nri/nri_result.h + create mode 100644 src/daemon/nri/plugin.h + +diff --git a/cmake/checker.cmake b/cmake/checker.cmake +index 13f0fd62..b877c1c8 100644 +--- a/cmake/checker.cmake ++++ b/cmake/checker.cmake +@@ -113,6 +113,14 @@ if (ENABLE_SHIM_V2) + _CHECK(LIBSHIM_V2_LIBRARY "LIBSHIM_V2_LIBRARY-NOTFOUND" "libshim_v2.so") + endif() + ++if (ENABLE_NRI) ++ find_path(NRI_INCLUDE_DIR nri_plugin.h) ++ _CHECK(NRI_INCLUDE_DIR "NRI_INCLUDE_DIR-NOTFOUND" "nri_plugin.h") ++ ++ find_library(LIBISULA_NRI_LIBRARY isula_nri) ++ _CHECK(LIBISULA_NRI_LIBRARY "LIBISULA_NRI_LIBRARY-NOTFOUND" "libisula_nri.so") ++endif() ++ + if (OPENSSL_VERIFY) + find_path(OPENSSL_INCLUDE_DIR openssl/x509.h) + _CHECK(OPENSSL_INCLUDE_DIR "OPENSSL_INCLUDE_DIR-NOTFOUND" "openssl/x509.h") +diff --git a/cmake/options.cmake b/cmake/options.cmake +index 5b17f631..41177fe0 100644 +--- a/cmake/options.cmake ++++ b/cmake/options.cmake +@@ -59,12 +59,23 @@ if (ENABLE_SANDBOXER STREQUAL "ON") + endif() + + option(ENABLE_OOM_MONITOR "Enable oom monitor" ON) +-IF (ENABLE_OOM_MONITOR STREQUAL "ON") ++if (ENABLE_OOM_MONITOR STREQUAL "ON") + add_definitions(-DENABLE_OOM_MONITOR) + set(ENABLE_OOM_MONITOR 1) + message("${Green}-- Enable oom monitor${ColourReset}") + endif() + ++option(ENABLE_NRI "Enable NRI API" OFF) ++if (ENABLE_NRI STREQUAL "ON") ++ if (ENABLE_CRI_API_V1) ++ add_definitions(-DENABLE_NRI) ++ set(ENABLE_NRI 1) ++ message("${Green}-- Enable NRI API${ColourReset}") ++ else() ++ message("${Yellow}-- Can not enable NRI, NRI need enable CRI API V1 first ${ColourReset}") ++ endif() ++endif() ++ + option(ENABLE_SYSTEMD_NOTIFY "Enable systemd notify" ON) + if (ENABLE_SYSTEMD_NOTIFY STREQUAL "ON") + add_definitions(-DSYSTEMD_NOTIFY) +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index 48c1bad0..70a8ac91 100644 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -15,6 +15,12 @@ if (ENABLE_SHIM_V2) + ) + endif() + ++if (ENABLE_NRI) ++ list(APPEND CHECKED_INCLUDE_DIRS ++ ${NRI_INCLUDE_DIR} ++ ) ++endif() ++ + if (GRPC_CONNECTOR) + list(APPEND CHECKED_INCLUDE_DIRS + ${GRPC_INCLUDE_DIR} +@@ -168,6 +174,10 @@ if (ENABLE_SHIM_V2) + target_link_libraries(isulad ${LIBSHIM_V2_LIBRARY}) + endif() + ++if (ENABLE_NRI) ++ target_link_libraries(isulad ${LIBISULA_NRI_LIBRARY}) ++endif() ++ + if (ENABLE_EMBEDDED_IMAGE) + target_link_libraries(isulad ${SQLITE3_LIBRARY}) + endif() +diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt +index 29af3dca..801f5797 100644 +--- a/src/daemon/CMakeLists.txt ++++ b/src/daemon/CMakeLists.txt +@@ -36,6 +36,15 @@ if (ENABLE_CRI_API_V1) + list (APPEND local_daemon_incs + ${SANDBOX_INCS} + ) ++ if (ENABLE_NRI) ++ add_subdirectory(nri) ++ list (APPEND local_daemon_srcs ++ ${NRI_SRCS} ++ ) ++ list (APPEND local_daemon_incs ++ ${NRI_INCS} ++ ) ++ endif() + endif() + + set(DAEMON_SRCS +diff --git a/src/daemon/common/CMakeLists.txt b/src/daemon/common/CMakeLists.txt +index e88578dd..ffeed4b3 100644 +--- a/src/daemon/common/CMakeLists.txt ++++ b/src/daemon/common/CMakeLists.txt +@@ -10,12 +10,16 @@ if (GRPC_CONNECTOR) + endif() + + add_subdirectory(cgroup) ++if (ENABLE_NRI) ++ add_subdirectory(nri) ++endif() + + set(local_daemon_common_srcs ${daemon_common_top_srcs}) + + set(DAEMON_COMMON_SRCS + ${COMMON_CRI_SRCS} + ${COMMON_CGROUP_SRCS} ++ ${COMMON_NRI_SRCS} + ${local_daemon_common_srcs} + PARENT_SCOPE + ) +@@ -23,6 +27,7 @@ set(DAEMON_COMMON_SRCS + set(DAEMON_COMMON_INCS + ${COMMON_CRI_INCS} + ${COMMON_CGROUP_INCS} ++ ${COMMON_NRI_INCS} + ${CMAKE_CURRENT_SOURCE_DIR} + PARENT_SCOPE + ) +diff --git a/src/daemon/common/nri/CMakeLists.txt b/src/daemon/common/nri/CMakeLists.txt +new file mode 100644 +index 00000000..512ba694 +--- /dev/null ++++ b/src/daemon/common/nri/CMakeLists.txt +@@ -0,0 +1,13 @@ ++# get current directory sources files ++aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_common_nri_srcs) ++set(local_common_nri_incs ${CMAKE_CURRENT_SOURCE_DIR}) ++ ++set(COMMON_NRI_SRCS ++ ${local_common_nri_srcs} ++ PARENT_SCOPE ++) ++ ++set(COMMON_NRI_INCS ++ ${local_common_nri_incs} ++ PARENT_SCOPE ++) +diff --git a/src/daemon/common/nri/nri_convert.h b/src/daemon/common/nri/nri_convert.h +new file mode 100644 +index 00000000..883f7c41 +--- /dev/null ++++ b/src/daemon/common/nri/nri_convert.h +@@ -0,0 +1,36 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-03-16 ++ * Description: provide nri convert functions ++ *********************************************************************************/ ++#ifndef DAEMON_COMMON_NRI_NRI_CONVERT_H ++#define DAEMON_COMMON_NRI_NRI_CONVERT_H ++#include <map> ++#include <memory> ++#include <string> ++#include <vector> ++ ++#include "isula_libutils/nri_pod_sandbox.h" ++#include "isula_libutils/nri_container.h" ++#include "isula_libutils/nri_mount.h" ++#include "isula_libutils/nri_linux_resources.h" ++ ++#include "sandbox.h" ++#include "api_v1.pb.h" ++ ++auto PodSandboxToNRI(const std::shared_ptr<const sandbox::Sandbox> &sandbox, nri_pod_sandbox *pod) -> bool; ++auto ContainerToNRIByConConfig(const runtime::v1::ContainerConfig &containerConfig, nri_container *con) -> bool; ++auto ContainerToNRIByID(const std::string &id, nri_container *con) -> bool; ++auto PodSandboxesToNRI(const std::vector<std::unique_ptr<sandbox::Sandbox>> &arrs, nri_pod_sandbox **pod, int pod_len) -> bool; ++ ++auto LinuxResourcesFromNRI(const nri_linux_resources *src, runtime::v1::LinuxContainerResources &resources) -> bool; ++#endif // DAEMON_COMMON_NRI_NRI_CONVERT_H +diff --git a/src/daemon/common/nri/nri_spec.h b/src/daemon/common/nri/nri_spec.h +new file mode 100644 +index 00000000..e7c5035d +--- /dev/null ++++ b/src/daemon/common/nri/nri_spec.h +@@ -0,0 +1,24 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-07-17 ++ * Description: provide nri oci functions ++ *********************************************************************************/ ++ ++#ifndef DAEMON_COMMON_NRI_NRI_SPEC_H ++#define DAEMON_COMMON_NRI_NRI_SPEC_H ++ ++#include <isula_libutils/oci_runtime_spec.h> ++#include <isula_libutils/nri_container_adjustment.h> ++ ++int nri_adjust_oci_spec(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec); ++ ++#endif // DAEMON_COMMON_NRI_NRI_SPEC_H +\ No newline at end of file +diff --git a/src/daemon/common/nri/nri_utils.h b/src/daemon/common/nri/nri_utils.h +new file mode 100644 +index 00000000..3aa50ae4 +--- /dev/null ++++ b/src/daemon/common/nri/nri_utils.h +@@ -0,0 +1,71 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-07-17 ++ * Description: provide nri utils functions ++ *********************************************************************************/ ++#ifndef DAEMON_COMMON_NRI_NRI_UTILS_H ++#define DAEMON_COMMON_NRI_NRI_UTILS_H ++ ++#include <stdio.h> ++#include <stdlib.h> ++#include <stddef.h> ++ ++#include <isula_libutils/nri_create_container_request.h> ++#include <isula_libutils/nri_create_container_response.h> ++#include <isula_libutils/nri_update_container_request.h> ++#include <isula_libutils/nri_update_container_response.h> ++#include <isula_libutils/nri_container_update.h> ++#include <isula_libutils/nri_mount.h> ++ ++#include <isula_libutils/container_config.h> ++#include <isula_libutils/host_config.h> ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++typedef enum { ++ UNKNOWN = 0, ++ RUN_POD_SANDBOX = 1, ++ STOP_POD_SANDBOX = 2, ++ REMOVE_POD_SANDBOX = 3, ++ CREATE_CONTAINER = 4, ++ POST_CREATE_CONTAINER = 5, ++ START_CONTAINER = 6, ++ POST_START_CONTAINER = 7, ++ UPDATE_CONTAINER = 8, ++ POST_UPDATE_CONTAINER = 9, ++ STOP_CONTAINER = 10, ++ REMOVE_CONTAINER = 11, ++ LAST = 12, ++} NRI_Event; ++ ++bool copy_nri_mount(const nri_mount *src, nri_mount **dest); ++bool copy_nri_key_value(const nri_key_value *src, nri_key_value **dest); ++bool copy_nri_hook(const nri_hook *src, nri_hook **dest); ++bool copy_nri_posix_rlimit(const nri_posix_rlimit *src, nri_posix_rlimit **dest); ++bool copy_nri_linux_resources(const nri_linux_resources *src, nri_linux_resources **dest); ++bool copy_nri_linux_cpu(const nri_linux_cpu *src, nri_linux_cpu **dest); ++ ++bool is_marked_for_removal(const char* key, char **out); ++ ++bool realloc_and_copy_nri_hooks(nri_hook **targetHooks, size_t targetSize, const nri_hook **sourceHooks, size_t sourceLen); ++ ++bool init_nri_container_adjust(nri_container_adjustment **adjust); ++bool init_nri_container_update(nri_container_update **update, const char *id, uint8_t ignore_failure); ++bool init_nri_linux_resources(nri_linux_resources **resources); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif // DAEMON_COMMON_NRI_NRI_UTILS_H +\ No newline at end of file +diff --git a/src/daemon/config/isulad_config.h b/src/daemon/config/isulad_config.h +index f29cd564..c57ca8d6 100644 +--- a/src/daemon/config/isulad_config.h ++++ b/src/daemon/config/isulad_config.h +@@ -40,6 +40,22 @@ struct isulad_conf { + #define DEFAULT_SANDBOXER_NAME "shim" + char *conf_get_sandbox_rootpath(void); + char *conf_get_sandbox_statepath(void); ++ ++#ifdef ENABLE_NRI ++#define DEFAULT_SOCKET_PATH "/var/run/nri/nri.sock" ++#define DEFAULT_MUX_SOCKET_PATH "/var/run/nri/nri_mux.sock" ++#define DEFAULT_PLUGIN_PATH "/opt/nri/plugins" ++#define DEFAULT_PLUGIN_CONFIG_PATH "/etc/nri/conf.d" ++#define DEFAULT_PLUGIN_REGISTRY_TIMEOUT 5 ++#define DEFAULT_PLUGIN_REQUST_TIMEOUT 2 ++bool conf_get_nri_support(void); ++bool conf_get_nri_external_support(void); ++char *conf_get_nri_plugin_config_path(void); ++char *conf_get_nri_plugin_path(void); ++char *conf_get_socket_path(void); ++uint64_t conf_get_nri_plugin_registration_timeout(void); ++uint64_t conf_get_nri_plugin_requst_timeout(void); ++#endif + #endif + + char *conf_get_isulad_pidfile(void); +diff --git a/src/daemon/modules/api/specs_api.h b/src/daemon/modules/api/specs_api.h +index 0999836b..6a1cd776 100644 +--- a/src/daemon/modules/api/specs_api.h ++++ b/src/daemon/modules/api/specs_api.h +@@ -58,6 +58,9 @@ int spec_module_init(void); + #ifdef ENABLE_CDI + int defs_process_add_multiple_env(defs_process *dp, const char **envs, size_t env_len); + int spec_add_multiple_process_env(oci_runtime_spec *oci_spec, const char **envs, size_t env_len); ++#endif/* ENABLE_CDI */ ++ ++#if defined (ENABLE_CDI) || defined(ENABLE_NRI) + int spec_add_device(oci_runtime_spec *oci_spec, defs_device *device); + int spec_add_linux_resources_device(oci_runtime_spec *oci_spec, bool allow, const char *dev_type, + int64_t major, int64_t minor, const char *access); +@@ -66,7 +69,12 @@ int spec_add_mount(oci_runtime_spec *oci_spec, defs_mount *mnt); + int spec_add_prestart_hook(oci_runtime_spec *oci_spec, defs_hook *prestart_hook); + int spec_add_poststart_hook(oci_runtime_spec *oci_spec, defs_hook *poststart_hook); + int spec_add_poststop_hook(oci_runtime_spec *oci_spec, defs_hook *poststop_hook); +-#endif /* ENABLE_CDI */ ++#endif /* ENABLE_CDI || ENABLE_NRI */ ++ ++#ifdef ENABLE_NRI ++int spec_add_linux_resources_hugepage_limit(oci_runtime_spec *oci_spec, const char *page_size, uint64_t limit); ++int spec_add_linux_resources_rlimit(oci_runtime_spec *oci_spec, const char *type, uint64_t hard, uint64_t soft); ++#endif /* ENABLE_NRI */ + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index f0538e26..1fd9e5a8 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -2608,7 +2608,7 @@ int spec_module_init(void) + static int add_env(defs_process *dp, const char *env, const char *key) + { + size_t i; +- ++ + for (i = 0; i < dp->env_len; i++) { + __isula_auto_free char *oci_key = NULL; + if (util_valid_split_env(dp->env[i], &oci_key, NULL) < 0) { +@@ -2684,7 +2684,9 @@ int spec_add_multiple_process_env(oci_runtime_spec *oci_spec, const char **envs, + + return ret; + } ++#endif /* ENABLE_CDI */ + ++#if defined (ENABLE_CDI) || defined(ENABLE_NRI) + int spec_add_device(oci_runtime_spec *oci_spec, defs_device *device) + { + int ret = 0; +@@ -2817,4 +2819,65 @@ SPEC_ADD_HOOKS_ITEM_DEF(prestart) + SPEC_ADD_HOOKS_ITEM_DEF(poststart) + SPEC_ADD_HOOKS_ITEM_DEF(poststop) + +-#endif /* ENABLE_CDI */ +\ No newline at end of file ++#endif /* ENABLE_CDI || ENABLE_NRI */ ++ ++#ifdef ENABLE_NRI ++int spec_add_linux_resources_hugepage_limit(oci_runtime_spec *oci_spec, const char *page_size, uint64_t limit) ++{ ++ int ret = 0; ++ defs_resources_hugepage_limits_element *hugepage_limit = NULL; ++ ++ ret = make_sure_oci_spec_linux_resources(oci_spec); ++ if (ret < 0) { ++ return -1; ++ } ++ ++ hugepage_limit = util_common_calloc_s(sizeof(*hugepage_limit)); ++ if (hugepage_limit == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ hugepage_limit->page_size = util_strdup_s(page_size); ++ hugepage_limit->limit = limit; ++ ++ if (util_mem_realloc((void **)&oci_spec->linux->resources->hugepage_limits, ++ (oci_spec->linux->resources->hugepage_limits_len + 1) * sizeof(defs_resources_hugepage_limits_element *), ++ (void *)oci_spec->linux->resources->hugepage_limits, ++ oci_spec->linux->resources->hugepage_limits_len * sizeof(defs_resources_hugepage_limits_element *)) != 0) { ++ ERROR("Out of memory"); ++ free_defs_resources_hugepage_limits_element(hugepage_limit); ++ return -1; ++ } ++ oci_spec->linux->resources->hugepage_limits[oci_spec->linux->resources->hugepage_limits_len] = hugepage_limit; ++ oci_spec->linux->resources->hugepage_limits_len++; ++ ++ return 0; ++} ++ ++int spec_add_linux_resources_rlimit(oci_runtime_spec *oci_spec, const char *type, uint64_t hard, uint64_t soft) ++{ ++ int ret = 0; ++ defs_process_rlimits_element *rlimit = NULL; ++ ++ ret = make_sure_oci_spec_linux_resources(oci_spec); ++ if (ret < 0) { ++ return -1; ++ } ++ ++ rlimit = util_common_calloc_s(sizeof(*rlimit)); ++ if (rlimit == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ rlimit->type = util_strdup_s(type); ++ rlimit->hard = hard; ++ rlimit->soft = soft; ++ ++ if (do_merge_one_ulimit_override(oci_spec, rlimit) != 0) { ++ ERROR("Failed to merge one nri ulimit to oci spec"); ++ return -1; ++ } ++ ++ return 0; ++} ++#endif /* ENABLE_NRI */ +\ No newline at end of file +diff --git a/src/daemon/nri/CMakeLists.txt b/src/daemon/nri/CMakeLists.txt +new file mode 100644 +index 00000000..361d79f8 +--- /dev/null ++++ b/src/daemon/nri/CMakeLists.txt +@@ -0,0 +1,6 @@ ++# get current directory sources files ++aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} nri_top_srcs) ++ ++set(NRI_INCS ${CMAKE_CURRENT_SOURCE_DIR} PARENT_SCOPE) ++ ++set(NRI_SRCS ${nri_top_srcs} PARENT_SCOPE) +diff --git a/src/daemon/nri/nri_adaption.h b/src/daemon/nri/nri_adaption.h +new file mode 100644 +index 00000000..7f0640df +--- /dev/null ++++ b/src/daemon/nri/nri_adaption.h +@@ -0,0 +1,123 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-03-15 ++ * Description: provide plugin manager(NRI adaption) class definition ++ *********************************************************************************/ ++ ++#ifndef DAEMON_NRI_PLUGIN_NRI_ADAPTION_H ++#define DAEMON_NRI_PLUGIN_NRI_ADAPTION_H ++ ++// #include "read_write_lock.h" ++#include <isula_libutils/nri_update_containers_request.h> ++#include <isula_libutils/nri_update_containers_response.h> ++ ++#include "plugin.h" ++#include "sandbox.h" ++#include "v1_cri_container_manager_service.h" ++ ++const std::string PluginNameEnv = "NRI_PLUGIN_NAME"; ++const std::string PluginIdxEnv = "NRI_PLUGIN_IDX"; ++const std::string PluginSocketEnv = "NRI_PLUGIN_SOCKET"; ++ ++struct nri_plugin_exec_args_t { ++ const char *workdir; ++ const char *cmd; ++ const char *name; ++ const char *index; ++ const uint32_t sockFd; ++}; ++ ++class NRIAdaptation { ++public: ++ // Singleton ++ static NRIAdaptation *GetInstance() noexcept; ++ ++ // initialize value ++ auto Init(Errors &error) -> bool; ++ ++ auto GetSockpath(std::vector<std::string> &paths) -> bool; ++ ++ // Stop plugins. ++ auto StopPlugins() -> bool; ++ ++ void RemoveClosedPlugins(); ++ ++ auto GetPluginByIndex(const std::string &index) -> std::shared_ptr<NRIPlugin>; ++ void AddPluginByIndex(const std::string &index, std::shared_ptr<NRIPlugin> plugin); ++ void RemovePluginByIndex(const std::string &index); ++ ++ auto RunPodSandbox(std::shared_ptr<const sandbox::Sandbox> sandbox, Errors &error) ->bool; ++ auto StopPodSandbox(std::shared_ptr<const sandbox::Sandbox> sandbox, Errors &error) ->bool; ++ auto RemovePodSandbox(std::shared_ptr<const sandbox::Sandbox> sandbox, Errors &error) ->bool; ++ auto CreateContainer(std::shared_ptr<const sandbox::Sandbox> sandbox, const std::string &conId, const runtime::v1::ContainerConfig &containerConfig, nri_container_adjustment **adjust, Errors &error) -> bool; ++ auto PostCreateContainer(const std::string &conId, Errors &error) ->bool; ++ auto UndoCreateContainer(std::shared_ptr<const sandbox::Sandbox> sandbox, const std::string &conId, Errors &error) -> bool; ++ auto StartContainer(const std::string &conId, Errors &error) ->bool; ++ auto PostStartContainer(const std::string &conId, Errors &error) ->bool; ++ auto UpdateContainer(const std::string &conId, Errors &error) ->bool; ++ auto PostUpdateContainer(const std::string &conId, Errors &error) ->bool; ++ auto StopContainer(const std::string &conId, Errors &error) ->bool; ++ auto RemoveContainer(const std::string &conId, Errors &error) ->bool; ++ auto StateChange(nri_state_change_event *evt, Errors &error) ->bool; ++ auto updateContainers(const nri_update_containers_request *req, nri_update_containers_response **resp) ->bool; ++ ++ auto NewExternalPlugin(int fd) -> bool; ++private: ++ NRIAdaptation() = default; ++ NRIAdaptation(const NRIAdaptation &other) = delete; ++ NRIAdaptation &operator=(const NRIAdaptation &) = delete; ++ virtual ~NRIAdaptation(); ++ ++ auto StartPlugin() -> bool; ++ auto NewLaunchedPlugin(const std::shared_ptr<NRIPlugin> &) -> bool; ++ auto DiscoverPlugins(std::map<std::string, std::shared_ptr<NRIPlugin>> &map) -> bool; ++ // Synchronizing NRI (plugin) with current runtime state ++ auto SyncPlugin() -> bool; ++ ++ auto SortPlugins() -> bool; ++ void GetClosedPlugins(std::vector<std::string> &closedPlugin); ++ ++ auto IsSupport() -> bool; ++ ++ auto ApplyUpdates(const std::vector<nri_container_update *> &update, std::vector<nri_container_update *> &failed, bool getFailed, ++ Errors &error) -> bool; ++ ++ auto NRIPodSandbox(const std::shared_ptr<const sandbox::Sandbox> &sandbox, Errors& error) -> std::unique_ptr<CStructWrapper<nri_pod_sandbox>>; ++ auto NRIContainerByConConfig(const std::shared_ptr<const sandbox::Sandbox> &sandbox, const runtime::v1::ContainerConfig &containerConfig, Errors& error) -> std::unique_ptr<CStructWrapper<nri_container>>; ++ auto NRIContainerByID(const std::string &id, Errors& error) -> std::unique_ptr<CStructWrapper<nri_container>>; ++ ++ auto GetNRIPluginConfigPath(void) -> std::string; ++ auto GetNRIPluginPath(void) -> std::string; ++ auto GetNRISockPath(void) -> std::string; ++private: ++ RWMutex m_mutex; ++ static std::atomic<NRIAdaptation *> m_instance; ++ bool m_support; ++ bool m_external_support; ++ std::string m_version; ++ std::string m_sock_path; ++ std::string m_pluginConfigPath; ++ std::string m_pluginPath; ++ std::vector<std::string> m_socketPathArr; ++ std::string m_disableConnections; ++ // id --> NRIPlugin map ++ std::map<std::string, std::shared_ptr<NRIPlugin>> m_storeMap; ++ // TODO:plugin monitor thread id?? ++ // shutdown() to clean resource ++ // init to create thread ++ // todo: if Singleton? ++ std::unique_ptr<CRIV1::ContainerManagerService> m_containerManager; ++ int64_t m_plugin_registration_timeout; ++ int64_t m_plugin_requst_timeout; ++}; ++ ++#endif // DAEMON_NRI_PLUGIN_NRI_ADAPTION_H +\ No newline at end of file +diff --git a/src/daemon/nri/nri_helpers.h b/src/daemon/nri/nri_helpers.h +new file mode 100644 +index 00000000..06ee8419 +--- /dev/null ++++ b/src/daemon/nri/nri_helpers.h +@@ -0,0 +1,63 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2017-2019. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-07-13 ++ * Description: provide nri helpers functions ++ *********************************************************************************/ ++#ifndef DAEMON_NRI_PLUGIN_NRI_HELPERS_H ++#define DAEMON_NRI_PLUGIN_NRI_HELPERS_H ++ ++#include <map> ++#include <memory> ++#include <string> ++#include <vector> ++ ++#include <isula_libutils/nri_create_container_request.h> ++#include <isula_libutils/nri_create_container_response.h> ++#include <isula_libutils/nri_update_container_request.h> ++#include <isula_libutils/nri_update_container_response.h> ++#include <isula_libutils/nri_container_update.h> ++#include <isula_libutils/nri_mount.h> ++ ++#include <isula_libutils/container_config.h> ++#include <isula_libutils/host_config.h> ++ ++#include "errors.h" ++ ++namespace NRIHelpers { ++std::string MarkForRemoval(const std::string &key); ++ ++auto GetPluginConfig(std::string &idx, std::string &name, std::string &config) -> bool; ++ ++std::string GenerateRandomExternalName(void); ++ ++bool CheckPluginIndex(const std::string &idx); ++ ++template <typename T> ++void freeArray(T ** &arr, int size) ++{ ++ if (arr == NULL) { ++ return; ++ } ++ ++ for (int i = 0; i < size; i++) { ++ if (arr[i] == NULL) { ++ return; ++ } ++ free(arr[i]); ++ } ++ ++ free(arr); ++ arr = NULL; ++} ++}; // namespace NRIHelpers ++ ++#endif // DAEMON_NRI_PLUGIN_NRI_HELPERS_H +diff --git a/src/daemon/nri/nri_plugin_ops.h b/src/daemon/nri/nri_plugin_ops.h +new file mode 100644 +index 00000000..37d437d2 +--- /dev/null ++++ b/src/daemon/nri/nri_plugin_ops.h +@@ -0,0 +1,38 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtaoo ++ * Create: 2024-03-26 ++ * Description: provide nri plugin api definition ++ ******************************************************************************/ ++ ++#ifndef DAEMON_NRI_PLUGIN_OPS_H ++#define DAEMON_NRI_PLUGIN_OPS_H ++ ++#include <isula_libutils/nri_update_containers_request.h> ++#include <isula_libutils/nri_update_containers_response.h> ++#include <isula_libutils/nri_register_plugin_request.h> ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++bool nri_adaption_init(void); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++int nri_update_containers(const nri_update_containers_request *request, nri_update_containers_response **response); ++int nri_registry_containers(const nri_register_plugin_request *request); ++ ++int nri_external_plugin_connect(int fd); ++ ++#endif // DAEMON_NRI_PLUGIN_OPS_H +diff --git a/src/daemon/nri/nri_result.h b/src/daemon/nri/nri_result.h +new file mode 100644 +index 00000000..f2896ea0 +--- /dev/null ++++ b/src/daemon/nri/nri_result.h +@@ -0,0 +1,114 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-06-29 ++ * Description: provide nri result definition ++ *********************************************************************************/ ++ ++#ifndef DAEMON_NRI_PLUGIN_NRI_RESULT_H ++#define DAEMON_NRI_PLUGIN_NRI_RESULT_H ++ ++#include <map> ++ ++#include <isula_libutils/nri_create_container_request.h> ++#include <isula_libutils/nri_create_container_response.h> ++#include <isula_libutils/nri_update_container_request.h> ++#include <isula_libutils/nri_update_container_response.h> ++#include <isula_libutils/nri_container_update.h> ++#include <isula_libutils/nri_mount.h> ++ ++#include "api_v1.pb.h" ++#include "nri_helpers.h" ++#include "nri_utils.h" ++ ++using EventMask = std::int32_t; ++ ++const EventMask ValidEvents = (1 << (LAST - 1)) - 1; ++ ++struct owners { ++ std::map<std::string, std::string> annotations; ++ std::map<std::string, std::string> mounts; ++ std::map<std::string, std::string> devices; ++ std::map<std::string, std::string> env; ++ std::string memLimit; ++ std::string memReservation; ++ std::string memSwapLimit; ++ std::string memKernelLimit; ++ std::string memTCPLimit; ++ std::string memSwappiness; ++ std::string memDisableOomKiller; ++ std::string memUseHierarchy; ++ std::string cpuShares; ++ std::string cpuQuota; ++ std::string cpuPeriod; ++ std::string cpuRealtimeRuntime; ++ std::string cpuRealtimePeriod; ++ std::string cpusetCpus; ++ std::string cpusetMems; ++ std::map<std::string, std::string> hugepageLimits; ++ std::string blockioClass; ++ std::string rdtClass; ++ std::map<std::string, std::string> unified; ++ std::string cgroupsPath; ++ std::map<std::string, std::string> rlimits; ++}; ++ ++struct resultReply { ++ nri_container_adjustment* adjust; ++ std::vector<nri_container_update*> update; ++}; ++ ++using resultOwners = std::map<std::string, owners>; ++ ++class pluginResult { ++public: ++ pluginResult() = default; ++ ++ ~pluginResult() = default; ++ ++ auto InitByConId(std::string conId) -> bool; ++ auto InitByUpdateReq(nri_update_container_request *req) -> bool; ++ ++ auto GetReplyUpdate() -> std::vector<nri_container_update *>; ++ auto GetReplyAdjust() -> nri_container_adjustment *; ++ ++ auto Apply(int32_t event, nri_container_adjustment *adjust, nri_container_update **update, size_t update_len, ++ const std::string &plugin) -> bool; ++ auto Update(nri_container_update **updates, size_t update_len, const std::string &plugin) -> bool; ++ ++private: ++ auto GetContainerUpdate(nri_container_update *update, const std::string &plugin, nri_container_update **out) -> bool; ++ auto UpdateResources(nri_container_update *reply, nri_container_update *u, const std::string &plugin) -> bool; ++ ++ auto InitReply(void) -> bool; ++ ++ auto Adjust(nri_container_adjustment *adjust, const std::string &plugin) -> bool; ++ ++ auto AdjustAnnotations(json_map_string_string *annos, const std::string &plugin) -> bool; ++ auto AdjustMounts(nri_mount **mounts, size_t mounts_size, const std::string &plugin) -> bool; ++ auto AdjustEnv(nri_key_value **envs, size_t envs_size, const std::string &plugin) -> bool; ++ auto AdjustHooks(nri_hooks *hooks, const std::string &plugin) -> bool; ++ auto AdjustDevices(nri_linux_device **devices, size_t devices_size, const std::string &plugin) -> bool; ++ auto AdjustResources(nri_linux_resources *resources, const std::string &plugin) -> bool; ++ bool ClaimAndCopyResources(nri_linux_resources *src, std::string &id, const std::string &plugin, ++ nri_linux_resources *dest); ++ auto AdjustCgroupsPath(char *path, const std::string &plugin) -> bool; ++ auto AdjustRlimits(nri_posix_rlimit **rlimits, size_t rlimits_len, const std::string &plugin) -> bool; ++ ++private: ++ std::string m_conId; ++ nri_linux_resources *m_update_req; ++ resultReply m_reply; ++ std::map<std::string, nri_container_update *> m_updates; ++ resultOwners m_owners; ++}; ++ ++#endif +\ No newline at end of file +diff --git a/src/daemon/nri/plugin.h b/src/daemon/nri/plugin.h +new file mode 100644 +index 00000000..f60a9b3d +--- /dev/null ++++ b/src/daemon/nri/plugin.h +@@ -0,0 +1,95 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-03-15 ++ * Description: provide plugin class definition ++ *********************************************************************************/ ++ ++#ifndef DAEMON_NRI_PLUGIN_PLUGIN_H ++#define DAEMON_NRI_PLUGIN_PLUGIN_H ++ ++#include <condition_variable> ++ ++#include <isula_libutils/nri_stop_container_request.h> ++#include <isula_libutils/nri_stop_container_response.h> ++#include <isula_libutils/nri_state_change_event.h> ++ ++#include "errors.h" ++#include "read_write_lock.h" ++#include "nri_result.h" ++ ++const std::string NRIRruntime = "v2"; ++const std::string NRIVersion = "2.0.0-beta.2+unknown"; ++ ++class NRIPlugin { ++public: ++ // init client conn ++ NRIPlugin(std::string &idx, std::string &name, std::string &config); ++ NRIPlugin(int fd, std::string &name); ++ // todo: close client conn ?? or single close func? ++ virtual ~NRIPlugin() = default; ++ // wait for plugin to register, then configure it. ++ auto Start(int64_t registry_timeout, int64_t request_timeout) -> bool; ++ // close a plugin shutting down its multiplexed ttrpc connections. ++ auto Close(void) -> bool; ++ // stop a plugin (if it was launched by us) ++ auto Stop(void) -> bool; ++ ++ // Name returns a string indentication for the plugin. ++ auto GetName(void) -> const std::string &; ++ auto GetIndex(void) -> const std::string &; ++ auto GetPeerSockFd(void) -> uint32_t; ++ auto GetQualifiedName(void) -> std::string; ++ ++ void SetReady(void); ++ void SetPid(int pid); ++ ++ auto IsClose(void) -> bool; ++ ++ auto CreateSocketPair(void) -> bool; ++ ++ auto Configure(Errors &error) -> bool; ++ // Only called in external plugin scenario ++ auto Synchronize(std::vector<std::unique_ptr<nri_pod_sandbox>> pods, ++ std::vector<std::unique_ptr<nri_container>> &containers, nri_container_update ***update, size_t update_len, ++ Errors &error) -> bool; ++ auto CreateContainer(nri_create_container_request *req, nri_create_container_response **resp, Errors &error) -> bool; ++ auto UpdateContainer(nri_update_container_request *req, nri_update_container_response **resp, Errors &error) -> bool; ++ auto StopContainer(nri_stop_container_request *req, nri_stop_container_response **resp, Errors &error) -> bool; ++ auto StateChange(nri_state_change_event *evt, Errors &error) -> bool; ++ ++private: ++ auto Connect(int64_t timeout) -> bool; ++ ++ auto WaitForReady(int64_t timeout) -> bool; ++ auto IsSetEvent(EventMask e) -> bool; ++ ++private: ++ RWMutex m_mutex; ++ bool m_external; ++ std::string m_idx; ++ std::string m_name; ++ std::string m_config; ++ int m_pid; ++ std::string m_cmd; ++ std::vector<uint32_t> m_sockFds; ++ std::string m_localFileName; ++ std::string m_peerFileName; ++ // TODO:zhontao monitor? ++ bool m_closed; ++ std::mutex m_readyMutex; ++ bool m_ready; ++ std::condition_variable m_condition; ++ EventMask m_events; ++}; ++ ++ ++#endif // DAEMON_NRI_PLUGIN_PLUGIN_H +\ No newline at end of file +diff --git a/src/utils/cpputils/transform.cc b/src/utils/cpputils/transform.cc +index 74c178a9..51c154fb 100644 +--- a/src/utils/cpputils/transform.cc ++++ b/src/utils/cpputils/transform.cc +@@ -89,4 +89,20 @@ void CharArrayToStringVector(const char **src, size_t len, std::vector<std::stri + } + } + ++ ++auto RepeatedPtrFieldToCharArray(const google::protobuf::RepeatedPtrField<std::string> &ptrs) -> char ** ++{ ++ size_t len = ptrs.size(); ++ char **result = (char **)util_smart_calloc_s(sizeof(char *), (len + 1)); ++ if (result == nullptr) { ++ return nullptr; ++ } ++ size_t i {}; ++ for (const auto &it : ptrs) { ++ result[i++] = util_strdup_s(it.c_str()); ++ } ++ ++ return result; ++} ++ + } // namespace Transform +diff --git a/src/utils/cpputils/transform.h b/src/utils/cpputils/transform.h +index 476f39a6..57c58d9e 100644 +--- a/src/utils/cpputils/transform.h ++++ b/src/utils/cpputils/transform.h +@@ -35,6 +35,8 @@ auto StringVectorToCharArray(std::vector<std::string> &strVec) -> char **; + + void CharArrayToStringVector(const char **src, size_t len, std::vector<std::string> &dest); + ++auto RepeatedPtrFieldToCharArray(const google::protobuf::RepeatedPtrField<std::string> &ptrs) -> char **; ++ + }; // namespace Transform + + #endif // UTILS_CPPUTILS_TRANSFORM_H +\ No newline at end of file +diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c +index 9a33f935..f81a9141 100644 +--- a/src/utils/cutils/utils.c ++++ b/src/utils/cutils/utils.c +@@ -1699,3 +1699,10 @@ int util_chown_for_shm(const char *shm_path, const char *user_remap) + + return 0; + } ++ ++void set_child_process_pdeathsig(void) ++{ ++ if (prctl(PR_SET_PDEATHSIG, SIGKILL) < 0) { ++ SYSERROR("Failed to set child process pdeathsig"); ++ } ++} +\ No newline at end of file +diff --git a/src/utils/cutils/utils.h b/src/utils/cutils/utils.h +index ce0ca703..76684ed3 100644 +--- a/src/utils/cutils/utils.h ++++ b/src/utils/cutils/utils.h +@@ -409,6 +409,8 @@ int util_create_shm_path(const char *spath, const int64_t shm_size); + + int util_chown_for_shm(const char *shm_path, const char *user_remap); + ++void set_child_process_pdeathsig(void); ++ + /** + * retry_cnt: max count of call cb; + * interval_us: how many us to sleep, after call cb; +diff --git a/src/utils/tar/util_archive.c b/src/utils/tar/util_archive.c +index 204dab83..985e0f16 100644 +--- a/src/utils/tar/util_archive.c ++++ b/src/utils/tar/util_archive.c +@@ -812,13 +812,6 @@ static void close_archive_pipes_fd(int *pipes, size_t pipe_size) + } + } + +-static void set_child_process_pdeathsig(void) +-{ +- if (prctl(PR_SET_PDEATHSIG, SIGKILL) < 0) { +- SYSERROR("Failed to set child process pdeathsig"); +- } +-} +- + int archive_unpack(const struct io_read_wrapper *content, const char *dstdir, const struct archive_options *options, + const char *root_dir, char **errmsg) + { +-- +2.25.1 + diff --git a/0119-skip-calling-cni-plugin-cleanup-when-network-namespa.patch b/0119-skip-calling-cni-plugin-cleanup-when-network-namespa.patch new file mode 100644 index 0000000..4b33de5 --- /dev/null +++ b/0119-skip-calling-cni-plugin-cleanup-when-network-namespa.patch @@ -0,0 +1,51 @@ +From ee720f966fdf14a99b8ebc685f3948bb8b29ba73 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Tue, 13 Aug 2024 10:56:44 +0800 +Subject: [PATCH 118/121] skip calling cni plugin cleanup when network + namespace is not mounted + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + .../entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc | 7 +++++++ + .../entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc | 7 +++++++ + 2 files changed, 14 insertions(+) + +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +index 2a458a6d..77faf48a 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +@@ -435,6 +435,13 @@ void PodSandboxManagerService::ClearCniNetwork(const std::shared_ptr<sandbox::Sa + return; + } + ++ // If the network namespace is not mounted, the network has been cleaned up ++ // and there is no need to call the cni plugin. ++ if (!util_detect_mounted(sandboxKey.c_str())) { ++ WARN("Network namespace %s not exist", sandboxKey.c_str()); ++ return; ++ } ++ + const auto config = sandbox->GetSandboxConfig(); + std::map<std::string, std::string> stdAnnos; + CRIHelpers::ProtobufAnnoMapToStd(config.annotations(), stdAnnos); +diff --git a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +index bc3f4031..5590827e 100644 +--- a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +@@ -848,6 +848,13 @@ auto PodSandboxManagerService::ClearCniNetwork(const std::string &realSandboxID, + goto cleanup; + } + ++ // If the network namespace is not mounted, the network has been cleaned up ++ // and there is no need to call the cni plugin. ++ if (!util_detect_mounted(netnsPath.c_str())) { ++ WARN("Network namespace %s not exist", netnsPath.c_str()); ++ goto cleanup; ++ } ++ + stdAnnos.insert(std::pair<std::string, std::string>(CRIHelpers::Constants::POD_SANDBOX_KEY, netnsPath)); + pluginErr.Clear(); + m_pluginManager->TearDownPod(ns, name, Network::DEFAULT_NETWORK_INTERFACE_NAME, realSandboxID, stdAnnos, +-- +2.25.1 + diff --git a/0120-nri-add-convert-and-utils-impl-for-nri.patch b/0120-nri-add-convert-and-utils-impl-for-nri.patch new file mode 100644 index 0000000..ba76c2e --- /dev/null +++ b/0120-nri-add-convert-and-utils-impl-for-nri.patch @@ -0,0 +1,2254 @@ +From c0d4b523c24e88b8c70cd3b121a46b7b3c841c17 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Tue, 13 Aug 2024 20:33:28 +0800 +Subject: [PATCH 119/121] [nri] add convert and utils impl for nri + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/common/nri/nri_convert.cc | 539 ++++++++++++++++++++++++ + src/daemon/common/nri/nri_convert.h | 9 +- + src/daemon/common/nri/nri_spec.c | 602 +++++++++++++++++++++++++++ + src/daemon/common/nri/nri_utils.c | 520 +++++++++++++++++++++++ + src/daemon/common/nri/nri_utils.h | 5 +- + src/daemon/config/isulad_config.c | 178 ++++++++ + src/daemon/modules/api/specs_api.h | 5 + + src/daemon/modules/spec/specs.c | 32 +- + src/daemon/nri/nri_adaption.h | 46 +- + src/daemon/nri/nri_helpers.cc | 93 +++++ + src/daemon/nri/nri_helpers.h | 2 +- + src/utils/cpputils/transform.cc | 2 +- + 12 files changed, 2002 insertions(+), 31 deletions(-) + create mode 100644 src/daemon/common/nri/nri_convert.cc + create mode 100644 src/daemon/common/nri/nri_spec.c + create mode 100644 src/daemon/common/nri/nri_utils.c + create mode 100644 src/daemon/nri/nri_helpers.cc + +diff --git a/src/daemon/common/nri/nri_convert.cc b/src/daemon/common/nri/nri_convert.cc +new file mode 100644 +index 00000000..7cce64ec +--- /dev/null ++++ b/src/daemon/common/nri/nri_convert.cc +@@ -0,0 +1,539 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-03-16 ++ * Description: provide nri convert functions ++ *********************************************************************************/ ++ ++#include "nri_convert.h" ++ ++#include "container_api.h" ++#include "v1_cri_helpers.h" ++#include "path.h" ++#include "transform.h" ++#include "nri_utils.h" ++ ++static int64_t DefaultOOMScoreAdj = 0; ++ ++static bool NRILinuxCpuFromCRI(const runtime::v1::LinuxContainerResources &config, nri_linux_cpu &cpu) ++{ ++ if (!config.cpuset_cpus().empty()) { ++ cpu.cpus = util_strdup_s(config.cpuset_cpus().c_str()); ++ } ++ ++ if (!config.cpuset_mems().empty()) { ++ cpu.mems = util_strdup_s(config.cpuset_mems().c_str()); ++ } ++ ++ cpu.period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if (cpu.period == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(cpu.period) = config.cpu_period(); ++ ++ cpu.quota = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (cpu.quota == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(cpu.quota) = config.cpu_quota(); ++ ++ cpu.shares = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if (cpu.shares == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(cpu.shares) = config.cpu_shares(); ++ ++ // consistent with other container engines, ++ // not obtained cpu.realtime_period & cpu.realtime_runtime ++ return true; ++} ++ ++static bool NRILinuxMemoryFromCRI(const runtime::v1::LinuxContainerResources &config, nri_linux_memory &memory) ++{ ++ memory.limit = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (memory.limit == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(memory.limit) = config.memory_limit_in_bytes(); ++ ++ // consistent with other container engines, ++ // not obtained other memory info ++ ++ return true; ++} ++ ++static bool NRIHugePageLimitFromCRI(const runtime::v1::LinuxContainerResources &config, nri_linux_resources &resources) ++{ ++ int i; ++ nri_hugepage_limit *tmp = nullptr; ++ ++ if (config.hugepage_limits_size() == 0) { ++ return true; ++ } ++ ++ resources.hugepage_limits = (nri_hugepage_limit **)util_smart_calloc_s(sizeof(nri_hugepage_limit *), ++ config.hugepage_limits_size()); ++ if (resources.hugepage_limits == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ for (i = 0; i < config.hugepage_limits_size(); i++) { ++ tmp = (nri_hugepage_limit *)util_common_calloc_s(sizeof(nri_hugepage_limit)); ++ if (tmp == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ tmp->page_size = util_strdup_s(config.hugepage_limits(i).page_size().c_str()); ++ tmp->limit = config.hugepage_limits(i).limit(); ++ resources.hugepage_limits[i] = tmp; ++ resources.hugepage_limits_len++; ++ tmp = nullptr; ++ } ++ return true; ++} ++ ++static auto NRILinuxResourcesFromCRI(const runtime::v1::LinuxContainerResources &config, ++ nri_linux_resources &resources) -> bool ++{ ++ if (!NRILinuxMemoryFromCRI(config, *resources.memory)) { ++ ERROR("Failed to transform memory to nri for container"); ++ return false; ++ } ++ ++ if (!NRILinuxCpuFromCRI(config, *resources.cpu)) { ++ ERROR("Failed to transform cpu to nri for container"); ++ return false; ++ } ++ ++ if (!NRIHugePageLimitFromCRI(config, resources)) { ++ ERROR("Failed to transform hugepage limits to nri for container"); ++ return false; ++ } ++ ++ // resources.blockio_class is not support ++ // resources.rdt_class is not support ++ // They are not standard fields in oci spec ++ ++ Errors tmpError; ++ ++ resources.unified = Transform::ProtobufMapToJsonMapForString(config.unified(), tmpError); ++ if (resources.unified == nullptr) { ++ ERROR("Failed to transform unified to nri for container : %s", tmpError.GetMessage().c_str()); ++ return false; ++ } ++ ++ // resources.devices is not set in pod ++ ++ return true; ++} ++ ++static auto NRILinuxFromCRI(const runtime::v1::LinuxPodSandboxConfig &config, nri_linux_pod_sandbox &linux) -> bool ++{ ++ if (!init_nri_linux_resources(&linux.pod_overhead)) { ++ ERROR("Failed to init nri linux overhead resources for pod"); ++ return false; ++ } ++ if (!init_nri_linux_resources(&linux.pod_resources)) { ++ ERROR("Failed to init nri linux resources resources for pod"); ++ return false; ++ } ++ if (config.has_overhead() && !NRILinuxResourcesFromCRI(config.overhead(), *linux.pod_overhead)) { ++ ERROR("Failed to transform overhead to nri for pod"); ++ return false; ++ } ++ ++ if (config.has_resources() && !NRILinuxResourcesFromCRI(config.resources(), *linux.pod_resources)) { ++ ERROR("Failed to transform resources to nri for pod"); ++ return false; ++ } ++ ++ linux.cgroup_parent = util_strdup_s(config.cgroup_parent().c_str()); ++ ++ // todo: other container engines get linux.cgroups_path/linux.resourses/linux.namespace from spec.linux, ++ // How does isulad get these values from CRI module? ++ return true; ++} ++ ++auto PodSandboxToNRI(const std::shared_ptr<const sandbox::Sandbox> &sandbox, nri_pod_sandbox &pod) -> bool ++{ ++ container_t *cont = nullptr; ++ Errors tmpError; ++ ++ cont = containers_store_get(sandbox->GetName().c_str()); ++ if (cont != nullptr) { ++ pod.pid = container_state_get_pid(cont->state); ++ container_unref(cont); ++ } ++ ++ pod.id = util_strdup_s(sandbox->GetId().c_str()); ++ pod.name = util_strdup_s(sandbox->GetName().c_str()); ++ if (sandbox->GetSandboxConfig().has_metadata()) { ++ pod.uid = util_strdup_s(sandbox->GetSandboxConfig().metadata().uid().c_str()); ++ pod._namespace = util_strdup_s(sandbox->GetSandboxConfig().metadata().namespace_().c_str()); ++ } ++ ++ ++ pod.labels = Transform::ProtobufMapToJsonMapForString(sandbox->GetSandboxConfig().labels(), tmpError); ++ if (pod.labels == nullptr) { ++ ERROR("Failed to transform labels to nri for pod : %s, : %s", pod.name, tmpError.GetMessage().c_str()); ++ return false; ++ } ++ ++ pod.annotations = Transform::ProtobufMapToJsonMapForString(sandbox->GetSandboxConfig().annotations(), tmpError); ++ if (pod.annotations == nullptr) { ++ ERROR("Failed to transform annotations to nri for pod : %s, : %s", pod.name, tmpError.GetMessage().c_str()); ++ return false; ++ } ++ ++ if (sandbox->GetSandboxConfig().has_linux()) { ++ pod.linux = (nri_linux_pod_sandbox *)util_common_calloc_s(sizeof(nri_linux_pod_sandbox)); ++ if (pod.linux == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ if (!NRILinuxFromCRI(sandbox->GetSandboxConfig().linux(), *pod.linux)) { ++ ERROR("Failed to transform linux to nri for pod : %s", pod.name); ++ return false; ++ } ++ } ++ ++ pod.runtime_handler = util_strdup_s(sandbox->GetRuntimeHandle().c_str()); ++ ++ return true; ++} ++ ++static auto CRIMountArrToNRI(const runtime::v1::ContainerConfig &containerConfig, nri_container &con) -> bool ++{ ++ size_t i, len; ++ ++ // get mount from cont ++ len = containerConfig.mounts_size(); ++ if (len == 0) { ++ return true; ++ } ++ con.mounts = (nri_mount **)util_smart_calloc_s(sizeof(nri_mount *), len); ++ if (con.mounts == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ nri_mount *tmp = nullptr; ++ ++ for (i = 0; i < len; i++) { ++ tmp = (nri_mount *)util_common_calloc_s(sizeof(nri_mount)); ++ if (tmp == nullptr) { ++ ERROR("Out of memory"); ++ goto error_out; ++ } ++ ++ if (containerConfig.mounts()[i].container_path().empty() || containerConfig.mounts()[i].host_path().empty()) { ++ ERROR("Mount path is empty"); ++ goto error_out; ++ } ++ ++ char path[PATH_MAX] = { 0 }; ++ if (!util_clean_path(containerConfig.mounts()[i].container_path().c_str(), path, sizeof(path))) { ++ ERROR("Failed to get clean path for mount src path: %s", containerConfig.mounts()[i].container_path().c_str()); ++ goto error_out; ++ } ++ ++ tmp->destination = util_strdup_s(path); ++ ++ if (!util_clean_path(containerConfig.mounts()[i].host_path().c_str(), path, sizeof(path))) { ++ ERROR("Failed to get clean path for mount src path: %s", containerConfig.mounts()[i].host_path().c_str()); ++ goto error_out; ++ } ++ tmp->source = util_strdup_s(path); ++ ++ if (util_array_append(&(tmp->options), "rbind") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ ++ if (containerConfig.mounts()[i].propagation() == runtime::v1::PROPAGATION_PRIVATE) { ++ DEBUG("noop, private is default"); ++ if (util_array_append(&(tmp->options), "rprivate") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ } else if (containerConfig.mounts()[i].propagation() == runtime::v1::PROPAGATION_BIDIRECTIONAL) { ++ if (util_array_append(&(tmp->options), "rshared") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ } else if (containerConfig.mounts()[i].propagation() == runtime::v1::PROPAGATION_HOST_TO_CONTAINER) { ++ if (util_array_append(&(tmp->options), "rslave") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ } else { ++ WARN("unknown propagation mode for hostPath %s", containerConfig.mounts()[i].host_path().c_str()); ++ if (util_array_append(&(tmp->options), "rprivate") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ } ++ ++ if (containerConfig.mounts()[i].readonly()) { ++ if (util_array_append(&(tmp->options), "ro") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ } else { ++ if (util_array_append(&(tmp->options), "rw") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ } ++ ++ tmp->type = util_strdup_s("bind"); ++ ++ con.mounts[i] = tmp; ++ tmp = nullptr; ++ con.mounts_len++; ++ } ++ return true; ++ ++error_out: ++ free_nri_mount(tmp); ++ return false; ++} ++ ++static auto MountPointsElementToNRI(container_config_v2_common_config_mount_points *mp, nri_container &con) -> bool ++{ ++ size_t i, len; ++ nri_mount *tmp = nullptr; ++ ++ if (mp == nullptr || mp->len == 0) { ++ return true; ++ } ++ len = mp->len; ++ ++ con.mounts = (nri_mount **)util_smart_calloc_s(sizeof(nri_mount *), len); ++ if (con.mounts == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ for (i = 0; i < len; i++) { ++ tmp = (nri_mount *)util_common_calloc_s(sizeof(nri_mount)); ++ char path[PATH_MAX] = { 0 }; ++ ++ if (!util_clean_path(mp->values[i]->destination, path, sizeof(path))) { ++ ERROR("Failed to get clean path for mount dest path: %s", mp->values[i]->destination); ++ goto error_out; ++ } ++ tmp->destination = util_strdup_s(path); ++ ++ if (!util_clean_path(mp->values[i]->source, path, sizeof(path))) { ++ ERROR("Failed to get clean path for mount src path: %s", mp->values[i]->source); ++ goto error_out; ++ } ++ tmp->source = util_strdup_s(path); ++ ++ if (util_array_append(&(tmp->options), "rbind") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ if (util_array_append(&(tmp->options), mp->values[i]->propagation) != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ ++ if (mp->values[i]->rw) { ++ if (util_array_append(&(tmp->options), "rw") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ } else { ++ if (util_array_append(&(tmp->options), "ro") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ } ++ ++ tmp->type = util_strdup_s("bind"); ++ con.mounts[i] = tmp; ++ con.mounts_len++; ++ tmp = nullptr; ++ } ++ ++ return true; ++ ++error_out: ++ free_nri_mount(tmp); ++ return false; ++} ++ ++// container info is incomplete because container in excution is not created ++auto ContainerToNRIByConConfig(const runtime::v1::ContainerConfig &containerConfig, nri_container &con) -> bool ++{ ++ // todo: can not get container id and state from containerConfig ++ if (containerConfig.has_metadata() && !containerConfig.metadata().name().empty()) { ++ con.name = util_strdup_s(containerConfig.metadata().name().c_str()); ++ } ++ ++ Errors tmpError; ++ ++ con.labels = Transform::ProtobufMapToJsonMapForString(containerConfig.labels(), tmpError); ++ if (con.labels == nullptr) { ++ ERROR("Failed to transform labels to nri for con : %s, : %s", con.name, tmpError.GetMessage().c_str()); ++ return false; ++ } ++ ++ con.annotations = Transform::ProtobufMapToJsonMapForString(containerConfig.annotations(), tmpError); ++ if (con.annotations == nullptr) { ++ ERROR("Failed to transform annotations to nri for con : %s, : %s", con.name, tmpError.GetMessage().c_str()); ++ return false; ++ } ++ ++ con.args = Transform::RepeatedPtrFieldToCharArray(containerConfig.args()); ++ if (con.args == nullptr) { ++ ERROR("Failed to transform args to nri for con : %s, : %s", con.name, tmpError.GetMessage().c_str()); ++ return false; ++ } ++ con.args_len = containerConfig.args_size(); ++ ++ auto envVect = CRIHelpersV1::GenerateEnvList(containerConfig.envs()); ++ con.env = Transform::StringVectorToCharArray(envVect); ++ if (con.env == nullptr) { ++ ERROR("Failed to transform env to nri for con : %s", con.name); ++ return false; ++ } ++ con.env_len = containerConfig.envs_size(); ++ ++ if (!CRIMountArrToNRI(containerConfig, con)) { ++ ERROR("Failed to transform mounts to nri for con : %s", con.name); ++ return false; ++ } ++ return true; ++ ++ // todo: can not get container hooks and pid from containerConfig ++} ++ ++// container info is incomplete because container in excution is not created ++auto ContainerToNRIByID(const std::string &id, nri_container &con) -> bool ++{ ++ container_t *cont = nullptr; ++ bool ret = false; ++ ++ cont = containers_store_get(id.c_str()); ++ if (cont == nullptr || cont->common_config == nullptr) { ++ ERROR("No such container:%s", id.c_str()); ++ goto out; ++ } ++ ++ con.id = util_strdup_s(id.c_str()); ++ ++ con.name = util_strdup_s(cont->common_config->name); ++ ++ con.labels = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); ++ if (con.labels == nullptr) { ++ ERROR("Out of memory"); ++ goto out; ++ } ++ con.annotations = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); ++ if (con.annotations == nullptr) { ++ ERROR("Out of memory"); ++ goto out; ++ } ++ // state ++ if (dup_json_map_string_string(cont->common_config->config->labels, con.labels) != 0) { ++ ERROR("Failed to copy labels for con: %s", cont->common_config->name); ++ goto out; ++ } ++ if (dup_json_map_string_string(cont->common_config->config->annotations, con.annotations) != 0) { ++ ERROR("Failed to copy labels for con: %s", cont->common_config->name); ++ goto out; ++ } ++ ++ con.args = util_copy_array_by_len(cont->common_config->args, cont->common_config->args_len); ++ if (cont->common_config->args_len != 0 && con.args == nullptr) { ++ ERROR("Failed to copy args for con: %s", cont->common_config->name); ++ goto out; ++ } ++ con.args_len = cont->common_config->args_len; ++ ++ con.env = util_copy_array_by_len(cont->common_config->config->env, cont->common_config->config->env_len); ++ if (cont->common_config->config->env_len != 0 && con.env == nullptr) { ++ ERROR("Failed to copy env for con: %s", cont->common_config->name); ++ goto out; ++ } ++ con.env_len = cont->common_config->config->env_len; ++ ++ if (!MountPointsElementToNRI(cont->common_config->mount_points, con)) { ++ ERROR("Failed to transform mounts to nri for con : %s", con.name); ++ goto out; ++ } ++ ++ // todo: can convert hostconfig's hook_spec to nri spec ++ ++ con.pid = container_state_get_pid(cont->state); ++ if (con.pid < 0) { ++ ERROR("Container %s pid %d invalid", cont->common_config->name, con.pid); ++ goto out; ++ } ++ ++ con.pod_sandbox_id = util_strdup_s(cont->common_config->sandbox_info->id); ++ ret = true; ++ ++out: ++ container_unref(cont); ++ return ret; ++} ++ ++auto LinuxResourcesFromNRI(const nri_linux_resources *src, runtime::v1::LinuxContainerResources &resources) -> bool ++{ ++ if (src == nullptr) { ++ return false; ++ } ++ ++ if (src->memory != nullptr) { ++ resources.set_memory_limit_in_bytes(*src->memory->limit); ++ resources.set_oom_score_adj(DefaultOOMScoreAdj); ++ } ++ ++ if (src->cpu != nullptr) { ++ if (src->cpu->shares != NULL) { ++ resources.set_cpu_shares(*src->cpu->shares); ++ } ++ if (src->cpu->quota != NULL) { ++ resources.set_cpu_quota(*src->cpu->quota); ++ } ++ if (src->cpu->period != NULL) { ++ resources.set_cpu_period(*src->cpu->period); ++ } ++ ++ resources.set_cpuset_cpus(src->cpu->cpus); ++ resources.set_cpuset_mems(src->cpu->mems); ++ } ++ ++ if (src->hugepage_limits != nullptr && src->hugepage_limits_len > 0) { ++ for (size_t i = 0; i < src->hugepage_limits_len; i++) { ++ if (src->hugepage_limits[i] != nullptr) { ++ auto limit = resources.add_hugepage_limits(); ++ limit->set_page_size(src->hugepage_limits[i]->page_size); ++ limit->set_limit(src->hugepage_limits[i]->limit); ++ } ++ } ++ } ++ ++ if (src->unified != nullptr) { ++ Transform::JsonMapToProtobufMapForString(src->unified, *resources.mutable_unified()); ++ } ++ ++ return true; ++} +\ No newline at end of file +diff --git a/src/daemon/common/nri/nri_convert.h b/src/daemon/common/nri/nri_convert.h +index 883f7c41..c04b14e4 100644 +--- a/src/daemon/common/nri/nri_convert.h ++++ b/src/daemon/common/nri/nri_convert.h +@@ -27,10 +27,11 @@ + #include "sandbox.h" + #include "api_v1.pb.h" + +-auto PodSandboxToNRI(const std::shared_ptr<const sandbox::Sandbox> &sandbox, nri_pod_sandbox *pod) -> bool; +-auto ContainerToNRIByConConfig(const runtime::v1::ContainerConfig &containerConfig, nri_container *con) -> bool; +-auto ContainerToNRIByID(const std::string &id, nri_container *con) -> bool; +-auto PodSandboxesToNRI(const std::vector<std::unique_ptr<sandbox::Sandbox>> &arrs, nri_pod_sandbox **pod, int pod_len) -> bool; ++auto PodSandboxToNRI(const std::shared_ptr<const sandbox::Sandbox> &sandbox, nri_pod_sandbox &pod) -> bool; ++auto ContainerToNRIByConConfig(const runtime::v1::ContainerConfig &containerConfig, nri_container &con) -> bool; ++auto ContainerToNRIByID(const std::string &id, nri_container &con) -> bool; ++auto PodSandboxesToNRI(const std::vector<std::unique_ptr<sandbox::Sandbox>> &arrs, nri_pod_sandbox **pod, ++ int pod_len) -> bool; + + auto LinuxResourcesFromNRI(const nri_linux_resources *src, runtime::v1::LinuxContainerResources &resources) -> bool; + #endif // DAEMON_COMMON_NRI_NRI_CONVERT_H +diff --git a/src/daemon/common/nri/nri_spec.c b/src/daemon/common/nri/nri_spec.c +new file mode 100644 +index 00000000..855fe3b3 +--- /dev/null ++++ b/src/daemon/common/nri/nri_spec.c +@@ -0,0 +1,602 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-07-17 ++ * Description: provide nri oci functions ++ *********************************************************************************/ ++ ++#include "nri_spec.h" ++ ++#include <isula_libutils/log.h> ++ ++#include "map.h" ++#include "utils.h" ++#include "utils_string.h" ++#include "nri_utils.h" ++#include "specs_api.h" ++#include "sysinfo.h" ++#include "verify.h" ++#include "specs_extend.h" ++ ++static defs_hook *nri_hook_to_oci(const nri_hook *h) ++{ ++ defs_hook *oci_hook = NULL; ++ ++ if (h == NULL) { ++ return NULL; ++ } ++ ++ oci_hook = util_common_calloc_s(sizeof(*oci_hook)); ++ if (oci_hook == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ oci_hook->path = util_strdup_s(h->path); ++ if (h->args_len != 0) { ++ oci_hook->args = util_copy_array_by_len(h->args, h->args_len); ++ if (oci_hook->args == NULL) { ++ ERROR("Failed to copy args"); ++ goto error_out; ++ } ++ oci_hook->args_len = h->args_len; ++ } ++ if (h->env_len != 0) { ++ oci_hook->env = util_copy_array_by_len(h->env, h->env_len); ++ if (oci_hook->env == NULL) { ++ ERROR("Failed to copy env"); ++ goto error_out; ++ } ++ oci_hook->env_len = h->env_len; ++ } ++ if (h->timeout != NULL) { ++ oci_hook->timeout = *(h->timeout); ++ } ++ return oci_hook; ++ ++error_out: ++ free_defs_hook(oci_hook); ++ return NULL; ++} ++ ++static defs_device *nri_device_to_oci(nri_linux_device *dev) ++{ ++ if (dev == NULL) { ++ return NULL; ++ } ++ ++ defs_device *oci_dev = util_common_calloc_s(sizeof(defs_device)); ++ if (oci_dev == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ oci_dev->path = util_strdup_s(dev->path); ++ oci_dev->type = util_strdup_s(dev->type); ++ oci_dev->major = dev->major; ++ oci_dev->minor = dev->minor; ++ if (dev->file_mode != NULL) { ++ oci_dev->file_mode = *dev->file_mode; ++ } ++ if (dev->uid != NULL) { ++ oci_dev->uid = *dev->uid; ++ } ++ if (dev->gid != NULL) { ++ oci_dev->gid = *dev->gid; ++ } ++ ++ return oci_dev; ++} ++ ++static defs_mount *nri_mount_to_oci(nri_mount *mount) ++{ ++ if (mount == NULL) { ++ return NULL; ++ } ++ ++ defs_mount *oci_mount = util_common_calloc_s(sizeof(defs_mount)); ++ if (oci_mount == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ oci_mount->destination = util_strdup_s(mount->destination); ++ oci_mount->type = util_strdup_s(mount->type); ++ oci_mount->source = util_strdup_s(mount->source); ++ if (mount->options_len != 0) { ++ oci_mount->options = util_copy_array_by_len(mount->options, mount->options_len); ++ if (oci_mount->options == NULL) { ++ ERROR("Failed to copy options"); ++ free_defs_mount(oci_mount); ++ return NULL; ++ } ++ oci_mount->options_len = mount->options_len; ++ } ++ ++ return oci_mount; ++} ++ ++static int nri_adjust_annotation(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ int ret = -1; ++ size_t i; ++ ++ if (adjust == NULL || adjust->annotations == NULL || adjust->annotations->len == 0) { ++ return 0; ++ } ++ ++ if (make_sure_oci_spec_annotations(oci_spec) != 0) { ++ ERROR("Failed to make sure oci spec annotations"); ++ return -1; ++ } ++ ++ json_map_string_string *cleard = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); ++ if (cleard == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ ++ map_t *del = map_new(MAP_STR_STR, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ if (del == NULL) { ++ ERROR("Out of memory"); ++ goto free_out; ++ } ++ ++ for (i = 0; i < adjust->annotations->len; i++) { ++ __isula_auto_free char *out = NULL; ++ if (is_marked_for_removal(adjust->annotations->keys[i], &out)) { ++ if (!map_insert(del, out, "")) { ++ ERROR("Failed to insert del map"); ++ goto free_out; ++ } ++ continue; ++ } ++ if (append_json_map_string_string(cleard, adjust->annotations->keys[i], ++ adjust->annotations->values[i]) != 0) { ++ ERROR("Failed to append annotation"); ++ goto free_out; ++ } ++ } ++ ++ for (i = 0; i < oci_spec->annotations->len; i++) { ++ if (map_search(del, oci_spec->annotations->keys[i]) != NULL) { ++ continue; ++ } ++ append_json_map_string_string(cleard, oci_spec->annotations->keys[i], ++ oci_spec->annotations->values[i]); ++ } ++ ++ free_json_map_string_string(oci_spec->annotations); ++ oci_spec->annotations = cleard; ++ ret = 0; ++ ++free_out: ++ free_json_map_string_string(cleard); ++ map_free(del); ++ return ret; ++} ++ ++static void nri_key_value_map_kvfree(void *key, void *value) ++{ ++ free(key); ++ ++ // no need to free nri_key_value ++ // nri_key_value *value will be free in nri_container_adjustment *adjust ++} ++ ++ ++static int nri_adjust_env(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ int ret = -1; ++ size_t i; ++ char **old_env = NULL; ++ size_t old_env_len = 0; ++ __isula_auto_array_t char **adjust_env = NULL; ++ size_t adjust_env_len = 0; ++ ++ if (adjust->env == NULL || adjust->env_len == 0) { ++ return 0; ++ } ++ ++ map_t *mod = map_new(MAP_STR_PTR, MAP_DEFAULT_CMP_FUNC, nri_key_value_map_kvfree); ++ if (mod == NULL) { ++ ERROR("Out of memory"); ++ goto free_out; ++ } ++ ++ for (i = 0; i < adjust->env_len; i++) { ++ nri_key_value *e = adjust->env[i]; ++ char *out = NULL; ++ (void)is_marked_for_removal(e->key, &out); ++ ++ if (!map_insert(mod, out, e) == false) { ++ ERROR("Failed to insert mod map"); ++ goto free_out; ++ } ++ } ++ ++ if (map_size(mod) <= 0 || oci_spec == NULL || oci_spec->process == NULL) { ++ ret = 0; ++ goto free_out; ++ } ++ ++ // modify existing environment ++ old_env = oci_spec->process->env; ++ old_env_len = oci_spec->process->env_len; ++ oci_spec->process->env = NULL; ++ oci_spec->process->env_len = 0; ++ ++ for (i = 0; i < old_env_len; i++) { ++ __isula_auto_array_t char **envArr = util_string_split_n(old_env[i], '=', 2); ++ if (envArr == NULL) { ++ continue; ++ } ++ ++ nri_key_value *target = map_search(mod, envArr[0]); ++ if (target != NULL) { ++ __isula_auto_free char *out = NULL; ++ if (!is_marked_for_removal(envArr[0], &out)) { ++ // If not marked for removal, append modified value ++ __isula_auto_free char *tmp_str = util_string_append(target->key, "="); ++ __isula_auto_free char *final_str = util_string_append(tmp_str, target->value); ++ ++ if (util_array_append(&adjust_env, final_str) != 0) { ++ ERROR("Failed to append env"); ++ goto free_out; ++ } ++ adjust_env_len++; ++ continue; ++ } ++ } ++ // If not found in mod map, append original value ++ if (util_array_append(&adjust_env, old_env[i]) != 0) { ++ ERROR("Failed to append env"); ++ goto free_out; ++ } ++ adjust_env_len++; ++ } ++ ++ ret = 0; ++free_out: ++ if (merge_env(oci_spec, (const char **)adjust_env, adjust_env_len) != 0) { ++ ERROR("Failed to merge env"); ++ goto free_out; ++ } ++ for (i = 0; i < old_env_len; i++) { ++ free(old_env[i]); ++ } ++ free(old_env); ++ map_free(mod); ++ return ret; ++} ++ ++static int nri_adjust_hooks(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ if (adjust->hooks == NULL) { ++ return 0; ++ } ++ ++ size_t i; ++ int ret = 0; ++ ++ if (make_sure_oci_spec_hooks(oci_spec) != 0) { ++ ERROR("Failed to make sure oci spec hooks"); ++ return -1; ++ } ++ ++ // todo: change to macro definition function call ++ for (i = 0; i < adjust->hooks->prestart_len; i++) { ++ defs_hook *oci_hook = nri_hook_to_oci(adjust->hooks->prestart[i]); ++ ret = spec_add_prestart_hook(oci_spec, oci_hook); ++ if (ret != 0) { ++ ERROR("Failed add hook %s", adjust->hooks->prestart[i]->path); ++ free_defs_hook(oci_hook); ++ return -1; ++ } ++ } ++ ++ for (i = 0; i < adjust->hooks->poststart_len; i++) { ++ defs_hook *oci_hook = nri_hook_to_oci(adjust->hooks->poststart[i]); ++ ret = spec_add_poststart_hook(oci_spec, oci_hook); ++ if (ret != 0) { ++ ERROR("Failed add hook %s", adjust->hooks->poststart[i]->path); ++ free_defs_hook(oci_hook); ++ return -1; ++ } ++ } ++ ++ for (i = 0; i < adjust->hooks->poststop_len; i++) { ++ defs_hook *oci_hook = nri_hook_to_oci(adjust->hooks->poststop[i]); ++ ret = spec_add_poststop_hook(oci_spec, oci_hook); ++ if (ret != 0) { ++ ERROR("Failed add hook %s", adjust->hooks->poststop[i]->path); ++ free_defs_hook(oci_hook); ++ return -1; ++ } ++ } ++ /* ++ * The OCI being used by the iSulad not supportes ++ * createRuntime/createContainer/startContainer currently. ++ */ ++ ++ return ret; ++} ++ ++static int nri_adjust_devices(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ if (adjust->linux == NULL || adjust->linux->devices == NULL || adjust->linux->devices_len == 0) { ++ return 0; ++ } ++ ++ size_t i; ++ ++ for (i = 0; i < adjust->linux->devices_len; i++) { ++ nri_linux_device *dev = adjust->linux->devices[i]; ++ if (spec_add_device(oci_spec, nri_device_to_oci(dev)) != 0) { ++ ERROR("Failed to add device %s", dev->path); ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ ++static int nri_adjust_cgroup_path(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ if (adjust->linux == NULL || adjust->linux->cgroups_path == NULL) { ++ return 0; ++ } ++ ++ free(oci_spec->linux->cgroups_path); ++ oci_spec->linux->cgroups_path = util_strdup_s(adjust->linux->cgroups_path); ++ ++ return 0; ++} ++ ++static void nri_adjust_cpu_memory(nri_linux_resources *resource, oci_runtime_spec *oci_spec) ++{ ++ if (resource->cpu == NULL) { ++ return; ++ } ++ if (make_sure_oci_spec_linux_resources_cpu(oci_spec) != 0) { ++ ERROR("Failed to make sure oci spec linux resources cpu"); ++ return; ++ } ++ if (resource->cpu->shares != NULL) { ++ oci_spec->linux->resources->cpu->shares = *resource->cpu->shares; ++ } ++ if (resource->cpu->quota != NULL) { ++ oci_spec->linux->resources->cpu->quota = *resource->cpu->quota; ++ } ++ if (resource->cpu->period != NULL) { ++ oci_spec->linux->resources->cpu->period = *resource->cpu->period; ++ } ++ if (resource->cpu->realtime_runtime != NULL) { ++ oci_spec->linux->resources->cpu->realtime_runtime = *resource->cpu->realtime_runtime; ++ } ++ if (resource->cpu->realtime_period != NULL) { ++ oci_spec->linux->resources->cpu->realtime_period = *resource->cpu->realtime_period; ++ } ++} ++ ++static void nri_adjust_memory_resource(nri_linux_resources *resource, oci_runtime_spec *oci_spec) ++{ ++ if (resource->memory == NULL) { ++ return; ++ } ++ ++ if (make_sure_oci_spec_linux_resources_mem(oci_spec) != 0) { ++ ERROR("Failed to make sure oci spec linux resources memory"); ++ return; ++ } ++ if (resource->memory->limit != NULL) { ++ oci_spec->linux->resources->memory->limit = *resource->memory->limit; ++ } ++ if (resource->memory->reservation != NULL) { ++ oci_spec->linux->resources->memory->reservation = *resource->memory->reservation; ++ } ++ if (resource->memory->swap != NULL) { ++ oci_spec->linux->resources->memory->swap = *resource->memory->swap; ++ } ++ if (resource->memory->kernel != NULL) { ++ oci_spec->linux->resources->memory->kernel = *resource->memory->kernel; ++ } ++ if (resource->memory->kernel_tcp != NULL) { ++ oci_spec->linux->resources->memory->kernel_tcp = *resource->memory->kernel_tcp; ++ } ++ if (resource->memory->swappiness != NULL) { ++ oci_spec->linux->resources->memory->swappiness = *resource->memory->swappiness; ++ } ++ if (resource->memory->disable_oom_killer != NULL) { ++ oci_spec->linux->resources->memory->disable_oom_killer = *resource->memory->disable_oom_killer; ++ } ++} ++ ++static int nri_adjust_hugepage_resource(nri_linux_resources *resource, oci_runtime_spec *oci_spec) ++{ ++ size_t i; ++ if (resource->hugepage_limits != NULL) { ++ for (i = 0; i < resource->hugepage_limits_len; i++) { ++ nri_hugepage_limit *limit = resource->hugepage_limits[i]; ++ if (limit->page_size != NULL) { ++ if (spec_add_linux_resources_hugepage_limit(oci_spec, limit->page_size, limit->limit) != 0) { ++ ERROR("Failed to add hugepage limit"); ++ return -1; ++ } ++ } ++ } ++ } ++ return 0; ++} ++ ++static int nri_adjust_unified_resource(nri_linux_resources *resource, oci_runtime_spec *oci_spec) ++{ ++ size_t i; ++ if (resource->unified != NULL) { ++ for (i = 0; i < resource->unified->len; i++) { ++ if (append_json_map_string_string(oci_spec->linux->resources->unified, resource->unified->keys[i], ++ resource->unified->values[i]) != 0) { ++ ERROR("Failed to append unified resource"); ++ return -1; ++ } ++ } ++ } ++ return 0; ++} ++ ++static int nri_adjust_resources(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ if (adjust->linux == NULL || adjust->linux->resources == NULL) { ++ return 0; ++ } ++ ++ nri_linux_resources *resource = adjust->linux->resources; ++ ++ nri_adjust_memory_resource(resource, oci_spec); ++ nri_adjust_cpu_memory(resource, oci_spec); ++ ++ if (nri_adjust_hugepage_resource(resource, oci_spec) != 0) { ++ ERROR("Failed to adjust hugepage resource"); ++ return -1; ++ } ++ ++ if (nri_adjust_unified_resource(resource, oci_spec) != 0) { ++ ERROR("Failed to adjust unified resource"); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++static int nri_adjust_mounts(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ if (adjust->mounts == NULL || adjust->mounts_len == 0) { ++ return 0; ++ } ++ ++ size_t i; ++ for (i = 0; i < adjust->mounts_len; i++) { ++ nri_mount *mount = adjust->mounts[i]; ++ defs_mount *oci_mount = nri_mount_to_oci(mount); ++ if (oci_mount == NULL) { ++ ERROR("Failed to convert nri mount to oci mount"); ++ return -1; ++ } ++ if (spec_add_mount(oci_spec, oci_mount) != 0) { ++ ERROR("Failed to add mount"); ++ free_defs_mount(oci_mount); ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ ++static int nri_adjust_rlimit(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ if (adjust->rlimits == NULL || adjust->rlimits_len == 0) { ++ return 0; ++ } ++ ++ size_t i; ++ for (i = 0; i < adjust->rlimits_len; i++) { ++ nri_posix_rlimit *rlimit = adjust->rlimits[i]; ++ if (rlimit->type == NULL) { ++ ERROR("Invalid rlimit type"); ++ return -1; ++ } ++ if (spec_add_linux_resources_rlimit(oci_spec, rlimit->type, rlimit->soft, rlimit->hard) != 0) { ++ ERROR("Failed to add rlimit"); ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ ++// todo: we do not support it blockio_class ++static int nri_adjust_blockio_class(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ if (adjust->linux == NULL || adjust->linux->resources->blockio_class == NULL) { ++ return 0; ++ } ++ ++ return 0; ++} ++ ++int nri_adjust_oci_spec(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ if (oci_spec == NULL || adjust == NULL) { ++ ERROR("Invalid input arguments"); ++ return -1; ++ } ++ ++ if (nri_adjust_annotation(adjust, oci_spec) != 0) { ++ ERROR("Failed to do nri adjust annotation in oci spec"); ++ return -1; ++ } ++ ++ if (nri_adjust_env(adjust, oci_spec) != 0) { ++ ERROR("Failed to do nri adjust env in oci spec"); ++ return -1; ++ } ++ ++ if (nri_adjust_hooks(adjust, oci_spec) != 0) { ++ ERROR("Failed to do nri adjust hooks in oci spec"); ++ return -1; ++ } ++ ++ if (nri_adjust_devices(adjust, oci_spec) != 0) { ++ ERROR("Failed to do nri adjust devices in oci spec"); ++ return -1; ++ } ++ ++ if (nri_adjust_cgroup_path(adjust, oci_spec) != 0) { ++ ERROR("Failed to do nri adjust cgroup path in oci spec"); ++ return -1; ++ } ++ ++ if (nri_adjust_resources(adjust, oci_spec) != 0) { ++ ERROR("Failed to do nri adjust resources in oci spec"); ++ return -1; ++ } ++ ++ if (nri_adjust_blockio_class(adjust, oci_spec) != 0) { ++ ERROR("Failed to do nri adjust blockio class in oci spec"); ++ return -1; ++ } ++ ++ // iSuald is not support IntelRdt ++ if (nri_adjust_mounts(adjust, oci_spec) != 0) { ++ ERROR("Failed to do nri adjust mount in oci spec"); ++ return -1; ++ } ++ ++ if (nri_adjust_rlimit(adjust, oci_spec) != 0) { ++ ERROR("Failed to do nri adjust rlimit in oci spec"); ++ return -1; ++ } ++ ++ __isula_auto_sysinfo_t sysinfo_t *sysinfo = NULL; ++ ++ sysinfo = get_sys_info(true); ++ if (sysinfo == NULL) { ++ ERROR("Failed to get system info"); ++ return -1; ++ } ++ ++ if (verify_container_settings(oci_spec, sysinfo) != 0) { ++ ERROR("Failed to verify oci runtime spec settings after adjust by nri"); ++ return -1; ++ } ++ ++ return 0; ++} +\ No newline at end of file +diff --git a/src/daemon/common/nri/nri_utils.c b/src/daemon/common/nri/nri_utils.c +new file mode 100644 +index 00000000..51054e32 +--- /dev/null ++++ b/src/daemon/common/nri/nri_utils.c +@@ -0,0 +1,520 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-07-17 ++ * Description: provide nri utils functions ++ *********************************************************************************/ ++ ++#include "nri_utils.h" ++ ++#include <isula_libutils/log.h> ++ ++#include "utils.h" ++ ++static bool copy_nri_hugepage_limit(const nri_hugepage_limit* src, nri_hugepage_limit** dest) ++{ ++ if (src == NULL || dest == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ *dest = (nri_hugepage_limit *)util_common_calloc_s(sizeof(nri_hugepage_limit)); ++ if (*dest == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ (*dest)->limit = src->limit; ++ (*dest)->page_size = util_strdup_s(src->page_size); ++ return true; ++} ++ ++static bool copy_nri_hook(const nri_hook *src, nri_hook **dest) ++{ ++ if (src == NULL || dest == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ *dest = (nri_hook *)util_common_calloc_s(sizeof(nri_hook)); ++ if (dest == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ (*dest)->args = util_copy_array_by_len(src->args, src->args_len); ++ (*dest)->args_len = src->args_len; ++ (*dest)->env = util_copy_array_by_len(src->env, src->env_len); ++ (*dest)->env_len = src->env_len; ++ (*dest)->path = util_strdup_s(src->path); ++ return true; ++} ++ ++static bool copy_nri_linux_device_cgroup(const nri_linux_device_cgroup *src, nri_linux_device_cgroup **dest) ++{ ++ if (src == NULL || dest == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ *dest = (nri_linux_device_cgroup *)util_common_calloc_s(sizeof(nri_linux_device_cgroup)); ++ if (dest == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ (*dest)->allow = src->allow; ++ (*dest)->type = util_strdup_s(src->type); ++ (*dest)->major = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if ((*dest)->major == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ (*dest)->minor = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if ((*dest)->minor == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ (*dest)->access = util_strdup_s(src->access); ++ return true; ++} ++ ++static bool copy_nri_linux_cpu(const nri_linux_cpu *src, nri_linux_cpu **dest) ++{ ++ if (src == NULL || dest == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ (*dest) = (nri_linux_cpu *)util_common_calloc_s(sizeof(nri_linux_cpu)); ++ if ((*dest) == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ (*dest)->cpus = util_strdup_s(src->cpus); ++ (*dest)->mems = util_strdup_s(src->mems); ++ if (src->period != NULL) { ++ (*dest)->period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if ((*dest)->period == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->period = *src->period; ++ } ++ ++ if (src->quota != NULL) { ++ (*dest)->quota = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if ((*dest)->quota == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->quota = *src->quota; ++ } ++ ++ if (src->realtime_period != NULL) { ++ (*dest)->realtime_period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if ((*dest)->realtime_period == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->realtime_period = *src->realtime_period; ++ } ++ ++ if (src->realtime_runtime != NULL) { ++ (*dest)->realtime_runtime = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if ((*dest)->realtime_runtime == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->realtime_runtime = *src->realtime_runtime; ++ } ++ ++ if (src->shares != NULL) { ++ (*dest)->shares = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if ((*dest)->shares == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->shares = *src->shares; ++ } ++ ++ return true; ++} ++ ++static bool copy_nri_linux_memory(const nri_linux_memory *src, nri_linux_memory **dest) ++{ ++ if (src == NULL || dest == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ *dest = (nri_linux_memory *)util_common_calloc_s(sizeof(nri_linux_memory)); ++ if (dest == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ if (src->limit != NULL) { ++ (*dest)->limit = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if ((*dest)->limit == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->limit = *src->limit; ++ } ++ ++ if (src->reservation != NULL) { ++ (*dest)->reservation = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if ((*dest)->reservation == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->reservation = *src->reservation; ++ } ++ ++ if (src->swap != NULL) { ++ (*dest)->swap = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if ((*dest)->swap == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->swap = *src->swap; ++ } ++ ++ if (src->kernel != NULL) { ++ (*dest)->kernel = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if ((*dest)->kernel == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->kernel = *src->kernel; ++ } ++ ++ ++ if (src->kernel_tcp != NULL) { ++ (*dest)->kernel_tcp = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if ((*dest)->kernel_tcp == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->kernel_tcp = *src->kernel_tcp; ++ } ++ ++ if (src->swappiness != NULL) { ++ (*dest)->swappiness = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if ((*dest)->swappiness == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->swappiness = *src->swappiness; ++ } ++ ++ if (src->disable_oom_killer != NULL) { ++ (*dest)->disable_oom_killer = (uint8_t *)util_common_calloc_s(sizeof(uint8_t)); ++ if ((*dest)->disable_oom_killer == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->disable_oom_killer = *src->disable_oom_killer; ++ } ++ ++ if (src->use_hierarchy != NULL) { ++ (*dest)->use_hierarchy = (uint8_t *)util_common_calloc_s(sizeof(uint8_t)); ++ if ((*dest)->use_hierarchy == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->use_hierarchy = *src->use_hierarchy; ++ } ++ return true; ++} ++ ++bool is_marked_for_removal(const char* key, char **out) ++{ ++ if (key == NULL || out == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ if (!util_has_prefix(key, "-")) { ++ *out = (char*)key; ++ return false; ++ } ++ ++ *out = util_sub_string(key, 1, strlen(key) - 1); ++ if (*out == NULL) { ++ ERROR("Failed to sub string"); ++ return false; ++ } ++ ++ return true; ++} ++ ++bool copy_nri_mount(const nri_mount *src, nri_mount **dest) ++{ ++ if (src == NULL || dest == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ *dest = (nri_mount *)util_common_calloc_s(sizeof(nri_mount)); ++ if (dest == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ (*dest)->destination = util_strdup_s(src->destination); ++ (*dest)->options = util_copy_array_by_len(src->options, src->options_len); ++ (*dest)->options_len = src->options_len; ++ (*dest)->source = util_strdup_s(src->source); ++ (*dest)->type = util_strdup_s(src->type); ++ return true; ++} ++ ++bool copy_nri_key_value(const nri_key_value *src, nri_key_value **dest) ++{ ++ if (src == NULL || dest == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ *dest = (nri_key_value *)util_common_calloc_s(sizeof(nri_key_value)); ++ if (dest == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ (*dest)->key = util_strdup_s(src->key); ++ (*dest)->value = util_strdup_s(src->value); ++ return true; ++} ++ ++bool copy_nri_posix_rlimit(const nri_posix_rlimit *src, nri_posix_rlimit **dest) ++{ ++ if (src == NULL || dest == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ *dest = (nri_posix_rlimit *)util_common_calloc_s(sizeof(nri_posix_rlimit)); ++ if (dest == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ (*dest)->hard = src->hard; ++ (*dest)->soft = src->soft; ++ (*dest)->type = util_strdup_s(src->type); ++ return true; ++} ++ ++bool copy_nri_linux_resources(const nri_linux_resources *src, nri_linux_resources **dest) ++{ ++ if (src == NULL || dest == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ *dest = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); ++ if (*dest == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ if (!init_nri_linux_resources(dest)) { ++ ERROR("Failed to init dest nri linux resources"); ++ goto free_out; ++ } ++ ++ if (!copy_nri_linux_cpu(src->cpu, &(*dest)->cpu)) { ++ ERROR("Failed to copy nri_linux_cpu"); ++ goto free_out; ++ } ++ ++ if (!copy_nri_linux_memory(src->memory, &(*dest)->memory)) { ++ ERROR("Failed to copy nri_linux_memory"); ++ goto free_out; ++ } ++ ++ (*dest)->blockio_class = util_strdup_s(src->blockio_class); ++ (*dest)->rdt_class = util_strdup_s(src->rdt_class); ++ ++ if (src->hugepage_limits_len > 0) { ++ (*dest)->hugepage_limits = (nri_hugepage_limit**)util_smart_calloc_s(sizeof(nri_hugepage_limit*), ++ src->hugepage_limits_len); ++ for (size_t i = 0; i < src->hugepage_limits_len; ++i) { ++ if (!copy_nri_hugepage_limit(src->hugepage_limits[i], &((*dest)->hugepage_limits[i]))) { ++ ERROR("Failed to copy nri_hugepage_limit"); ++ goto free_out; ++ } ++ } ++ } ++ ++ if (src->devices_len > 0) { ++ (*dest)->devices = (nri_linux_device_cgroup**)util_smart_calloc_s(sizeof(nri_linux_device_cgroup*), src->devices_len); ++ for (size_t i = 0; i < src->devices_len; ++i) { ++ if (!copy_nri_linux_device_cgroup(src->devices[i], &((*dest)->devices[i]))) { ++ ERROR("Failed to copy nri_linux_device_cgroup"); ++ goto free_out; ++ } ++ } ++ } ++ ++ if (dup_json_map_string_string(src->unified, (*dest)->unified)) { ++ ERROR("Failed to copy json_map_string_string"); ++ goto free_out; ++ } ++ ++ return true; ++ ++free_out: ++ free_nri_linux_resources(*dest); ++ return false; ++} ++ ++bool merge_nri_hooks(nri_hook **targetHooks, size_t targetSize, const nri_hook **sourceHooks, ++ size_t sourceLen) ++{ ++ size_t oldSize = targetSize * sizeof(nri_hook *); ++ size_t newSize = oldSize + sourceLen * sizeof(nri_hook *); ++ ++ if (sourceHooks == NULL || targetHooks == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ if (util_mem_realloc((void**)&targetHooks, newSize, (void**)&targetHooks, oldSize) != 0) { ++ ERROR("Failed to realloc and assign hook array"); ++ return false; ++ } ++ ++ for (size_t i = 0; i < sourceLen; i++) { ++ if (!copy_nri_hook(sourceHooks[i], &targetHooks[targetSize++])) { ++ ERROR("Failed to copy hook"); ++ return false; ++ } ++ } ++ ++ return true; ++} ++ ++bool init_nri_container_adjust(nri_container_adjustment **adjust) ++{ ++ if (adjust == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ *adjust = (nri_container_adjustment *)util_common_calloc_s(sizeof(nri_container_adjustment)); ++ if (*adjust == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ (*adjust)->annotations = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); ++ if ((*adjust)->annotations == NULL) { ++ goto free_out; ++ } ++ ++ (*adjust)->env = (nri_key_value **)util_common_calloc_s(sizeof(nri_key_value *)); ++ if ((*adjust)->env == NULL) { ++ goto free_out; ++ } ++ (*adjust)->env_len = 0; ++ ++ (*adjust)->hooks = (nri_hooks *)util_common_calloc_s(sizeof(nri_hooks)); ++ if ((*adjust)->hooks == NULL) { ++ goto free_out; ++ } ++ ++ (*adjust)->linux = (nri_linux_container_adjustment *)util_common_calloc_s(sizeof(nri_linux_container_adjustment)); ++ if ((*adjust)->linux == NULL) { ++ goto free_out; ++ } ++ ++ (*adjust)->linux->resources = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); ++ if ((*adjust)->linux->resources == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ (*adjust)->mounts = (nri_mount **)util_common_calloc_s(sizeof(nri_mount *)); ++ if ((*adjust)->mounts == NULL) { ++ goto free_out; ++ } ++ (*adjust)->mounts_len = 0; ++ ++ (*adjust)->rlimits = (nri_posix_rlimit **)util_common_calloc_s(sizeof(nri_posix_rlimit *)); ++ if ((*adjust)->rlimits == NULL) { ++ goto free_out; ++ } ++ (*adjust)->rlimits_len = 0; ++ ++ return true; ++ ++free_out: ++ ERROR("Out of memory"); ++ free_nri_container_adjustment(*adjust); ++ return false; ++} ++ ++bool init_nri_container_update(nri_container_update **update, const char *id, uint8_t ignore_failure) ++{ ++ if (update == NULL || id == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ *update = (nri_container_update *)util_common_calloc_s(sizeof(nri_container_update)); ++ if (*update == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ (*update)->container_id = util_strdup_s(id); ++ (*update)->linux = (nri_linux_container_update *)util_common_calloc_s(sizeof(nri_linux_container_update)); ++ if ((*update)->linux == NULL) { ++ goto free_out; ++ } ++ ++ (*update)->ignore_failure = ignore_failure; ++ return true; ++ ++free_out: ++ ERROR("Out of memory"); ++ free_nri_container_update(*update); ++ return false; ++} ++ ++bool init_nri_linux_resources(nri_linux_resources **resources) ++{ ++ if (resources == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ *resources = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); ++ if (*resources == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ (*resources)->cpu = (nri_linux_cpu *)util_common_calloc_s(sizeof(nri_linux_cpu)); ++ if ((*resources)->cpu == NULL) { ++ goto free_out; ++ } ++ ++ (*resources)->memory = (nri_linux_memory *)util_common_calloc_s(sizeof(nri_linux_memory)); ++ if ((*resources)->memory == NULL) { ++ goto free_out; ++ } ++ ++ (*resources)->unified = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); ++ if ((*resources)->unified == NULL) { ++ goto free_out; ++ } ++ return true; ++ ++free_out: ++ ERROR("Out of memory"); ++ free_nri_linux_resources(*resources); ++ return false; ++} +\ No newline at end of file +diff --git a/src/daemon/common/nri/nri_utils.h b/src/daemon/common/nri/nri_utils.h +index 3aa50ae4..7bf54a71 100644 +--- a/src/daemon/common/nri/nri_utils.h ++++ b/src/daemon/common/nri/nri_utils.h +@@ -51,14 +51,13 @@ typedef enum { + + bool copy_nri_mount(const nri_mount *src, nri_mount **dest); + bool copy_nri_key_value(const nri_key_value *src, nri_key_value **dest); +-bool copy_nri_hook(const nri_hook *src, nri_hook **dest); + bool copy_nri_posix_rlimit(const nri_posix_rlimit *src, nri_posix_rlimit **dest); + bool copy_nri_linux_resources(const nri_linux_resources *src, nri_linux_resources **dest); +-bool copy_nri_linux_cpu(const nri_linux_cpu *src, nri_linux_cpu **dest); + + bool is_marked_for_removal(const char* key, char **out); + +-bool realloc_and_copy_nri_hooks(nri_hook **targetHooks, size_t targetSize, const nri_hook **sourceHooks, size_t sourceLen); ++bool merge_nri_hooks(nri_hook **targetHooks, size_t targetSize, const nri_hook **sourceHooks, ++ size_t sourceLen); + + bool init_nri_container_adjust(nri_container_adjustment **adjust); + bool init_nri_container_update(nri_container_update **update, const char *id, uint8_t ignore_failure); +diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c +index d7b54498..9ba1c8a0 100644 +--- a/src/daemon/config/isulad_config.c ++++ b/src/daemon/config/isulad_config.c +@@ -456,6 +456,175 @@ out: + (void)isulad_server_conf_unlock(); + return path; + } ++ ++#ifdef ENABLE_NRI ++bool conf_get_nri_support(void) ++{ ++ bool nri_support = false; ++ struct service_arguments *conf = NULL; ++ ++ if (isulad_server_conf_rdlock() != 0) { ++ return false; ++ } ++ ++ conf = conf_get_server_conf(); ++ if (conf == NULL || conf->json_confs == NULL) { ++ goto out; ++ } ++ ++ nri_support = conf->json_confs->nri_support; ++ ++out: ++ (void)isulad_server_conf_unlock(); ++ return nri_support; ++} ++ ++bool conf_get_nri_external_support(void) ++{ ++ bool nri_external_support = false; ++ struct service_arguments *conf = NULL; ++ ++ if (isulad_server_conf_rdlock() != 0) { ++ return false; ++ } ++ ++ conf = conf_get_server_conf(); ++ if (conf == NULL || conf->json_confs == NULL) { ++ goto out; ++ } ++ ++ nri_external_support = conf->json_confs->disable_connections; ++ ++out: ++ (void)isulad_server_conf_unlock(); ++ return !nri_external_support; ++} ++ ++char *conf_get_nri_plugin_config_path(void) ++{ ++ char *path = NULL; ++ struct service_arguments *conf = NULL; ++ ++ if (isulad_server_conf_rdlock() != 0) { ++ return NULL; ++ } ++ ++ conf = conf_get_server_conf(); ++ if (conf == NULL || conf->json_confs == NULL || conf->json_confs->plugin_config_path == NULL) { ++ path = util_strdup_s(DEFAULT_PLUGIN_CONFIG_PATH); ++ goto out; ++ } ++ ++ path = util_strdup_s(conf->json_confs->plugin_config_path); ++ ++out: ++ (void)isulad_server_conf_unlock(); ++ return path; ++} ++ ++char *conf_get_nri_plugin_path(void) ++{ ++ char *path = NULL; ++ struct service_arguments *conf = NULL; ++ ++ if (isulad_server_conf_rdlock() != 0) { ++ return NULL; ++ } ++ ++ conf = conf_get_server_conf(); ++ if (conf == NULL || conf->json_confs == NULL) { ++ goto out; ++ } ++ ++ if (conf->json_confs->plugin_path == NULL) { ++ path = util_strdup_s(DEFAULT_PLUGIN_PATH); ++ goto out; ++ } ++ ++ path = util_strdup_s(conf->json_confs->plugin_path); ++ ++out: ++ (void)isulad_server_conf_unlock(); ++ return path; ++} ++ ++char *conf_get_socket_path(void) ++{ ++ char *path = NULL; ++ struct service_arguments *conf = NULL; ++ ++ if (isulad_server_conf_rdlock() != 0) { ++ return NULL; ++ } ++ ++ conf = conf_get_server_conf(); ++ if (conf == NULL || conf->json_confs == NULL) { ++ goto out; ++ } ++ ++ if (conf->json_confs->nri_socket_path == NULL) { ++ path = util_strdup_s(DEFAULT_SOCKET_PATH); ++ goto out; ++ } ++ ++ path = util_strdup_s(conf->json_confs->nri_socket_path); ++ ++out: ++ (void)isulad_server_conf_unlock(); ++ return path; ++} ++ ++uint64_t conf_get_nri_plugin_registration_timeout(void) ++{ ++ uint64_t timeout = false; ++ struct service_arguments *conf = NULL; ++ ++ if (isulad_server_conf_rdlock() != 0) { ++ return false; ++ } ++ ++ conf = conf_get_server_conf(); ++ if (conf == NULL || conf->json_confs == NULL) { ++ goto out; ++ } ++ ++ if (conf->json_confs->plugin_registration_timeout == 0) { ++ timeout = DEFAULT_PLUGIN_REGISTRY_TIMEOUT; ++ goto out; ++ } ++ ++ timeout = conf->json_confs->plugin_registration_timeout; ++ ++out: ++ (void)isulad_server_conf_unlock(); ++ return timeout; ++} ++uint64_t conf_get_nri_plugin_requst_timeout(void) ++{ ++ uint64_t timeout = false; ++ struct service_arguments *conf = NULL; ++ ++ if (isulad_server_conf_rdlock() != 0) { ++ return false; ++ } ++ ++ conf = conf_get_server_conf(); ++ if (conf == NULL || conf->json_confs == NULL) { ++ goto out; ++ } ++ ++ if (conf->json_confs->plugin_requst_timeout == 0) { ++ timeout = DEFAULT_PLUGIN_REQUST_TIMEOUT; ++ goto out; ++ } ++ ++ timeout = conf->json_confs->plugin_requst_timeout; ++ ++out: ++ (void)isulad_server_conf_unlock(); ++ return timeout; ++} ++#endif + #endif + + /* conf get isulad rootdir */ +@@ -1762,6 +1931,15 @@ int merge_json_confs_into_global(struct service_arguments *args) + tmp_json_confs->cri_sandboxers = NULL; + #endif + args->json_confs->enable_cri_v1 = tmp_json_confs->enable_cri_v1; ++#ifdef ENABLE_NRI ++ args->json_confs->nri_support = tmp_json_confs->nri_support; ++ args->json_confs->disable_connections = tmp_json_confs->disable_connections; ++ override_string_value(&args->json_confs->plugin_config_path, &tmp_json_confs->plugin_config_path); ++ override_string_value(&args->json_confs->plugin_path, &tmp_json_confs->plugin_path); ++ args->json_confs->plugin_registration_timeout = tmp_json_confs->plugin_registration_timeout; ++ args->json_confs->plugin_requst_timeout = tmp_json_confs->plugin_requst_timeout; ++ override_string_value(&args->json_confs->nri_socket_path, &tmp_json_confs->nri_socket_path); ++#endif + args->json_confs->enable_pod_events = tmp_json_confs->enable_pod_events; + #endif + +diff --git a/src/daemon/modules/api/specs_api.h b/src/daemon/modules/api/specs_api.h +index 6a1cd776..d5ea0c7c 100644 +--- a/src/daemon/modules/api/specs_api.h ++++ b/src/daemon/modules/api/specs_api.h +@@ -76,6 +76,11 @@ int spec_add_linux_resources_hugepage_limit(oci_runtime_spec *oci_spec, const ch + int spec_add_linux_resources_rlimit(oci_runtime_spec *oci_spec, const char *type, uint64_t hard, uint64_t soft); + #endif /* ENABLE_NRI */ + ++int make_sure_oci_spec_annotations(oci_runtime_spec *oci_spec); ++int make_sure_oci_spec_linux_resources_cpu(oci_runtime_spec *oci_spec); ++int make_sure_oci_spec_linux_resources_mem(oci_runtime_spec *oci_spec); ++int make_sure_oci_spec_hooks(oci_runtime_spec *oci_spec); ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index 1fd9e5a8..002431d8 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -87,8 +87,11 @@ struct readonly_default_oci_spec { + + static struct readonly_default_oci_spec g_rdspec; + +-static int make_sure_oci_spec_annotations(oci_runtime_spec *oci_spec) ++int make_sure_oci_spec_annotations(oci_runtime_spec *oci_spec) + { ++ if (oci_spec == NULL) { ++ return -1; ++ } + if (oci_spec->annotations == NULL) { + oci_spec->annotations = util_common_calloc_s(sizeof(json_map_string_string)); + if (oci_spec->annotations == NULL) { +@@ -464,10 +467,14 @@ out: + return ret; + } + +-static int make_sure_oci_spec_linux_resources_cpu(oci_runtime_spec *oci_spec) ++int make_sure_oci_spec_linux_resources_cpu(oci_runtime_spec *oci_spec) + { + int ret = 0; + ++ if (oci_spec == NULL) { ++ return -1; ++ } ++ + ret = make_sure_oci_spec_linux_resources(oci_spec); + if (ret < 0) { + return -1; +@@ -589,10 +596,14 @@ out: + return ret; + } + +-static int make_sure_oci_spec_linux_resources_mem(oci_runtime_spec *oci_spec) ++int make_sure_oci_spec_linux_resources_mem(oci_runtime_spec *oci_spec) + { + int ret = 0; + ++ if (oci_spec == NULL) { ++ return -1; ++ } ++ + ret = make_sure_oci_spec_linux_resources(oci_spec); + if (ret < 0) { + return -1; +@@ -731,8 +742,11 @@ out: + return ret; + } + +-static int make_sure_oci_spec_hooks(oci_runtime_spec *oci_spec) ++int make_sure_oci_spec_hooks(oci_runtime_spec *oci_spec) + { ++ if (oci_spec == NULL) { ++ return -1; ++ } + if (oci_spec->hooks == NULL) { + oci_spec->hooks = util_common_calloc_s(sizeof(oci_runtime_spec_hooks)); + if (oci_spec->hooks == NULL) { +@@ -2827,6 +2841,11 @@ int spec_add_linux_resources_hugepage_limit(oci_runtime_spec *oci_spec, const ch + int ret = 0; + defs_resources_hugepage_limits_element *hugepage_limit = NULL; + ++ if (oci_spec == NULL || page_size == NULL) { ++ ERROR("Invalid arguments"); ++ return -1; ++ } ++ + ret = make_sure_oci_spec_linux_resources(oci_spec); + if (ret < 0) { + return -1; +@@ -2859,6 +2878,11 @@ int spec_add_linux_resources_rlimit(oci_runtime_spec *oci_spec, const char *type + int ret = 0; + defs_process_rlimits_element *rlimit = NULL; + ++ if (oci_spec == NULL || type == NULL) { ++ ERROR("Invalid arguments"); ++ return -1; ++ } ++ + ret = make_sure_oci_spec_linux_resources(oci_spec); + if (ret < 0) { + return -1; +diff --git a/src/daemon/nri/nri_adaption.h b/src/daemon/nri/nri_adaption.h +index 7f0640df..874662cf 100644 +--- a/src/daemon/nri/nri_adaption.h ++++ b/src/daemon/nri/nri_adaption.h +@@ -46,7 +46,6 @@ public: + + auto GetSockpath(std::vector<std::string> &paths) -> bool; + +- // Stop plugins. + auto StopPlugins() -> bool; + + void RemoveClosedPlugins(); +@@ -58,19 +57,23 @@ public: + auto RunPodSandbox(std::shared_ptr<const sandbox::Sandbox> sandbox, Errors &error) ->bool; + auto StopPodSandbox(std::shared_ptr<const sandbox::Sandbox> sandbox, Errors &error) ->bool; + auto RemovePodSandbox(std::shared_ptr<const sandbox::Sandbox> sandbox, Errors &error) ->bool; +- auto CreateContainer(std::shared_ptr<const sandbox::Sandbox> sandbox, const std::string &conId, const runtime::v1::ContainerConfig &containerConfig, nri_container_adjustment **adjust, Errors &error) -> bool; +- auto PostCreateContainer(const std::string &conId, Errors &error) ->bool; +- auto UndoCreateContainer(std::shared_ptr<const sandbox::Sandbox> sandbox, const std::string &conId, Errors &error) -> bool; +- auto StartContainer(const std::string &conId, Errors &error) ->bool; +- auto PostStartContainer(const std::string &conId, Errors &error) ->bool; +- auto UpdateContainer(const std::string &conId, Errors &error) ->bool; +- auto PostUpdateContainer(const std::string &conId, Errors &error) ->bool; +- auto StopContainer(const std::string &conId, Errors &error) ->bool; +- auto RemoveContainer(const std::string &conId, Errors &error) ->bool; +- auto StateChange(nri_state_change_event *evt, Errors &error) ->bool; +- auto updateContainers(const nri_update_containers_request *req, nri_update_containers_response **resp) ->bool; ++ auto CreateContainer(std::shared_ptr<const sandbox::Sandbox> sandbox, const std::string &conId, ++ const runtime::v1::ContainerConfig &containerConfig, nri_container_adjustment **adjust, ++ Errors &error) -> bool; ++ auto PostCreateContainer(const std::string &conId, Errors &error) -> bool; ++ auto UndoCreateContainer(std::shared_ptr<const sandbox::Sandbox> sandbox, const std::string &conId, ++ Errors &error) -> bool; ++ auto StartContainer(const std::string &conId, Errors &error) -> bool; ++ auto PostStartContainer(const std::string &conId, Errors &error) -> bool; ++ auto UpdateContainer(const std::string &conId, Errors &error) -> bool; ++ auto PostUpdateContainer(const std::string &conId, Errors &error) -> bool; ++ auto StopContainer(const std::string &conId, Errors &error) -> bool; ++ auto RemoveContainer(const std::string &conId, Errors &error) -> bool; ++ auto StateChange(nri_state_change_event *evt, Errors &error) -> bool; ++ auto updateContainers(const nri_update_containers_request *req, nri_update_containers_response **resp) -> bool; + + auto NewExternalPlugin(int fd) -> bool; ++ + private: + NRIAdaptation() = default; + NRIAdaptation(const NRIAdaptation &other) = delete; +@@ -86,18 +89,25 @@ private: + auto SortPlugins() -> bool; + void GetClosedPlugins(std::vector<std::string> &closedPlugin); + +- auto IsSupport() -> bool; ++ auto ApplyUpdates(const std::vector<nri_container_update *> &update, std::vector<nri_container_update *> &failed, ++ bool getFailed, Errors &error) -> bool; + +- auto ApplyUpdates(const std::vector<nri_container_update *> &update, std::vector<nri_container_update *> &failed, bool getFailed, +- Errors &error) -> bool; ++ auto IsSupport() -> bool; + +- auto NRIPodSandbox(const std::shared_ptr<const sandbox::Sandbox> &sandbox, Errors& error) -> std::unique_ptr<CStructWrapper<nri_pod_sandbox>>; +- auto NRIContainerByConConfig(const std::shared_ptr<const sandbox::Sandbox> &sandbox, const runtime::v1::ContainerConfig &containerConfig, Errors& error) -> std::unique_ptr<CStructWrapper<nri_container>>; +- auto NRIContainerByID(const std::string &id, Errors& error) -> std::unique_ptr<CStructWrapper<nri_container>>; ++ auto NRIPodSandbox(const std::shared_ptr<const sandbox::Sandbox> &sandbox, ++ Errors &error) -> std::unique_ptr<CStructWrapper<nri_pod_sandbox>>; ++ auto NRIContainerByConConfig(const std::shared_ptr<const sandbox::Sandbox> &sandbox, ++ const runtime::v1::ContainerConfig &containerConfig, Errors &error) -> std::unique_ptr<CStructWrapper<nri_container>>; ++ auto NRIContainerByID(const std::string &id, Errors &error) -> std::unique_ptr<CStructWrapper<nri_container>>; + + auto GetNRIPluginConfigPath(void) -> std::string; + auto GetNRIPluginPath(void) -> std::string; + auto GetNRISockPath(void) -> std::string; ++ ++ void PluginsStateChange(nri_state_change_event *evt); ++ bool PluginsCreateContainer(nri_create_container_request *req, const std::string &conId, pluginResult &result); ++ bool PluginsUpdateContainer(nri_update_container_request *req, const std::string &conId, pluginResult &result); ++ + private: + RWMutex m_mutex; + static std::atomic<NRIAdaptation *> m_instance; +diff --git a/src/daemon/nri/nri_helpers.cc b/src/daemon/nri/nri_helpers.cc +new file mode 100644 +index 00000000..ff9d67c1 +--- /dev/null ++++ b/src/daemon/nri/nri_helpers.cc +@@ -0,0 +1,93 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-07-13 ++ * Description: provide nri helpers functions ++ *********************************************************************************/ ++ ++#include "nri_helpers.h" ++ ++#include <isula_libutils/log.h> ++ ++#include "utils.h" ++#include "isulad_config.h" ++ ++namespace NRIHelpers { ++std::string MarkForRemoval(const std::string &key) ++{ ++ return "-" + key; ++} ++ ++auto GetPluginConfig(std::string &idx, std::string &name, std::string &config) -> bool ++{ ++ __isula_auto_free char *plugin_path = NULL; ++ ++ plugin_path = conf_get_nri_plugin_config_path(); ++ if (plugin_path == NULL) { ++ return false; ++ } ++ std::string compleName = idx + "-" + name; ++ std::vector<std::string> dropIns = { ++ std::string(plugin_path) + "/" + compleName + ".conf", ++ std::string(plugin_path) + "/" + name + ".conf" ++ }; ++ ++ for (const std::string &path : dropIns) { ++ char buf[MAX_BUFFER_SIZE + 1] = { 0 }; ++ __isula_auto_close int fd = util_open(path.c_str(), O_RDONLY, 0); ++ if (fd < 0) { ++ ERROR("Failed to open '%s'", path.c_str()); ++ return false; ++ } ++ int len = util_read_nointr(fd, buf, sizeof(buf) - 1); ++ if (len < 0) { ++ SYSERROR("Failed to read nri plugin config : %s", path.c_str()); ++ return false; ++ } ++ config = std::string(buf); ++ return true; ++ } ++ return true; ++} ++ ++void GenerateRandomExternalName(std::string &ret) ++{ ++ __isula_auto_free char *external_name = NULL; ++ ++ external_name = (char *)util_smart_calloc_s(sizeof(char), (CONTAINER_ID_MAX_LEN + 1)); ++ if (external_name == NULL) { ++ ERROR("Out of memory"); ++ return; ++ } ++ ++ if (util_generate_random_str(external_name, (size_t)CONTAINER_ID_MAX_LEN)) { ++ ERROR("Generate exec suffix failed"); ++ return; ++ } ++ ++ ret = std::string(external_name); ++} ++ ++bool CheckPluginIndex(const std::string &idx) ++{ ++ if (idx.length() != 2) { ++ ERROR("Invalid plugin index \"%s\", must be 2 digits", idx.c_str()); ++ return false; ++ } ++ ++ if (!std::isdigit(idx[0]) || !std::isdigit(idx[1])) { ++ ERROR("Invalid plugin index \"%s\", (not [0-9][0-9])", idx.c_str()); ++ return false; ++ } ++ ++ return true; ++} ++}// namespace NRIHelpers +\ No newline at end of file +diff --git a/src/daemon/nri/nri_helpers.h b/src/daemon/nri/nri_helpers.h +index 06ee8419..1a2f488e 100644 +--- a/src/daemon/nri/nri_helpers.h ++++ b/src/daemon/nri/nri_helpers.h +@@ -37,7 +37,7 @@ std::string MarkForRemoval(const std::string &key); + + auto GetPluginConfig(std::string &idx, std::string &name, std::string &config) -> bool; + +-std::string GenerateRandomExternalName(void); ++void GenerateRandomExternalName(std::string &ret); + + bool CheckPluginIndex(const std::string &idx); + +diff --git a/src/utils/cpputils/transform.cc b/src/utils/cpputils/transform.cc +index 51c154fb..ba8c1f7a 100644 +--- a/src/utils/cpputils/transform.cc ++++ b/src/utils/cpputils/transform.cc +@@ -97,7 +97,7 @@ auto RepeatedPtrFieldToCharArray(const google::protobuf::RepeatedPtrField<std::s + if (result == nullptr) { + return nullptr; + } +- size_t i {}; ++ size_t i = 0; + for (const auto &it : ptrs) { + result[i++] = util_strdup_s(it.c_str()); + } +-- +2.25.1 + diff --git a/0121-get-realpath-before-ns-mountpoint-verification.patch b/0121-get-realpath-before-ns-mountpoint-verification.patch new file mode 100644 index 0000000..3e607ca --- /dev/null +++ b/0121-get-realpath-before-ns-mountpoint-verification.patch @@ -0,0 +1,72 @@ +From 6357caaf6bcf413b58e587fe3df5c508275713ee Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Thu, 15 Aug 2024 19:21:19 +1400 +Subject: [PATCH 120/121] get realpath before ns mountpoint verification + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + .../entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc | 9 +++++++-- + .../entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc | 9 +++++++-- + 2 files changed, 14 insertions(+), 4 deletions(-) + +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +index 77faf48a..3ece885f 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +@@ -424,6 +424,7 @@ cleanup_sandbox: + + void PodSandboxManagerService::ClearCniNetwork(const std::shared_ptr<sandbox::Sandbox> sandbox, Errors &error) + { ++ char real_path[PATH_MAX] = { 0 }; + std::string networkMode = sandbox->GetNetMode(); + if (!namespace_is_cni(networkMode.c_str()) || !sandbox->GetNetworkReady()) { + return; +@@ -435,10 +436,14 @@ void PodSandboxManagerService::ClearCniNetwork(const std::shared_ptr<sandbox::Sa + return; + } + ++ if (realpath(sandboxKey.c_str(), real_path) == NULL) { ++ ERROR("Failed to get %s realpath", sandboxKey.c_str()); ++ } ++ + // If the network namespace is not mounted, the network has been cleaned up + // and there is no need to call the cni plugin. +- if (!util_detect_mounted(sandboxKey.c_str())) { +- WARN("Network namespace %s not exist", sandboxKey.c_str()); ++ if (strlen(real_path) != 0 && !util_detect_mounted(real_path)) { ++ ERROR("Network namespace %s not exist", real_path); + return; + } + +diff --git a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +index 5590827e..1c343cda 100644 +--- a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +@@ -826,6 +826,7 @@ auto PodSandboxManagerService::ClearCniNetwork(const std::string &realSandboxID, + /*error*/) -> int + { + Errors networkErr; ++ char real_path[PATH_MAX] = { 0 }; + + bool ready = GetNetworkReady(realSandboxID, networkErr); + if (hostNetwork || (!ready && networkErr.Empty())) { +@@ -848,10 +849,14 @@ auto PodSandboxManagerService::ClearCniNetwork(const std::string &realSandboxID, + goto cleanup; + } + ++ if (realpath(netnsPath.c_str(), real_path) == NULL) { ++ ERROR("Failed to get %s realpath", netnsPath.c_str()); ++ } ++ + // If the network namespace is not mounted, the network has been cleaned up + // and there is no need to call the cni plugin. +- if (!util_detect_mounted(netnsPath.c_str())) { +- WARN("Network namespace %s not exist", netnsPath.c_str()); ++ if (strlen(real_path) != 0 && !util_detect_mounted(real_path)) { ++ ERROR("Network namespace %s not exist", real_path); + goto cleanup; + } + +-- +2.25.1 + diff --git a/0122-nri-impl-for-nri-plugin-and-adaption.patch b/0122-nri-impl-for-nri-plugin-and-adaption.patch new file mode 100644 index 0000000..fc13950 --- /dev/null +++ b/0122-nri-impl-for-nri-plugin-and-adaption.patch @@ -0,0 +1,4724 @@ +From 343be112a3d5cfd3857f166fb5dc2473534e64d7 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Thu, 15 Aug 2024 15:19:25 +0800 +Subject: [PATCH 121/121] [nri] impl for nri plugin and adaption + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/cmd/isulad/main.c | 15 + + src/daemon/common/nri/nri_convert.cc | 148 ++- + src/daemon/common/nri/nri_convert.h | 7 +- + src/daemon/common/nri/nri_spec.c | 223 +++- + src/daemon/common/nri/nri_spec.h | 6 + + src/daemon/common/nri/nri_utils.c | 476 +++---- + src/daemon/common/nri/nri_utils.h | 14 +- + src/daemon/config/isulad_config.c | 2 +- + .../v1/v1_cri_container_manager_service.cc | 104 +- + .../v1/v1_cri_pod_sandbox_manager_service.cc | 37 +- + .../executor/container_cb/execution_create.c | 21 + + .../modules/service/service_container.c | 12 + + src/daemon/modules/spec/specs.c | 7 +- + src/daemon/nri/nri_adaption.cc | 1165 +++++++++++++++++ + src/daemon/nri/nri_adaption.h | 13 +- + src/daemon/nri/nri_helpers.cc | 21 + + src/daemon/nri/nri_helpers.h | 4 + + src/daemon/nri/nri_plugin_ops.cc | 123 ++ + src/daemon/nri/nri_plugin_ops.h | 6 +- + src/daemon/nri/nri_result.cc | 977 ++++++++++++++ + src/daemon/nri/nri_result.h | 17 +- + src/daemon/nri/plugin.cc | 417 ++++++ + src/daemon/nri/plugin.h | 20 +- + src/utils/cutils/utils.c | 2 +- + 24 files changed, 3527 insertions(+), 310 deletions(-) + create mode 100644 src/daemon/nri/nri_adaption.cc + create mode 100644 src/daemon/nri/nri_plugin_ops.cc + create mode 100644 src/daemon/nri/nri_result.cc + create mode 100644 src/daemon/nri/plugin.cc + +diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c +index 52ac3172..0228caa8 100644 +--- a/src/cmd/isulad/main.c ++++ b/src/cmd/isulad/main.c +@@ -86,6 +86,9 @@ + #ifdef ENABLE_CDI + #include "cdi_operate_api.h" + #endif /* ENABLE_CDI */ ++#ifdef ENABLE_NRI ++#include "nri_plugin_ops.h" ++#endif + + sem_t g_daemon_shutdown_sem; + sem_t g_daemon_wait_shutdown_sem; +@@ -289,6 +292,11 @@ static void daemon_shutdown() + EVENT("Network module exit completed"); + #endif + ++#ifdef ENABLE_NRI ++ nri_adaption_shutdown(); ++ EVENT("nri module exit completed"); ++#endif ++ + clean_residual_files(); + EVENT("Clean residual files completed"); + +@@ -1834,6 +1842,13 @@ int main(int argc, char **argv) + } + #endif + ++#ifdef ENABLE_NRI ++ if (!nri_adaption_init()) { ++ ERROR("Failed to init nri adaption"); ++ goto failure; ++ } ++#endif ++ + clock_gettime(CLOCK_MONOTONIC, &t_end); + use_time = (double)(t_end.tv_sec - t_start.tv_sec) * (double)1000000000 + (double)(t_end.tv_nsec - t_start.tv_nsec); + use_time /= 1000000000; +diff --git a/src/daemon/common/nri/nri_convert.cc b/src/daemon/common/nri/nri_convert.cc +index 7cce64ec..30caf1dd 100644 +--- a/src/daemon/common/nri/nri_convert.cc ++++ b/src/daemon/common/nri/nri_convert.cc +@@ -20,6 +20,7 @@ + #include "path.h" + #include "transform.h" + #include "nri_utils.h" ++#include "cstruct_wrapper.h" + + static int64_t DefaultOOMScoreAdj = 0; + +@@ -142,22 +143,28 @@ static auto NRILinuxResourcesFromCRI(const runtime::v1::LinuxContainerResources + + static auto NRILinuxFromCRI(const runtime::v1::LinuxPodSandboxConfig &config, nri_linux_pod_sandbox &linux) -> bool + { +- if (!init_nri_linux_resources(&linux.pod_overhead)) { +- ERROR("Failed to init nri linux overhead resources for pod"); +- return false; +- } +- if (!init_nri_linux_resources(&linux.pod_resources)) { +- ERROR("Failed to init nri linux resources resources for pod"); +- return false; +- } +- if (config.has_overhead() && !NRILinuxResourcesFromCRI(config.overhead(), *linux.pod_overhead)) { +- ERROR("Failed to transform overhead to nri for pod"); +- return false; ++ if (config.has_overhead()) { ++ linux.pod_overhead = init_nri_linux_resources(); ++ if (linux.pod_overhead == nullptr) { ++ ERROR("Failed to init nri linux overhead resources for pod"); ++ return false; ++ } ++ if (!NRILinuxResourcesFromCRI(config.overhead(), *linux.pod_overhead)) { ++ ERROR("Failed to transform overhead to nri for pod"); ++ return false; ++ } + } + +- if (config.has_resources() && !NRILinuxResourcesFromCRI(config.resources(), *linux.pod_resources)) { +- ERROR("Failed to transform resources to nri for pod"); +- return false; ++ if (config.has_resources()) { ++ linux.pod_resources = init_nri_linux_resources(); ++ if (linux.pod_resources == nullptr) { ++ ERROR("Failed to init nri linux resources resources for pod"); ++ return false; ++ } ++ if (!NRILinuxResourcesFromCRI(config.resources(), *linux.pod_resources)) { ++ ERROR("Failed to transform resources to nri for pod"); ++ return false; ++ } + } + + linux.cgroup_parent = util_strdup_s(config.cgroup_parent().c_str()); +@@ -507,13 +514,13 @@ auto LinuxResourcesFromNRI(const nri_linux_resources *src, runtime::v1::LinuxCon + } + + if (src->cpu != nullptr) { +- if (src->cpu->shares != NULL) { ++ if (src->cpu->shares != nullptr) { + resources.set_cpu_shares(*src->cpu->shares); + } +- if (src->cpu->quota != NULL) { ++ if (src->cpu->quota != nullptr) { + resources.set_cpu_quota(*src->cpu->quota); + } +- if (src->cpu->period != NULL) { ++ if (src->cpu->period != nullptr) { + resources.set_cpu_period(*src->cpu->period); + } + +@@ -535,5 +542,112 @@ auto LinuxResourcesFromNRI(const nri_linux_resources *src, runtime::v1::LinuxCon + Transform::JsonMapToProtobufMapForString(src->unified, *resources.mutable_unified()); + } + ++ return true; ++} ++ ++auto LinuxResourcesToNRI(const runtime::v1::LinuxContainerResources &src) -> nri_linux_resources * ++{ ++ nri_linux_resources *resources = nullptr; ++ ++ resources = init_nri_linux_resources(); ++ if (resources == nullptr) { ++ ERROR("Failed to init nri linux resources"); ++ return nullptr; ++ } ++ ++ resources->cpu->shares = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if (resources->cpu->shares == nullptr) { ++ ERROR("Out of memory"); ++ goto error_out; ++ } ++ *(resources->cpu->shares) = src.cpu_shares(); ++ ++ resources->cpu->quota = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (resources->cpu->quota == nullptr) { ++ ERROR("Out of memory"); ++ goto error_out; ++ } ++ *(resources->cpu->quota) = src.cpu_quota(); ++ ++ resources->cpu->period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if (resources->cpu->period == nullptr) { ++ ERROR("Out of memory"); ++ goto error_out; ++ } ++ *(resources->cpu->period) = src.cpu_period(); ++ ++ resources->cpu->cpus = util_strdup_s(src.cpuset_cpus().c_str()); ++ resources->cpu->mems = util_strdup_s(src.cpuset_mems().c_str()); ++ ++ resources->memory->limit = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (resources->memory->limit == nullptr) { ++ ERROR("Out of memory"); ++ goto error_out; ++ } ++ *(resources->memory->limit) = src.memory_limit_in_bytes(); ++ ++ resources->hugepage_limits = (nri_hugepage_limit **)util_smart_calloc_s(sizeof(nri_hugepage_limit *), ++ src.hugepage_limits_size()); ++ if (resources->hugepage_limits == nullptr) { ++ ERROR("Out of memory"); ++ goto error_out; ++ } ++ ++ for (int i = 0; i < src.hugepage_limits_size(); i++) { ++ resources->hugepage_limits[i] = (nri_hugepage_limit *)util_common_calloc_s(sizeof(nri_hugepage_limit)); ++ if (resources->hugepage_limits[i] == nullptr) { ++ ERROR("Out of memory"); ++ goto error_out; ++ } ++ resources->hugepage_limits[i]->page_size = util_strdup_s(src.hugepage_limits(i).page_size().c_str()); ++ resources->hugepage_limits[i]->limit = src.hugepage_limits(i).limit(); ++ resources->hugepage_limits_len++; ++ } ++ ++ return resources; ++ ++error_out: ++ free_nri_linux_resources(resources); ++ resources = nullptr; ++ return resources; ++} ++ ++auto PodSandboxesToNRI(const std::vector<std::shared_ptr<sandbox::Sandbox>> &arrs, ++ std::vector<nri_pod_sandbox *> &pods) -> bool ++{ ++ size_t i = 0; ++ for (i = 0; i < arrs.size(); i++) { ++ nri_pod_sandbox *pod = (nri_pod_sandbox *)util_common_calloc_s(sizeof(nri_pod_sandbox)); ++ if (pod == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ if (!PodSandboxToNRI(arrs[i], *pod)) { ++ ERROR("Failed to transform pod to nri for pod : %s", arrs[i]->GetName().c_str()); ++ return false; ++ } ++ pods.push_back(pod); ++ } ++ ++ return true; ++} ++ ++auto ContainersToNRI(std::vector<std::unique_ptr<runtime::v1::Container>> &containers, ++ std::vector<nri_container *> &cons) -> bool ++{ ++ size_t i = 0; ++ for (i = 0; i < containers.size(); i++) { ++ nri_container *con = (nri_container *)util_common_calloc_s(sizeof(nri_container)); ++ if (con == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ if (!ContainerToNRIByID(containers[i].get()->id(), *con)) { ++ ERROR("Failed to transform container to nri for container : %s", containers[i]->metadata().name().c_str()); ++ return false; ++ } ++ cons.push_back(con); ++ } ++ + return true; + } +\ No newline at end of file +diff --git a/src/daemon/common/nri/nri_convert.h b/src/daemon/common/nri/nri_convert.h +index c04b14e4..a0caf6ce 100644 +--- a/src/daemon/common/nri/nri_convert.h ++++ b/src/daemon/common/nri/nri_convert.h +@@ -30,8 +30,11 @@ + auto PodSandboxToNRI(const std::shared_ptr<const sandbox::Sandbox> &sandbox, nri_pod_sandbox &pod) -> bool; + auto ContainerToNRIByConConfig(const runtime::v1::ContainerConfig &containerConfig, nri_container &con) -> bool; + auto ContainerToNRIByID(const std::string &id, nri_container &con) -> bool; +-auto PodSandboxesToNRI(const std::vector<std::unique_ptr<sandbox::Sandbox>> &arrs, nri_pod_sandbox **pod, +- int pod_len) -> bool; ++auto PodSandboxesToNRI(const std::vector<std::shared_ptr<sandbox::Sandbox>> &arrs, ++ std::vector<nri_pod_sandbox *> &pods) -> bool; ++auto ContainersToNRI(std::vector<std::unique_ptr<runtime::v1::Container>> &containers, ++ std::vector<nri_container *> &cons) -> bool; + + auto LinuxResourcesFromNRI(const nri_linux_resources *src, runtime::v1::LinuxContainerResources &resources) -> bool; ++auto LinuxResourcesToNRI(const runtime::v1::LinuxContainerResources &src) -> nri_linux_resources *; + #endif // DAEMON_COMMON_NRI_NRI_CONVERT_H +diff --git a/src/daemon/common/nri/nri_spec.c b/src/daemon/common/nri/nri_spec.c +index 855fe3b3..327d31e1 100644 +--- a/src/daemon/common/nri/nri_spec.c ++++ b/src/daemon/common/nri/nri_spec.c +@@ -176,6 +176,7 @@ static int nri_adjust_annotation(const nri_container_adjustment *adjust, oci_run + + free_json_map_string_string(oci_spec->annotations); + oci_spec->annotations = cleard; ++ cleard = NULL; + ret = 0; + + free_out: +@@ -330,7 +331,34 @@ static int nri_adjust_hooks(const nri_container_adjustment *adjust, oci_runtime_ + return ret; + } + +-static int nri_adjust_devices(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++static int host_spec_add_device(host_config *spec, defs_device *device) ++{ ++ size_t i; ++ ++ if (device == NULL) { ++ return -1; ++ } ++ ++ for (i = 0; i < spec->nri_devices_len; i++) { ++ if (strcmp(spec->nri_devices[i]->path, device->path) == 0) { ++ free_defs_device(spec->nri_devices[i]); ++ spec->nri_devices[i] = device; ++ return 0; ++ } ++ } ++ ++ if (util_mem_realloc((void **)&spec->nri_devices, (spec->nri_devices_len + 1) * sizeof(defs_device *), ++ (void *)spec->nri_devices, spec->nri_devices_len * sizeof(defs_device *)) != 0) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ spec->nri_devices[spec->nri_devices_len] = device; ++ spec->nri_devices_len++; ++ ++ return 0; ++} ++ ++static int nri_adjust_oci_spec_devices(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) + { + if (adjust->linux == NULL || adjust->linux->devices == NULL || adjust->linux->devices_len == 0) { + return 0; +@@ -349,6 +377,25 @@ static int nri_adjust_devices(const nri_container_adjustment *adjust, oci_runtim + return 0; + } + ++static int nri_adjust_host_spec_devices(const nri_container_adjustment *adjust, host_config *spec) ++{ ++ if (adjust->linux == NULL || adjust->linux->devices == NULL || adjust->linux->devices_len == 0) { ++ return 0; ++ } ++ ++ size_t i; ++ ++ for (i = 0; i < adjust->linux->devices_len; i++) { ++ nri_linux_device *dev = adjust->linux->devices[i]; ++ if (host_spec_add_device(spec, nri_device_to_oci(dev)) != 0) { ++ ERROR("Failed to add device %s", dev->path); ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ + static int nri_adjust_cgroup_path(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) + { + if (adjust->linux == NULL || adjust->linux->cgroups_path == NULL) { +@@ -500,7 +547,50 @@ static int nri_adjust_mounts(const nri_container_adjustment *adjust, oci_runtime + return 0; + } + +-static int nri_adjust_rlimit(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++static int host_spec_add_linux_resources_rlimit(host_config *spec, const char *type, uint64_t hard, uint64_t soft) ++{ ++ size_t j; ++ bool exists = false; ++ host_config_nri_rlimits_element *rlimit = NULL; ++ ++ if (spec == NULL || type == NULL) { ++ ERROR("Invalid arguments"); ++ return -1; ++ } ++ ++ rlimit = util_common_calloc_s(sizeof(host_config_nri_rlimits_element)); ++ if (rlimit == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ rlimit->type = util_strdup_s(type); ++ rlimit->hard = hard; ++ rlimit->soft = soft; ++ ++ for (j = 0; j < spec->nri_rlimits_len; j++) { ++ if (spec->nri_rlimits[j]->type == NULL) { ++ ERROR("rlimit type is empty"); ++ free_host_config_nri_rlimits_element(rlimit); ++ return -1; ++ } ++ if (strcmp(spec->nri_rlimits[j]->type, rlimit->type) == 0) { ++ exists = true; ++ break; ++ } ++ } ++ if (exists) { ++ /* override ulimit */ ++ free_host_config_nri_rlimits_element(spec->nri_rlimits[j]); ++ spec->nri_rlimits[j] = rlimit; ++ } else { ++ spec->nri_rlimits[spec->nri_rlimits_len] = rlimit; ++ spec->nri_rlimits_len++; ++ } ++ ++ return 0; ++} ++ ++static int nri_adjust_oci_spec_rlimit(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) + { + if (adjust->rlimits == NULL || adjust->rlimits_len == 0) { + return 0; +@@ -522,6 +612,36 @@ static int nri_adjust_rlimit(const nri_container_adjustment *adjust, oci_runtime + return 0; + } + ++static int nri_adjust_host_spec_rlimit(const nri_container_adjustment *adjust, host_config *spec) ++{ ++ if (adjust->rlimits == NULL || adjust->rlimits_len == 0) { ++ return 0; ++ } ++ ++ if (spec->nri_rlimits == NULL) { ++ spec->nri_rlimits = util_common_calloc_s(sizeof(host_config_nri_rlimits_element *) * adjust->rlimits_len); ++ if (spec->nri_rlimits == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ } ++ ++ size_t i; ++ for (i = 0; i < adjust->rlimits_len; i++) { ++ nri_posix_rlimit *rlimit = adjust->rlimits[i]; ++ if (rlimit->type == NULL) { ++ ERROR("Invalid rlimit type"); ++ return -1; ++ } ++ if (host_spec_add_linux_resources_rlimit(spec, rlimit->type, rlimit->soft, rlimit->hard) != 0) { ++ ERROR("Failed to add rlimit"); ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ + // todo: we do not support it blockio_class + static int nri_adjust_blockio_class(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) + { +@@ -554,7 +674,7 @@ int nri_adjust_oci_spec(const nri_container_adjustment *adjust, oci_runtime_spec + return -1; + } + +- if (nri_adjust_devices(adjust, oci_spec) != 0) { ++ if (nri_adjust_oci_spec_devices(adjust, oci_spec) != 0) { + ERROR("Failed to do nri adjust devices in oci spec"); + return -1; + } +@@ -580,7 +700,7 @@ int nri_adjust_oci_spec(const nri_container_adjustment *adjust, oci_runtime_spec + return -1; + } + +- if (nri_adjust_rlimit(adjust, oci_spec) != 0) { ++ if (nri_adjust_oci_spec_rlimit(adjust, oci_spec) != 0) { + ERROR("Failed to do nri adjust rlimit in oci spec"); + return -1; + } +@@ -598,5 +718,100 @@ int nri_adjust_oci_spec(const nri_container_adjustment *adjust, oci_runtime_spec + return -1; + } + ++ return 0; ++} ++ ++int nri_adjust_host_spec(const nri_container_adjustment *adjust, host_config *host_spec) ++{ ++ if (adjust == NULL || host_spec == NULL) { ++ ERROR("Invalid input arguments"); ++ return -1; ++ } ++ ++ if (nri_adjust_host_spec_devices(adjust, host_spec) != 0) { ++ ERROR("Failed to do nri adjust devices in host config"); ++ return -1; ++ } ++ ++ if (nri_adjust_host_spec_rlimit(adjust, host_spec) != 0) { ++ ERROR("Failed to do nri adjust rlimit in host config"); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++static defs_device *copy_def_devices(const defs_device *dev) ++{ ++ defs_device *tmp_dev = util_common_calloc_s(sizeof(defs_device)); ++ if (tmp_dev == NULL) { ++ return NULL; ++ } ++ tmp_dev->type = util_strdup_s(dev->type); ++ tmp_dev->path = util_strdup_s(dev->path); ++ tmp_dev->file_mode = dev->file_mode; ++ tmp_dev->major = dev->major; ++ tmp_dev->minor = dev->minor; ++ tmp_dev->uid = dev->uid; ++ tmp_dev->gid = dev->gid; ++ return tmp_dev; ++} ++ ++static int merge_nri_devices(oci_runtime_spec *oci_spec, host_config *host_spec) ++{ ++ size_t i; ++ ++ if (oci_spec == NULL || host_spec == NULL) { ++ ERROR("Invalid input arguments"); ++ return -1; ++ } ++ ++ for (i = 0; i < host_spec->nri_devices_len; i++) { ++ if (spec_add_device(oci_spec, copy_def_devices(host_spec->nri_devices[i])) != 0) { ++ ERROR("Failed to add device %s", host_spec->nri_devices[i]->path); ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ ++static int merge_nri_ulimits(oci_runtime_spec *oci_spec, host_config *host_spec) ++{ ++ size_t i; ++ ++ if (oci_spec == NULL || host_spec == NULL) { ++ ERROR("Invalid input arguments"); ++ return -1; ++ } ++ ++ for (i = 0; i < host_spec->nri_rlimits_len; i++) { ++ host_config_nri_rlimits_element *rlimit = host_spec->nri_rlimits[i]; ++ if (spec_add_linux_resources_rlimit(oci_spec, rlimit->type, rlimit->hard, rlimit->soft) != 0) { ++ ERROR("Failed to add rlimit %s", rlimit->type); ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ ++int update_oci_nri(oci_runtime_spec *oci_spec, host_config *host_spec) ++{ ++ if (oci_spec == NULL || host_spec == NULL) { ++ ERROR("Invalid input arguments"); ++ return -1; ++ } ++ ++ if (merge_nri_devices(oci_spec, host_spec) != 0) { ++ ERROR("Failed to merge nri devices"); ++ return -1; ++ } ++ ++ if (merge_nri_ulimits(oci_spec, host_spec) != 0) { ++ ERROR("Failed to merge nri ulimits"); ++ return -1; ++ } ++ + return 0; + } +\ No newline at end of file +diff --git a/src/daemon/common/nri/nri_spec.h b/src/daemon/common/nri/nri_spec.h +index e7c5035d..1a622284 100644 +--- a/src/daemon/common/nri/nri_spec.h ++++ b/src/daemon/common/nri/nri_spec.h +@@ -18,7 +18,13 @@ + + #include <isula_libutils/oci_runtime_spec.h> + #include <isula_libutils/nri_container_adjustment.h> ++#include <isula_libutils/host_config.h> + + int nri_adjust_oci_spec(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec); + ++// the device and ulimit will be updated when starting, so they need to be stored in host-spec ++int nri_adjust_host_spec(const nri_container_adjustment *adjust, host_config *host_spec); ++ ++int update_oci_nri(oci_runtime_spec *oci_spec, host_config *host_spec); ++ + #endif // DAEMON_COMMON_NRI_NRI_SPEC_H +\ No newline at end of file +diff --git a/src/daemon/common/nri/nri_utils.c b/src/daemon/common/nri/nri_utils.c +index 51054e32..d2b62117 100644 +--- a/src/daemon/common/nri/nri_utils.c ++++ b/src/daemon/common/nri/nri_utils.c +@@ -19,218 +19,235 @@ + + #include "utils.h" + +-static bool copy_nri_hugepage_limit(const nri_hugepage_limit* src, nri_hugepage_limit** dest) ++static nri_hugepage_limit*copy_nri_hugepage_limit(const nri_hugepage_limit *src) + { +- if (src == NULL || dest == NULL) { ++ nri_hugepage_limit *dest = NULL; ++ if (src == NULL) { + ERROR("Invalid input arguments"); + return false; + } + +- *dest = (nri_hugepage_limit *)util_common_calloc_s(sizeof(nri_hugepage_limit)); +- if (*dest == NULL) { ++ dest = (nri_hugepage_limit *)util_common_calloc_s(sizeof(nri_hugepage_limit)); ++ if (dest == NULL) { + ERROR("Out of memory"); + return false; + } + +- (*dest)->limit = src->limit; +- (*dest)->page_size = util_strdup_s(src->page_size); +- return true; ++ dest->limit = src->limit; ++ dest->page_size = util_strdup_s(src->page_size); ++ return dest; + } + +-static bool copy_nri_hook(const nri_hook *src, nri_hook **dest) ++static nri_hook *copy_nri_hook(const nri_hook *src) + { +- if (src == NULL || dest == NULL) { ++ nri_hook *dest = NULL; ++ if (src == NULL) { + ERROR("Invalid input arguments"); + return false; + } + +- *dest = (nri_hook *)util_common_calloc_s(sizeof(nri_hook)); ++ dest = (nri_hook *)util_common_calloc_s(sizeof(nri_hook)); + if (dest == NULL) { + ERROR("Out of memory"); + return false; + } +- (*dest)->args = util_copy_array_by_len(src->args, src->args_len); +- (*dest)->args_len = src->args_len; +- (*dest)->env = util_copy_array_by_len(src->env, src->env_len); +- (*dest)->env_len = src->env_len; +- (*dest)->path = util_strdup_s(src->path); +- return true; ++ dest->args = util_copy_array_by_len(src->args, src->args_len); ++ dest->args_len = src->args_len; ++ dest->env = util_copy_array_by_len(src->env, src->env_len); ++ dest->env_len = src->env_len; ++ dest->path = util_strdup_s(src->path); ++ return dest; + } + +-static bool copy_nri_linux_device_cgroup(const nri_linux_device_cgroup *src, nri_linux_device_cgroup **dest) ++static nri_linux_device_cgroup *copy_nri_linux_device_cgroup(const nri_linux_device_cgroup *src) + { +- if (src == NULL || dest == NULL) { ++ nri_linux_device_cgroup *dest = NULL; ++ if (src == NULL) { + ERROR("Invalid input arguments"); + return false; + } + +- *dest = (nri_linux_device_cgroup *)util_common_calloc_s(sizeof(nri_linux_device_cgroup)); ++ dest = (nri_linux_device_cgroup *)util_common_calloc_s(sizeof(nri_linux_device_cgroup)); + if (dest == NULL) { + ERROR("Out of memory"); + return false; + } +- (*dest)->allow = src->allow; +- (*dest)->type = util_strdup_s(src->type); +- (*dest)->major = (int64_t *)util_common_calloc_s(sizeof(int64_t)); +- if ((*dest)->major == NULL) { ++ dest->allow = src->allow; ++ dest->type = util_strdup_s(src->type); ++ dest->major = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (dest->major == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- (*dest)->minor = (int64_t *)util_common_calloc_s(sizeof(int64_t)); +- if ((*dest)->minor == NULL) { ++ dest->minor = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (dest->minor == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- (*dest)->access = util_strdup_s(src->access); +- return true; ++ dest->access = util_strdup_s(src->access); ++ return dest; ++free_out: ++ free_nri_linux_device_cgroup(dest); ++ return NULL; + } + +-static bool copy_nri_linux_cpu(const nri_linux_cpu *src, nri_linux_cpu **dest) ++static nri_linux_cpu *copy_nri_linux_cpu(const nri_linux_cpu *src) + { +- if (src == NULL || dest == NULL) { ++ nri_linux_cpu *dest = NULL; ++ if (src == NULL) { + ERROR("Invalid input arguments"); + return false; + } + +- (*dest) = (nri_linux_cpu *)util_common_calloc_s(sizeof(nri_linux_cpu)); +- if ((*dest) == NULL) { ++ dest = (nri_linux_cpu *)util_common_calloc_s(sizeof(nri_linux_cpu)); ++ if (dest == NULL) { + ERROR("Out of memory"); + return false; + } +- (*dest)->cpus = util_strdup_s(src->cpus); +- (*dest)->mems = util_strdup_s(src->mems); ++ dest->cpus = util_strdup_s(src->cpus); ++ dest->mems = util_strdup_s(src->mems); + if (src->period != NULL) { +- (*dest)->period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); +- if ((*dest)->period == NULL) { ++ dest->period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if (dest->period == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->period = *src->period; ++ *dest->period = *src->period; + } + + if (src->quota != NULL) { +- (*dest)->quota = (int64_t *)util_common_calloc_s(sizeof(int64_t)); +- if ((*dest)->quota == NULL) { ++ dest->quota = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (dest->quota == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->quota = *src->quota; ++ *dest->quota = *src->quota; + } + + if (src->realtime_period != NULL) { +- (*dest)->realtime_period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); +- if ((*dest)->realtime_period == NULL) { ++ dest->realtime_period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if (dest->realtime_period == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->realtime_period = *src->realtime_period; ++ *dest->realtime_period = *src->realtime_period; + } + + if (src->realtime_runtime != NULL) { +- (*dest)->realtime_runtime = (int64_t *)util_common_calloc_s(sizeof(int64_t)); +- if ((*dest)->realtime_runtime == NULL) { ++ dest->realtime_runtime = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (dest->realtime_runtime == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->realtime_runtime = *src->realtime_runtime; ++ *dest->realtime_runtime = *src->realtime_runtime; + } + + if (src->shares != NULL) { +- (*dest)->shares = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); +- if ((*dest)->shares == NULL) { ++ dest->shares = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if (dest->shares == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->shares = *src->shares; ++ *dest->shares = *src->shares; + } + +- return true; ++ return dest; ++ ++free_out: ++ free_nri_linux_cpu(dest); ++ return NULL; + } + +-static bool copy_nri_linux_memory(const nri_linux_memory *src, nri_linux_memory **dest) ++static nri_linux_memory *copy_nri_linux_memory(const nri_linux_memory *src) + { +- if (src == NULL || dest == NULL) { ++ nri_linux_memory *dest = NULL; ++ if (src == NULL) { + ERROR("Invalid input arguments"); + return false; + } +- *dest = (nri_linux_memory *)util_common_calloc_s(sizeof(nri_linux_memory)); ++ ++ dest = (nri_linux_memory *)util_common_calloc_s(sizeof(nri_linux_memory)); + if (dest == NULL) { + ERROR("Out of memory"); + return false; + } + if (src->limit != NULL) { +- (*dest)->limit = (int64_t *)util_common_calloc_s(sizeof(int64_t)); +- if ((*dest)->limit == NULL) { ++ dest->limit = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (dest->limit == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->limit = *src->limit; ++ *dest->limit = *src->limit; + } + + if (src->reservation != NULL) { +- (*dest)->reservation = (int64_t *)util_common_calloc_s(sizeof(int64_t)); +- if ((*dest)->reservation == NULL) { ++ dest->reservation = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (dest->reservation == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->reservation = *src->reservation; ++ *dest->reservation = *src->reservation; + } + + if (src->swap != NULL) { +- (*dest)->swap = (int64_t *)util_common_calloc_s(sizeof(int64_t)); +- if ((*dest)->swap == NULL) { ++ dest->swap = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (dest->swap == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->swap = *src->swap; ++ *dest->swap = *src->swap; + } + + if (src->kernel != NULL) { +- (*dest)->kernel = (int64_t *)util_common_calloc_s(sizeof(int64_t)); +- if ((*dest)->kernel == NULL) { ++ dest->kernel = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (dest->kernel == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->kernel = *src->kernel; ++ *dest->kernel = *src->kernel; + } + + + if (src->kernel_tcp != NULL) { +- (*dest)->kernel_tcp = (int64_t *)util_common_calloc_s(sizeof(int64_t)); +- if ((*dest)->kernel_tcp == NULL) { ++ dest->kernel_tcp = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (dest->kernel_tcp == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->kernel_tcp = *src->kernel_tcp; ++ *dest->kernel_tcp = *src->kernel_tcp; + } + + if (src->swappiness != NULL) { +- (*dest)->swappiness = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); +- if ((*dest)->swappiness == NULL) { ++ dest->swappiness = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if (dest->swappiness == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->swappiness = *src->swappiness; ++ *dest->swappiness = *src->swappiness; + } + + if (src->disable_oom_killer != NULL) { +- (*dest)->disable_oom_killer = (uint8_t *)util_common_calloc_s(sizeof(uint8_t)); +- if ((*dest)->disable_oom_killer == NULL) { ++ dest->disable_oom_killer = (uint8_t *)util_common_calloc_s(sizeof(uint8_t)); ++ if (dest->disable_oom_killer == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->disable_oom_killer = *src->disable_oom_killer; ++ *dest->disable_oom_killer = *src->disable_oom_killer; + } + + if (src->use_hierarchy != NULL) { +- (*dest)->use_hierarchy = (uint8_t *)util_common_calloc_s(sizeof(uint8_t)); +- if ((*dest)->use_hierarchy == NULL) { ++ dest->use_hierarchy = (uint8_t *)util_common_calloc_s(sizeof(uint8_t)); ++ if (dest->use_hierarchy == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->use_hierarchy = *src->use_hierarchy; ++ *dest->use_hierarchy = *src->use_hierarchy; + } +- return true; ++ return dest; ++ ++free_out: ++ free_nri_linux_memory(dest); ++ return NULL; + } + + bool is_marked_for_removal(const char* key, char **out) +@@ -241,7 +258,7 @@ bool is_marked_for_removal(const char* key, char **out) + } + + if (!util_has_prefix(key, "-")) { +- *out = (char*)key; ++ *out = util_strdup_s(key); + return false; + } + +@@ -254,94 +271,153 @@ bool is_marked_for_removal(const char* key, char **out) + return true; + } + +-bool copy_nri_mount(const nri_mount *src, nri_mount **dest) ++nri_mount *copy_nri_mount(const nri_mount *src) + { +- if (src == NULL || dest == NULL) { ++ nri_mount *dest = NULL; ++ if (src == NULL) { + ERROR("Invalid input arguments"); + return false; + } +- *dest = (nri_mount *)util_common_calloc_s(sizeof(nri_mount)); ++ ++ dest = (nri_mount *)util_common_calloc_s(sizeof(nri_mount)); + if (dest == NULL) { + ERROR("Out of memory"); + return false; + } +- (*dest)->destination = util_strdup_s(src->destination); +- (*dest)->options = util_copy_array_by_len(src->options, src->options_len); +- (*dest)->options_len = src->options_len; +- (*dest)->source = util_strdup_s(src->source); +- (*dest)->type = util_strdup_s(src->type); +- return true; ++ dest->destination = util_strdup_s(src->destination); ++ dest->options = util_copy_array_by_len(src->options, src->options_len); ++ dest->options_len = src->options_len; ++ dest->source = util_strdup_s(src->source); ++ dest->type = util_strdup_s(src->type); ++ return dest; + } + +-bool copy_nri_key_value(const nri_key_value *src, nri_key_value **dest) ++nri_linux_device *copy_nri_device(const nri_linux_device *src) + { +- if (src == NULL || dest == NULL) { ++ nri_linux_device *dest = NULL; ++ ++ if (src == NULL) { + ERROR("Invalid input arguments"); + return false; + } +- *dest = (nri_key_value *)util_common_calloc_s(sizeof(nri_key_value)); ++ ++ dest = (nri_linux_device *)util_common_calloc_s(sizeof(nri_linux_device)); + if (dest == NULL) { + ERROR("Out of memory"); + return false; + } +- (*dest)->key = util_strdup_s(src->key); +- (*dest)->value = util_strdup_s(src->value); +- return true; ++ if (src->file_mode != NULL) { ++ dest->file_mode = (uint32_t *)util_common_calloc_s(sizeof(uint32_t)); ++ if (dest->file_mode == NULL) { ++ ERROR("Out of memory"); ++ goto free_out; ++ } ++ *dest->file_mode = *src->file_mode; ++ } ++ ++ if (src->uid != NULL) { ++ dest->uid = (uint32_t *)util_common_calloc_s(sizeof(uint32_t)); ++ if (dest->uid == NULL) { ++ ERROR("Out of memory"); ++ goto free_out; ++ } ++ *dest->uid = *src->uid; ++ } ++ ++ if (src->gid != NULL) { ++ dest->gid = (uint32_t *)util_common_calloc_s(sizeof(uint32_t)); ++ if (dest->gid == NULL) { ++ ERROR("Out of memory"); ++ goto free_out; ++ } ++ *dest->gid = *src->gid; ++ } ++ ++ dest->major = src->major; ++ dest->minor = src->minor; ++ dest->path = util_strdup_s(src->path); ++ dest->type = util_strdup_s(src->type); ++ ++ return dest; ++free_out: ++ free_nri_linux_device(dest); ++ return NULL; + } + +-bool copy_nri_posix_rlimit(const nri_posix_rlimit *src, nri_posix_rlimit **dest) ++nri_key_value *copy_nri_key_value(const nri_key_value *src) + { +- if (src == NULL || dest == NULL) { ++ nri_key_value *dest = NULL; ++ if (src == NULL) { + ERROR("Invalid input arguments"); + return false; + } +- *dest = (nri_posix_rlimit *)util_common_calloc_s(sizeof(nri_posix_rlimit)); ++ dest = (nri_key_value *)util_common_calloc_s(sizeof(nri_key_value)); + if (dest == NULL) { + ERROR("Out of memory"); + return false; + } +- (*dest)->hard = src->hard; +- (*dest)->soft = src->soft; +- (*dest)->type = util_strdup_s(src->type); +- return true; ++ dest->key = util_strdup_s(src->key); ++ dest->value = util_strdup_s(src->value); ++ return dest; + } + +-bool copy_nri_linux_resources(const nri_linux_resources *src, nri_linux_resources **dest) ++nri_posix_rlimit *copy_nri_posix_rlimit(const nri_posix_rlimit *src) + { +- if (src == NULL || dest == NULL) { ++ nri_posix_rlimit *dest = NULL; ++ if (src == NULL) { + ERROR("Invalid input arguments"); + return false; + } +- +- *dest = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); +- if (*dest == NULL) { ++ dest = (nri_posix_rlimit *)util_common_calloc_s(sizeof(nri_posix_rlimit)); ++ if (dest == NULL) { + ERROR("Out of memory"); + return false; + } ++ dest->hard = src->hard; ++ dest->soft = src->soft; ++ dest->type = util_strdup_s(src->type); ++ return dest; ++} + +- if (!init_nri_linux_resources(dest)) { +- ERROR("Failed to init dest nri linux resources"); +- goto free_out; ++nri_linux_resources *copy_nri_linux_resources(const nri_linux_resources *src) ++{ ++ nri_linux_resources *dest = NULL; ++ if (src == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; + } + +- if (!copy_nri_linux_cpu(src->cpu, &(*dest)->cpu)) { +- ERROR("Failed to copy nri_linux_cpu"); +- goto free_out; ++ dest = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); ++ if (dest == NULL) { ++ ERROR("Out of memory"); ++ return false; + } + +- if (!copy_nri_linux_memory(src->memory, &(*dest)->memory)) { +- ERROR("Failed to copy nri_linux_memory"); +- goto free_out; ++ if (src->cpu != NULL) { ++ dest->cpu = copy_nri_linux_cpu(src->cpu); ++ if (dest->cpu == NULL) { ++ ERROR("Failed to copy nri_linux_cpu"); ++ goto free_out; ++ } + } + +- (*dest)->blockio_class = util_strdup_s(src->blockio_class); +- (*dest)->rdt_class = util_strdup_s(src->rdt_class); ++ if (src->memory != NULL) { ++ dest->memory = copy_nri_linux_memory(src->memory); ++ if (dest->memory == NULL) { ++ ERROR("Failed to copy nri_linux_memory"); ++ goto free_out; ++ } ++ } ++ ++ dest->blockio_class = util_strdup_s(src->blockio_class); ++ dest->rdt_class = util_strdup_s(src->rdt_class); + + if (src->hugepage_limits_len > 0) { +- (*dest)->hugepage_limits = (nri_hugepage_limit**)util_smart_calloc_s(sizeof(nri_hugepage_limit*), ++ dest->hugepage_limits = (nri_hugepage_limit**)util_smart_calloc_s(sizeof(nri_hugepage_limit*), + src->hugepage_limits_len); +- for (size_t i = 0; i < src->hugepage_limits_len; ++i) { +- if (!copy_nri_hugepage_limit(src->hugepage_limits[i], &((*dest)->hugepage_limits[i]))) { ++ for (size_t i = 0; i < src->hugepage_limits_len; i++) { ++ dest->hugepage_limits[i] = copy_nri_hugepage_limit(src->hugepage_limits[i]); ++ if (dest->hugepage_limits[i] == NULL) { + ERROR("Failed to copy nri_hugepage_limit"); + goto free_out; + } +@@ -349,25 +425,26 @@ bool copy_nri_linux_resources(const nri_linux_resources *src, nri_linux_resource + } + + if (src->devices_len > 0) { +- (*dest)->devices = (nri_linux_device_cgroup**)util_smart_calloc_s(sizeof(nri_linux_device_cgroup*), src->devices_len); +- for (size_t i = 0; i < src->devices_len; ++i) { +- if (!copy_nri_linux_device_cgroup(src->devices[i], &((*dest)->devices[i]))) { ++ dest->devices = (nri_linux_device_cgroup**)util_smart_calloc_s(sizeof(nri_linux_device_cgroup*), src->devices_len); ++ for (size_t i = 0; i < src->devices_len; i++) { ++ dest->devices[i] = copy_nri_linux_device_cgroup(src->devices[i]); ++ if (dest->devices[i] == NULL) { + ERROR("Failed to copy nri_linux_device_cgroup"); + goto free_out; + } + } + } + +- if (dup_json_map_string_string(src->unified, (*dest)->unified)) { ++ if (dup_json_map_string_string(src->unified, dest->unified)) { + ERROR("Failed to copy json_map_string_string"); + goto free_out; + } + +- return true; ++ return dest; + + free_out: +- free_nri_linux_resources(*dest); +- return false; ++ free_nri_linux_resources(dest); ++ return NULL; + } + + bool merge_nri_hooks(nri_hook **targetHooks, size_t targetSize, const nri_hook **sourceHooks, +@@ -381,140 +458,67 @@ bool merge_nri_hooks(nri_hook **targetHooks, size_t targetSize, const nri_hook * + return false; + } + +- if (util_mem_realloc((void**)&targetHooks, newSize, (void**)&targetHooks, oldSize) != 0) { ++ if (util_mem_realloc((void**)&targetHooks, newSize, targetHooks, oldSize) != 0) { + ERROR("Failed to realloc and assign hook array"); + return false; + } + + for (size_t i = 0; i < sourceLen; i++) { +- if (!copy_nri_hook(sourceHooks[i], &targetHooks[targetSize++])) { ++ targetHooks[targetSize] = copy_nri_hook(sourceHooks[i]); ++ if (targetHooks[targetSize] == NULL) { + ERROR("Failed to copy hook"); + return false; + } ++ targetSize++; + } + + return true; + } + +-bool init_nri_container_adjust(nri_container_adjustment **adjust) ++nri_container_update *init_nri_container_update(const char *id, const uint8_t ignore_failure) + { +- if (adjust == NULL) { ++ nri_container_update *update = NULL; ++ if (id == NULL) { + ERROR("Invalid input arguments"); + return false; + } + +- *adjust = (nri_container_adjustment *)util_common_calloc_s(sizeof(nri_container_adjustment)); +- if (*adjust == NULL) { ++ update = (nri_container_update *)util_common_calloc_s(sizeof(nri_container_update)); ++ if (update == NULL) { + ERROR("Out of memory"); + return false; + } + +- (*adjust)->annotations = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); +- if ((*adjust)->annotations == NULL) { +- goto free_out; +- } +- +- (*adjust)->env = (nri_key_value **)util_common_calloc_s(sizeof(nri_key_value *)); +- if ((*adjust)->env == NULL) { +- goto free_out; +- } +- (*adjust)->env_len = 0; +- +- (*adjust)->hooks = (nri_hooks *)util_common_calloc_s(sizeof(nri_hooks)); +- if ((*adjust)->hooks == NULL) { +- goto free_out; +- } +- +- (*adjust)->linux = (nri_linux_container_adjustment *)util_common_calloc_s(sizeof(nri_linux_container_adjustment)); +- if ((*adjust)->linux == NULL) { +- goto free_out; +- } +- +- (*adjust)->linux->resources = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); +- if ((*adjust)->linux->resources == NULL) { +- ERROR("Out of memory"); +- return false; +- } +- +- (*adjust)->mounts = (nri_mount **)util_common_calloc_s(sizeof(nri_mount *)); +- if ((*adjust)->mounts == NULL) { +- goto free_out; +- } +- (*adjust)->mounts_len = 0; +- +- (*adjust)->rlimits = (nri_posix_rlimit **)util_common_calloc_s(sizeof(nri_posix_rlimit *)); +- if ((*adjust)->rlimits == NULL) { +- goto free_out; +- } +- (*adjust)->rlimits_len = 0; ++ update->container_id = util_strdup_s(id); + +- return true; +- +-free_out: +- ERROR("Out of memory"); +- free_nri_container_adjustment(*adjust); +- return false; ++ update->ignore_failure = ignore_failure; ++ return update; + } + +-bool init_nri_container_update(nri_container_update **update, const char *id, uint8_t ignore_failure) ++nri_linux_resources *init_nri_linux_resources() + { +- if (update == NULL || id == NULL) { +- ERROR("Invalid input arguments"); +- return false; +- } +- +- *update = (nri_container_update *)util_common_calloc_s(sizeof(nri_container_update)); +- if (*update == NULL) { +- ERROR("Out of memory"); +- return false; +- } +- +- (*update)->container_id = util_strdup_s(id); +- (*update)->linux = (nri_linux_container_update *)util_common_calloc_s(sizeof(nri_linux_container_update)); +- if ((*update)->linux == NULL) { +- goto free_out; +- } ++ nri_linux_resources *resources = NULL; + +- (*update)->ignore_failure = ignore_failure; +- return true; +- +-free_out: +- ERROR("Out of memory"); +- free_nri_container_update(*update); +- return false; +-} +- +-bool init_nri_linux_resources(nri_linux_resources **resources) +-{ ++ resources = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); + if (resources == NULL) { +- ERROR("Invalid input arguments"); +- return false; +- } +- +- *resources = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); +- if (*resources == NULL) { + ERROR("Out of memory"); + return false; + } + +- (*resources)->cpu = (nri_linux_cpu *)util_common_calloc_s(sizeof(nri_linux_cpu)); +- if ((*resources)->cpu == NULL) { ++ resources->cpu = (nri_linux_cpu *)util_common_calloc_s(sizeof(nri_linux_cpu)); ++ if (resources->cpu == NULL) { + goto free_out; + } + +- (*resources)->memory = (nri_linux_memory *)util_common_calloc_s(sizeof(nri_linux_memory)); +- if ((*resources)->memory == NULL) { ++ resources->memory = (nri_linux_memory *)util_common_calloc_s(sizeof(nri_linux_memory)); ++ if (resources->memory == NULL) { + goto free_out; + } + +- (*resources)->unified = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); +- if ((*resources)->unified == NULL) { +- goto free_out; +- } +- return true; ++ return resources; + + free_out: + ERROR("Out of memory"); +- free_nri_linux_resources(*resources); +- return false; ++ free_nri_linux_resources(resources); ++ return NULL; + } +\ No newline at end of file +diff --git a/src/daemon/common/nri/nri_utils.h b/src/daemon/common/nri/nri_utils.h +index 7bf54a71..171055df 100644 +--- a/src/daemon/common/nri/nri_utils.h ++++ b/src/daemon/common/nri/nri_utils.h +@@ -49,19 +49,19 @@ typedef enum { + LAST = 12, + } NRI_Event; + +-bool copy_nri_mount(const nri_mount *src, nri_mount **dest); +-bool copy_nri_key_value(const nri_key_value *src, nri_key_value **dest); +-bool copy_nri_posix_rlimit(const nri_posix_rlimit *src, nri_posix_rlimit **dest); +-bool copy_nri_linux_resources(const nri_linux_resources *src, nri_linux_resources **dest); ++nri_mount *copy_nri_mount(const nri_mount *src); ++nri_posix_rlimit *copy_nri_posix_rlimit(const nri_posix_rlimit *src); ++nri_linux_resources *copy_nri_linux_resources(const nri_linux_resources *src); ++nri_key_value *copy_nri_key_value(const nri_key_value *src); ++nri_linux_device *copy_nri_device(const nri_linux_device *src); + + bool is_marked_for_removal(const char* key, char **out); + + bool merge_nri_hooks(nri_hook **targetHooks, size_t targetSize, const nri_hook **sourceHooks, + size_t sourceLen); + +-bool init_nri_container_adjust(nri_container_adjustment **adjust); +-bool init_nri_container_update(nri_container_update **update, const char *id, uint8_t ignore_failure); +-bool init_nri_linux_resources(nri_linux_resources **resources); ++nri_container_update *init_nri_container_update(const char *id, const uint8_t ignore_failure); ++nri_linux_resources *init_nri_linux_resources(void); + + #ifdef __cplusplus + } +diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c +index 9ba1c8a0..020c9f32 100644 +--- a/src/daemon/config/isulad_config.c ++++ b/src/daemon/config/isulad_config.c +@@ -1938,7 +1938,7 @@ int merge_json_confs_into_global(struct service_arguments *args) + override_string_value(&args->json_confs->plugin_path, &tmp_json_confs->plugin_path); + args->json_confs->plugin_registration_timeout = tmp_json_confs->plugin_registration_timeout; + args->json_confs->plugin_requst_timeout = tmp_json_confs->plugin_requst_timeout; +- override_string_value(&args->json_confs->nri_socket_path, &tmp_json_confs->nri_socket_path); ++ // Setting socket plugin path is not supported now + #endif + args->json_confs->enable_pod_events = tmp_json_confs->enable_pod_events; + #endif +diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +index 1cee68ec..d3fdd76a 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +@@ -27,6 +27,11 @@ + #include "stream_server.h" + #include "sandbox_manager.h" + ++#ifdef ENABLE_NRI ++#include "nri_adaption.h" ++#include "nri_helpers.h" ++#endif ++ + namespace CRIV1 { + auto ContainerManagerService::GetContainerOrSandboxRuntime(const std::string &realID, Errors &error) -> std::string + { +@@ -505,11 +510,24 @@ std::string ContainerManagerService::CreateContainer(const std::string &podSandb + return response_id; + } + ++#ifdef ENABLE_NRI ++ Errors nriErr; ++ nri_container_adjustment *adjust = NULL; ++ if (!NRIAdaptation::GetInstance()->CreateContainer(sandbox, response_id, containerConfig, &adjust, nriErr)) { ++ ERROR("Failed to get NRI adjustment for container: %s", nriErr.GetCMessage()); ++ NRIAdaptation::GetInstance()->UndoCreateContainer(sandbox, response_id, nriErr); ++ return response_id; ++ } ++#endif ++ + request = GenerateCreateContainerRequest(*sandbox, containerConfig, podSandboxConfig, error); + if (error.NotEmpty()) { + error.SetError("Failed to generate create container request"); + goto cleanup; + } ++#ifdef ENABLE_NRI ++ request->adjust = adjust; ++#endif + + if (m_cb->container.create(request, &response) != 0) { + if (response != nullptr && (response->errmsg != nullptr)) { +@@ -522,6 +540,12 @@ std::string ContainerManagerService::CreateContainer(const std::string &podSandb + + response_id = response->id; + ++#ifdef ENABLE_NRI ++ if (!NRIAdaptation::GetInstance()->PostCreateContainer(response_id, nriErr)) { ++ ERROR("NRI post-create notification failed: %s", nriErr.GetCMessage()); ++ } ++#endif ++ + cleanup: + free_container_create_request(request); + free_container_create_response(response); +@@ -530,6 +554,9 @@ cleanup: + + void ContainerManagerService::StartContainer(const std::string &containerID, Errors &error) + { ++#ifdef ENABLE_NRI ++ Errors nriErr; ++#endif + if (containerID.empty()) { + error.SetError("Invalid empty container id."); + return; +@@ -563,6 +590,14 @@ void ContainerManagerService::StartContainer(const std::string &containerID, Err + goto cleanup; + } + ++#ifdef ENABLE_NRI ++ if (!NRIAdaptation::GetInstance()->StartContainer(containerID, nriErr)) { ++ ERROR("NRI container start failed: %s", nriErr.GetCMessage()); ++ NRIAdaptation::GetInstance()->StopContainer(containerID, nriErr); ++ goto cleanup; ++ } ++#endif ++ + if (ret != 0) { + if (response != nullptr && response->errmsg != nullptr) { + error.SetError(response->errmsg); +@@ -571,6 +606,11 @@ void ContainerManagerService::StartContainer(const std::string &containerID, Err + } + goto cleanup; + } ++#ifdef ENABLE_NRI ++ if (!NRIAdaptation::GetInstance()->PostStartContainer(containerID, nriErr)) { ++ ERROR("NRI PostStartContainer notification failed: %s", nriErr.GetCMessage()); ++ } ++#endif + cleanup: + free_container_start_request(request); + free_container_start_response(response); +@@ -578,11 +618,25 @@ cleanup: + + void ContainerManagerService::StopContainer(const std::string &containerID, int64_t timeout, Errors &error) + { ++#ifdef ENABLE_NRI ++ Errors nriErr; ++#endif + CRIHelpers::StopContainer(m_cb, containerID, timeout, error); ++#ifdef ENABLE_NRI ++ if (!NRIAdaptation::GetInstance()->StopContainer(containerID, nriErr)) { ++ ERROR("NRI StopContainer notification failed: %s", nriErr.GetCMessage()); ++ } ++#endif + } + + void ContainerManagerService::RemoveContainer(const std::string &containerID, Errors &error) + { ++#ifdef ENABLE_NRI ++ Errors nriErr; ++ if (!NRIAdaptation::GetInstance()->RemoveContainer(containerID, nriErr)) { ++ ERROR("NRI RemoveContainer notification failed: %s", nriErr.GetCMessage()); ++ } ++#endif + CRIHelpers::RemoveContainer(m_cb, containerID, error); + if (error.NotEmpty()) { + WARN("Failed to remove container %s", containerID.c_str()); +@@ -1048,6 +1102,18 @@ void ContainerManagerService::UpdateContainerResources(const std::string &contai + struct parser_context ctx { + OPT_GEN_SIMPLIFY, 0 + }; ++ ++ runtime::v1::LinuxContainerResources updateRes = resources; ++#ifdef ENABLE_NRI ++ Errors nriErr; ++ runtime::v1::LinuxContainerResources adjust; ++ if (!NRIAdaptation::GetInstance()->UpdateContainer(containerID, resources, adjust, nriErr)) { ++ ERROR("NRI UpdateContainer notification failed: %s", nriErr.GetCMessage()); ++ goto cleanup; ++ } ++ updateRes = adjust; ++#endif ++ + request = (container_update_request *)util_common_calloc_s(sizeof(container_update_request)); + if (request == nullptr) { + error.SetError("Out of memory"); +@@ -1061,17 +1127,17 @@ void ContainerManagerService::UpdateContainerResources(const std::string &contai + goto cleanup; + } + +- hostconfig->cpu_period = resources.cpu_period(); +- hostconfig->cpu_quota = resources.cpu_quota(); +- hostconfig->cpu_shares = resources.cpu_shares(); ++ hostconfig->cpu_period = updateRes.cpu_period(); ++ hostconfig->cpu_quota = updateRes.cpu_quota(); ++ hostconfig->cpu_shares = updateRes.cpu_shares(); + +- if (!resources.unified().empty()) { ++ if (!updateRes.unified().empty()) { + hostconfig->unified = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); + if (hostconfig->unified == nullptr) { + error.SetError("Out of memory"); + goto cleanup; + } +- for (auto &iter : resources.unified()) { ++ for (auto &iter : updateRes.unified()) { + if (append_json_map_string_string(hostconfig->unified, iter.first.c_str(), iter.second.c_str()) != 0) { + error.SetError("Failed to append string"); + goto cleanup; +@@ -1079,30 +1145,30 @@ void ContainerManagerService::UpdateContainerResources(const std::string &contai + } + } + +- hostconfig->memory = resources.memory_limit_in_bytes(); +- hostconfig->memory_swap = resources.memory_swap_limit_in_bytes(); +- if (!resources.cpuset_cpus().empty()) { +- hostconfig->cpuset_cpus = util_strdup_s(resources.cpuset_cpus().c_str()); ++ hostconfig->memory = updateRes.memory_limit_in_bytes(); ++ hostconfig->memory_swap = updateRes.memory_swap_limit_in_bytes(); ++ if (!updateRes.cpuset_cpus().empty()) { ++ hostconfig->cpuset_cpus = util_strdup_s(updateRes.cpuset_cpus().c_str()); + } +- if (!resources.cpuset_mems().empty()) { +- hostconfig->cpuset_mems = util_strdup_s(resources.cpuset_mems().c_str()); ++ if (!updateRes.cpuset_mems().empty()) { ++ hostconfig->cpuset_mems = util_strdup_s(updateRes.cpuset_mems().c_str()); + } +- if (resources.hugepage_limits_size() != 0) { ++ if (updateRes.hugepage_limits_size() != 0) { + hostconfig->hugetlbs = (host_config_hugetlbs_element **)util_smart_calloc_s( +- sizeof(host_config_hugetlbs_element *), resources.hugepage_limits_size()); ++ sizeof(host_config_hugetlbs_element *), updateRes.hugepage_limits_size()); + if (hostconfig->hugetlbs == nullptr) { + error.SetError("Out of memory"); + goto cleanup; + } +- for (int i = 0; i < resources.hugepage_limits_size(); i++) { ++ for (int i = 0; i < updateRes.hugepage_limits_size(); i++) { + hostconfig->hugetlbs[i] = + (host_config_hugetlbs_element *)util_common_calloc_s(sizeof(host_config_hugetlbs_element)); + if (hostconfig->hugetlbs[i] == nullptr) { + error.SetError("Out of memory"); + goto cleanup; + } +- hostconfig->hugetlbs[i]->page_size = util_strdup_s(resources.hugepage_limits(i).page_size().c_str()); +- hostconfig->hugetlbs[i]->limit = resources.hugepage_limits(i).limit(); ++ hostconfig->hugetlbs[i]->page_size = util_strdup_s(updateRes.hugepage_limits(i).page_size().c_str()); ++ hostconfig->hugetlbs[i]->limit = updateRes.hugepage_limits(i).limit(); + hostconfig->hugetlbs_len++; + } + } +@@ -1121,6 +1187,12 @@ void ContainerManagerService::UpdateContainerResources(const std::string &contai + error.SetError("Failed to call update container callback"); + } + } ++#ifdef ENABLE_NRI ++ if (!NRIAdaptation::GetInstance()->PostUpdateContainer(containerID, nriErr)) { ++ ERROR("NRI PostUpdateContainer notification failed: %s", nriErr.GetCMessage()); ++ goto cleanup; ++ } ++#endif + cleanup: + free_container_update_request(request); + free_container_update_response(response); +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +index 77faf48a..0140eb99 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +@@ -37,6 +37,9 @@ + #include "transform.h" + #include "isulad_config.h" + #include "mailbox.h" ++#ifdef ENABLE_NRI ++#include "nri_adaption.h" ++#endif + + namespace CRIV1 { + void PodSandboxManagerService::PrepareSandboxData(const runtime::v1::PodSandboxConfig &config, +@@ -304,6 +307,9 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1::PodSandboxConfig + std::string network_setting_json; + runtime::v1::PodSandboxConfig copyConfig = config; + cri_container_message_t msg = { 0 }; ++#ifdef ENABLE_NRI ++ Errors nriErr; ++#endif + + // Step 1: Parepare sandbox name, runtime and networkMode + PrepareSandboxData(config, runtimeHandler, sandboxName, runtimeInfo, networkMode, error); +@@ -401,6 +407,14 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1::PodSandboxConfig + msg.type = CRI_CONTAINER_MESSAGE_TYPE_STARTED; + mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &msg); + ++#ifdef ENABLE_NRI ++ if (!NRIAdaptation::GetInstance()->RunPodSandbox(sandbox, nriErr)) { ++ ERROR("NRI RunPodSandbox failed: %s", nriErr.GetCMessage()); ++ error.Errorf("NRI RunPodSandbox failed: %s", nriErr.GetCMessage()); ++ return response_id; ++ } ++#endif ++ + return sandbox->GetId(); + + cleanup_network: +@@ -418,6 +432,11 @@ cleanup_sandbox: + if (error.NotEmpty()) { + ERROR("Failed to delete sandbox: %s", sandbox->GetId().c_str()); + } ++#ifdef ENABLE_NRI ++ if (!NRIAdaptation::GetInstance()->RemovePodSandbox(sandbox, nriErr)) { ++ DEBUG("NRI RemovePodSandbox failed: %s", nriErr.GetCMessage()); ++ } ++#endif + + return response_id; + } +@@ -618,7 +637,15 @@ void PodSandboxManagerService::StopPodSandbox(const std::string &podSandboxID, E + return; + } + +- sandbox->Stop(sandbox::DEFAULT_STOP_TIMEOUT, error); ++ if (!sandbox->Stop(sandbox::DEFAULT_STOP_TIMEOUT, error)) { ++ return; ++ } ++#ifdef ENABLE_NRI ++ Errors nriStopPodErr; ++ if (!NRIAdaptation::GetInstance()->StopPodSandbox(sandbox, nriStopPodErr)) { ++ ERROR("NRI sandbox stop notification failed %s", nriStopPodErr.GetCMessage()); ++ } ++#endif + } + + void PodSandboxManagerService::RemoveAllContainersInSandbox(const std::string &readSandboxID, +@@ -656,6 +683,9 @@ void PodSandboxManagerService::RemovePodSandbox(const std::string &podSandboxID, + { + std::vector<std::string> errors; + std::string realSandboxID; ++#ifdef ENABLE_NRI ++ Errors nriErr; ++#endif + + if (m_cb == nullptr || m_cb->container.remove == nullptr) { + ERROR("Unimplemented callback"); +@@ -724,6 +754,11 @@ void PodSandboxManagerService::RemovePodSandbox(const std::string &podSandboxID, + msg.type = CRI_CONTAINER_MESSAGE_TYPE_DELETED; + mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &msg); + } ++#ifdef ENABLE_NRI ++ if (!NRIAdaptation::GetInstance()->RemovePodSandbox(sandbox, nriErr)) { ++ ERROR("NRI RemovePodSandbox failed: %s", nriErr.GetCMessage()); ++ } ++#endif + } + + auto PodSandboxManagerService::SharesHostNetwork(const container_inspect *inspect) -> runtime::v1::NamespaceMode +diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c +index 2b6b2a15..dcbdd1d3 100644 +--- a/src/daemon/executor/container_cb/execution_create.c ++++ b/src/daemon/executor/container_cb/execution_create.c +@@ -65,6 +65,10 @@ + #include "mailbox.h" + #include "specs_mount.h" + ++#ifdef ENABLE_NRI ++#include "nri_spec.h" ++#endif ++ + #ifdef ENABLE_CRI_API_V1 + static bool validate_sandbox_info(const container_sandbox_info *sandbox) + { +@@ -1465,6 +1469,14 @@ int container_create_cb(const container_create_request *request, container_creat + skip_sandbox_key_manage = (is_sandbox_container(request->sandbox) && namespace_is_cni(host_spec->network_mode)); + #endif + ++#ifdef ENABLE_NRI ++ if (request->adjust != NULL && nri_adjust_host_spec(request->adjust, host_spec) != 0) { ++ ERROR("Failed to adjust host spec"); ++ cc = ISULAD_ERR_INPUT; ++ goto clean_container_root_dir; ++ } ++#endif ++ + if (save_container_config_before_create(id, runtime_root, host_spec, v2_spec) != 0) { + ERROR("Failed to malloc container_config_v2_common_config"); + cc = ISULAD_ERR_INPUT; +@@ -1553,6 +1565,15 @@ int container_create_cb(const container_create_request *request, container_creat + } + #endif + ++#ifdef ENABLE_NRI ++ // modify oci spec by nri plugin ++ if (request->adjust != NULL && nri_adjust_oci_spec(request->adjust, oci_spec) != 0) { ++ ERROR("Failed to adjust oci spec"); ++ cc = ISULAD_ERR_EXEC; ++ goto clean_netns; ++ } ++#endif ++ + host_channel = dup_host_channel(host_spec->host_channel); + if (prepare_host_channel(host_channel, host_spec->user_remap)) { + ERROR("Failed to prepare host channel"); +diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c +index 0b95cdad..4157c631 100644 +--- a/src/daemon/modules/service/service_container.c ++++ b/src/daemon/modules/service/service_container.c +@@ -76,6 +76,9 @@ + #include "sandbox_ops.h" + #include "vsock_io_handler.h" + #endif ++#ifdef ENABLE_NRI ++#include "nri_spec.h" ++#endif + + #define KATA_RUNTIME "kata-runtime" + +@@ -726,6 +729,15 @@ static int do_oci_spec_update(const char *id, oci_runtime_spec *oci_spec, contai + return -1; + } + ++#ifdef ENABLE_NRI ++ // update oci spec with nri ++ ret = update_oci_nri(oci_spec, hostconfig); ++ if (ret != 0) { ++ ERROR("Failed to update oci spec with nri"); ++ return -1; ++ } ++#endif ++ + // renew_oci_config() will update process->user and share namespace after. + + return 0; +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index 002431d8..36e89343 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -2883,7 +2883,12 @@ int spec_add_linux_resources_rlimit(oci_runtime_spec *oci_spec, const char *type + return -1; + } + +- ret = make_sure_oci_spec_linux_resources(oci_spec); ++ ret = make_sure_oci_spec_process(oci_spec); ++ if (ret < 0) { ++ return -1; ++ } ++ ++ ret = merge_ulimits_pre(oci_spec, 1); + if (ret < 0) { + return -1; + } +diff --git a/src/daemon/nri/nri_adaption.cc b/src/daemon/nri/nri_adaption.cc +new file mode 100644 +index 00000000..3190767f +--- /dev/null ++++ b/src/daemon/nri/nri_adaption.cc +@@ -0,0 +1,1165 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-03-15 ++ * Description: provide plugin manager(NRI adaption) class definition ++ *********************************************************************************/ ++ ++#include "nri_adaption.h" ++ ++#include <sys/stat.h> ++ ++#include <isula_libutils/log.h> ++#include <isula_libutils/auto_cleanup.h> ++ ++#include "isulad_config.h" ++#include "utils_file.h" ++#include "utils_string.h" ++#include "utils.h" ++#include "nri_convert.h" ++#include "nri_plugin.h" ++#include "nri_result.h" ++#include "sandbox_manager.h" ++ ++std::atomic<NRIAdaptation *> NRIAdaptation::m_instance; ++ ++NRIAdaptation *NRIAdaptation::GetInstance() noexcept ++{ ++ static std::once_flag flag; ++ ++ std::call_once(flag, [] { m_instance = new NRIAdaptation; }); ++ ++ return m_instance; ++} ++ ++NRIAdaptation::~NRIAdaptation() ++{ ++ for (const auto &pair : m_storeMap) { ++ auto plugin = pair.second; ++ plugin->shutdown(); ++ } ++} ++ ++auto NRIAdaptation::Init(Errors &error) -> bool ++{ ++ std::map<std::string, std::shared_ptr<NRIPlugin>> tmp_storeMap; ++ ++ m_support = conf_get_nri_support(); ++ if (!m_support) { ++ return true; ++ } ++ ++ m_external_support = conf_get_nri_external_support(); ++ service_executor_t *cb = get_service_executor(); ++ if (cb == nullptr) { ++ ERROR("Init isulad service executor failure."); ++ return false; ++ } ++ ++ m_containerManager = std::make_unique<CRIV1::ContainerManagerService>(cb); ++ ++ m_pluginConfigPath = GetNRIPluginConfigPath(); ++ m_pluginPath = GetNRIPluginPath(); ++ m_plugin_registration_timeout = conf_get_nri_plugin_registration_timeout(); ++ m_plugin_requst_timeout = conf_get_nri_plugin_requst_timeout(); ++ m_sock_path = GetNRISockPath(); ++ ++ if (!StartPlugin()) { ++ ERROR("Failed to do StartPlugin"); ++ return false; ++ } ++ ++ if (!SortPlugins()) { ++ ERROR("Failed to do SortPlugins"); ++ return false; ++ } ++ ++ return true; ++} ++ ++void NRIAdaptation::RemoveClosedPlugins() ++{ ++ std::vector<std::string> closedPlugin; ++ ++ GetClosedPlugins(closedPlugin); ++ ++ for (const auto &key : closedPlugin) { ++ RemovePluginByIndex(key); ++ } ++} ++ ++void NRIAdaptation::GetClosedPlugins(std::vector<std::string> &closedPlugin) ++{ ++ ReadGuard<RWMutex> lock(m_mutex); ++ for (const auto &pair : m_storeMap) { ++ auto plugin = pair.second; ++ if (plugin->IsClose()) { ++ closedPlugin.push_back(pair.first); ++ } ++ } ++} ++ ++auto NRIAdaptation::IsSupport() -> bool ++{ ++ return m_support; ++} ++ ++auto NRIAdaptation::GetPluginByIndex(const std::string &index) -> std::shared_ptr<NRIPlugin> ++{ ++ ReadGuard<RWMutex> lock(m_mutex); ++ return m_storeMap[index]; ++} ++ ++void NRIAdaptation::RemovePluginByIndex(const std::string &index) ++{ ++ WriteGuard<RWMutex> lock(m_mutex); ++ m_storeMap.erase(index); ++} ++ ++void NRIAdaptation::AddPluginByIndex(const std::string &index, std::shared_ptr<NRIPlugin> plugin) ++{ ++ WriteGuard<RWMutex> lock(m_mutex); ++ m_storeMap[index] = plugin; ++} ++ ++auto NRIAdaptation::ApplyUpdates(const std::vector<nri_container_update *> &update, ++ std::vector<nri_container_update *> &failed, bool getFailed, Errors &error) -> bool ++{ ++ for (auto &u : update) { ++ runtime::v1::LinuxContainerResources resources; ++ if (!LinuxResourcesFromNRI(u->linux->resources, resources)) { ++ ERROR("Failed to convert Linux resources from NRI"); ++ error.Errorf("Failed to convert Linux resources from NRI"); ++ return false; ++ } ++ ++ m_containerManager->UpdateContainerResources(u->container_id, resources, error); ++ ++ if (error.NotEmpty()) { ++ ERROR("Failed to update container: %s resources: %s", u->container_id, error.GetCMessage()); ++ if (!u->ignore_failure && getFailed) { ++ failed.push_back(u); ++ } ++ continue; ++ } ++ ++ TRACE("NRI update of container %s successful", u->container_id); ++ } ++ ++ if (failed.size() != 0) { ++ error.Errorf("NRI update of some containers failed"); ++ return false; ++ } ++ return true; ++} ++ ++auto NRIAdaptation::NewExternalPlugin(int fd) -> bool ++{ ++ if (!m_external_support) { ++ ERROR("External plugin support is disabled"); ++ return false; ++ } ++ if (fd < 0) { ++ ERROR("Invalid fd"); ++ return false; ++ } ++ ++ std::string plugin_name; ++ NRIHelpers::GenerateRandomExternalName(plugin_name); ++ if (plugin_name.empty()) { ++ ERROR("Failed to generate random external name"); ++ return false; ++ } ++ ++ auto plugin = std::make_shared<NRIPlugin>(fd, plugin_name); ++ ++ AddPluginByIndex(plugin_name, plugin); ++ ++ if (!plugin->Start(m_plugin_registration_timeout, m_plugin_requst_timeout)) { ++ ERROR("Failed to start plugin ready for conn fd %d", fd); ++ RemovePluginByIndex(plugin_name); ++ return false; ++ } ++ ++ std::vector<nri_pod_sandbox *> pods; ++ std::vector<nri_container *> cons; ++ nri_container_update **updateRes; ++ size_t update_len = 0; ++ ++ std::vector<std::shared_ptr<sandbox::Sandbox>> sandboxes; ++ runtime::v1::PodSandboxFilter podFilter; ++ std::vector<std::unique_ptr<runtime::v1::Container>> containers; ++ Errors tmpError; ++ ++ std::vector<nri_container_update *> updates; ++ std::vector<nri_container_update *> failed; ++ ++ sandbox::SandboxManager::GetInstance()->ListAllSandboxes(podFilter, sandboxes); ++ ++ if (!PodSandboxesToNRI(sandboxes, pods)) { ++ ERROR("Failed to convert podsandbox to nri"); ++ NRIHelpers::FreeNriPodVector(pods); ++ return false; ++ } ++ ++ m_containerManager->ListContainers(nullptr, containers, tmpError); ++ ++ if (!ContainersToNRI(containers, cons)) { ++ ERROR("Failed to convert container to nri"); ++ NRIHelpers::FreeNriPodVector(pods); ++ NRIHelpers::FreeNriContainerVector(cons); ++ return false; ++ } ++ ++ // pods and cons's memory transfer to nri_synchronize_request, ++ // and is automatically freed when nri_synchronize_request is freed ++ if (!plugin->Synchronize(pods, cons, &updateRes, update_len, tmpError)) { ++ ERROR("Failed to synchronize plugin"); ++ return false; ++ } ++ ++ for (size_t i = 0; i < update_len; i++) { ++ updates.push_back(updateRes[i]); ++ } ++ ++ if (!ApplyUpdates(updates, failed, false, tmpError)) { ++ ERROR("Failed to update post-sync"); ++ } ++ ++ NRIHelpers::FreeNriContainerUpdateVector(updates); ++ NRIHelpers::FreeNriContainerUpdateVector(failed); ++ INFO("plugin %s connected", plugin_name.c_str()); ++ return true; ++} ++ ++auto NRIAdaptation::RunPodSandbox(std::shared_ptr<const sandbox::Sandbox> sandbox, Errors &error) -> bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto runPodEvent = makeUniquePtrCStructWrapper<nri_state_change_event>(free_nri_state_change_event); ++ if (runPodEvent == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ runPodEvent->get()->pod = pod->move(); ++ runPodEvent->get()->event = RUN_POD_SANDBOX; ++ return StateChange(runPodEvent->get(), error); ++} ++ ++auto NRIAdaptation::StopPodSandbox(std::shared_ptr<const sandbox::Sandbox> sandbox, Errors &error) -> bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto stopPodEvent = makeUniquePtrCStructWrapper<nri_state_change_event>(free_nri_state_change_event); ++ if (stopPodEvent == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ stopPodEvent->get()->pod = pod->move(); ++ stopPodEvent->get()->event = STOP_POD_SANDBOX; ++ return StateChange(stopPodEvent->get(), error); ++} ++ ++auto NRIAdaptation::RemovePodSandbox(std::shared_ptr<const sandbox::Sandbox> sandbox, Errors &error) -> bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto removePodEvent = makeUniquePtrCStructWrapper<nri_state_change_event>(free_nri_state_change_event); ++ if (removePodEvent == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ removePodEvent->get()->pod = pod->move(); ++ removePodEvent->get()->event = REMOVE_POD_SANDBOX; ++ return StateChange(removePodEvent->get(), error); ++} ++ ++auto NRIAdaptation::CreateContainer(std::shared_ptr<const sandbox::Sandbox> sandbox, const std::string &conId, ++ const runtime::v1::ContainerConfig &containerConfig, nri_container_adjustment **adjust, Errors &error) -> bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto con = NRIContainerByConConfig(sandbox, containerConfig, error); ++ if (con == nullptr) { ++ ERROR("Failed to covert container to nri: %s", conId.c_str()); ++ return false; ++ } ++ ++ auto req = makeUniquePtrCStructWrapper<nri_create_container_request>(free_nri_create_container_request); ++ if (req == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ req->get()->container = con->move(); ++ req->get()->pod = pod->move(); ++ ++ pluginResult result; ++ result.InitByConId(conId); ++ ++ if (!PluginsCreateContainer(req->get(), conId, result)) { ++ ERROR("Failed to call create container to all plugins"); ++ return false; ++ } ++ ++ RemoveClosedPlugins(); ++ ++ // TODO:evict container do not aply ++ ++ // TODO:how can i rollback on failure ++ std::vector<nri_container_update *> failed; ++ if (!ApplyUpdates(result.GetReplyUpdate(), failed, false, error)) { ++ ERROR("Failed to apply updates"); ++ NRIHelpers::FreeNriContainerUpdateVector(failed); ++ return false; ++ } ++ ++ *adjust = result.MoveReplyAdjust(); ++ NRIHelpers::FreeNriContainerUpdateVector(failed); ++ return true; ++} ++ ++bool NRIAdaptation::PluginsCreateContainer(nri_create_container_request *req, const std::string &conId, ++ pluginResult &result) ++{ ++ ReadGuard<RWMutex> lock(m_mutex); ++ ++ for (const auto &pair : m_storeMap) { ++ auto plugin = pair.second; ++ Errors tmpError; ++ nri_create_container_response *resp = nullptr; ++ ++ if (!plugin->CreateContainer(req, &resp, tmpError)) { ++ ERROR("Failed to call create container: %s to pliugin: %s", conId.c_str(), plugin->GetName().c_str()); ++ (void)plugin->shutdown(); ++ continue; ++ } ++ ++ if (resp == nullptr) { ++ ERROR("Empty CreateContainer resp : %s", plugin->GetName().c_str()); ++ continue; ++ } ++ ++ auto resp_wrapper = ++ makeUniquePtrCStructWrapper<nri_create_container_response>(resp, free_nri_create_container_response); ++ if (resp_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ result.Apply(CREATE_CONTAINER, resp_wrapper->get()->adjust, resp_wrapper->get()->update, ++ resp_wrapper->get()->update_len, plugin->GetName()); ++ } ++ return true; ++} ++ ++auto NRIAdaptation::PostCreateContainer(const std::string &conId, Errors &error) -> bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto con = NRIContainerByID(conId, error); ++ if (con == nullptr) { ++ ERROR("Failed to covert container to nri: %s", conId.c_str()); ++ return false; ++ } ++ ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(con->get()->pod_sandbox_id); ++ if (sandbox == nullptr) { ++ ERROR("Failed to get sandbox info for nri"); ++ return false; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto postCreateConEvent = makeUniquePtrCStructWrapper<nri_state_change_event>(free_nri_state_change_event); ++ if (postCreateConEvent == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ postCreateConEvent->get()->container = con->move(); ++ postCreateConEvent->get()->pod = pod->move(); ++ postCreateConEvent->get()->event = POST_CREATE_CONTAINER; ++ return StateChange(postCreateConEvent->get(), error); ++} ++ ++auto NRIAdaptation::StartContainer(const std::string &conId, Errors &error) -> bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto con = NRIContainerByID(conId, error); ++ if (con == nullptr) { ++ ERROR("Failed to covert container to nri: %s", conId.c_str()); ++ return false; ++ } ++ ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(con->get()->pod_sandbox_id); ++ if (sandbox == nullptr) { ++ ERROR("Failed to get sandbox info for nri"); ++ return false; ++ } ++ ++ if (sandbox == nullptr) { ++ ERROR("Failed to get sandbox for container: %s, sandbox id: %s", conId.c_str(), con->get()->pod_sandbox_id); ++ return false; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto startConEvent = makeUniquePtrCStructWrapper<nri_state_change_event>(free_nri_state_change_event); ++ if (startConEvent == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ startConEvent->get()->container = con->move(); ++ startConEvent->get()->pod = pod->move(); ++ startConEvent->get()->event = START_CONTAINER; ++ return StateChange(startConEvent->get(), error); ++} ++ ++auto NRIAdaptation::PostStartContainer(const std::string &conId, Errors &error) -> bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto con = NRIContainerByID(conId, error); ++ if (con == nullptr) { ++ ERROR("Failed to covert container to nri: %s", conId.c_str()); ++ return false; ++ } ++ ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(con->get()->pod_sandbox_id); ++ if (sandbox == nullptr) { ++ ERROR("Failed to get sandbox info for nri"); ++ return false; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto postStartConEvent = makeUniquePtrCStructWrapper<nri_state_change_event>(free_nri_state_change_event); ++ if (postStartConEvent == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ postStartConEvent->get()->container = con->move(); ++ postStartConEvent->get()->pod = pod->move(); ++ postStartConEvent->get()->event = POST_START_CONTAINER; ++ return StateChange(postStartConEvent->get(), error); ++} ++ ++auto NRIAdaptation::UndoCreateContainer(std::shared_ptr<const sandbox::Sandbox> sandbox, const std::string &conId, ++ Errors &error) -> bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ if (!StopContainer(conId, error)) { ++ ERROR("container creation undo (stop) failed: %s", conId.c_str()); ++ } ++ ++ if (!RemoveContainer(conId, error)) { ++ ERROR("container creation undo (remove) failed: %s", conId.c_str()); ++ } ++ ++ return true; ++} ++ ++auto NRIAdaptation::UpdateContainer(const std::string &conId, const runtime::v1::LinuxContainerResources &resources, ++ runtime::v1::LinuxContainerResources &adjust, Errors &error) -> bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto con = NRIContainerByID(conId, error); ++ if (con == nullptr) { ++ ERROR("Failed to covert container to nri: %s", conId.c_str()); ++ return false; ++ } ++ ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(con->get()->pod_sandbox_id); ++ if (sandbox == nullptr) { ++ ERROR("Failed to get sandbox info for nri"); ++ return false; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto req = makeUniquePtrCStructWrapper<nri_update_container_request>(free_nri_update_container_request); ++ if (req == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ auto reqRes = LinuxResourcesToNRI(resources); ++ ++ req->get()->container = con->move(); ++ req->get()->pod = pod->move(); ++ req->get()->linux_resources = reqRes; ++ ++ pluginResult result; ++ result.InitByUpdateReq(req->get()); ++ ++ if (!PluginsUpdateContainer(req->get(), conId, result)) { ++ ERROR("Failed to call update container to all plugins"); ++ return false; ++ } ++ ++ RemoveClosedPlugins(); ++ ++ // TODO:evict container do not aply ++ ++ // TODO:how can i rollback on failure ++ std::vector<nri_container_update *> failed; ++ if (!ApplyUpdates(result.GetReplyUpdate(), failed, false, error)) { ++ ERROR("Failed to apply updates"); ++ NRIHelpers::FreeNriContainerUpdateVector(failed); ++ return false; ++ } ++ ++ NRIHelpers::FreeNriContainerUpdateVector(failed); ++ ++ if (!LinuxResourcesFromNRI(result.GetReplyResources(conId), adjust)) { ++ ERROR("Failed to convert Linux resources from NRI"); ++ return false; ++ } ++ ++ return true; ++} ++ ++bool NRIAdaptation::PluginsUpdateContainer(nri_update_container_request *req, const std::string &conId, ++ pluginResult &result) ++{ ++ ReadGuard<RWMutex> lock(m_mutex); ++ ++ for (const auto &pair : m_storeMap) { ++ auto plugin = pair.second; ++ Errors tmpError; ++ nri_update_container_response *resp = nullptr; ++ ++ if (!plugin->UpdateContainer(req, &resp, tmpError)) { ++ ERROR("Failed to call create container: %s to pliugin: %s", conId.c_str(), plugin->GetName().c_str()); ++ plugin->shutdown(); ++ continue; ++ } ++ ++ if (resp == nullptr) { ++ ERROR("Empty UpdateContainer resp : %s", plugin->GetName().c_str()); ++ continue; ++ } ++ ++ auto resp_wrapper = ++ makeUniquePtrCStructWrapper<nri_update_container_response>(resp, free_nri_update_container_response); ++ if (resp_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ result.Apply(UPDATE_CONTAINER, nullptr, resp_wrapper->get()->update, resp_wrapper->get()->update_len, ++ plugin->GetName()); ++ } ++ return true; ++} ++ ++bool NRIAdaptation::PluginsStopContainer(nri_stop_container_request *req, const std::string &conId, ++ pluginResult &result) ++{ ++ ReadGuard<RWMutex> lock(m_mutex); ++ ++ for (const auto &pair : m_storeMap) { ++ auto plugin = pair.second; ++ Errors tmpError; ++ nri_stop_container_response *resp = nullptr; ++ ++ if (!plugin->StopContainer(req, &resp, tmpError)) { ++ ERROR("Failed to call create container: %s to pliugin: %s", conId.c_str(), plugin->GetName().c_str()); ++ plugin->shutdown(); ++ continue; ++ } ++ ++ if (resp == nullptr) { ++ ERROR("Empty StopContainer resp : %s", plugin->GetName().c_str()); ++ continue; ++ } ++ ++ auto resp_wrapper = makeUniquePtrCStructWrapper<nri_stop_container_response>(resp, free_nri_stop_container_response); ++ if (resp_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ result.Apply(STOP_CONTAINER, nullptr, resp_wrapper->get()->update, resp_wrapper->get()->update_len, plugin->GetName()); ++ } ++ return true; ++} ++ ++auto NRIAdaptation::PostUpdateContainer(const std::string &conId, Errors &error) ->bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto con = NRIContainerByID(conId, error); ++ if (con == nullptr) { ++ ERROR("Failed to covert container to nri: %s", conId.c_str()); ++ return false; ++ } ++ ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(con->get()->pod_sandbox_id); ++ if (sandbox == nullptr) { ++ ERROR("Failed to get sandbox info for nri"); ++ return false; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto postUpdateConEvent = makeUniquePtrCStructWrapper<nri_state_change_event>(free_nri_state_change_event); ++ if (postUpdateConEvent == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ postUpdateConEvent->get()->container = con->move(); ++ postUpdateConEvent->get()->pod = pod->move(); ++ postUpdateConEvent->get()->event = POST_UPDATE_CONTAINER; ++ return StateChange(postUpdateConEvent->get(), error); ++} ++ ++auto NRIAdaptation::StopContainer(const std::string &conId, Errors &error) ->bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto con = NRIContainerByID(conId, error); ++ if (con == nullptr) { ++ ERROR("Failed to covert container to nri: %s", conId.c_str()); ++ return false; ++ } ++ ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(con->get()->pod_sandbox_id); ++ if (sandbox == nullptr) { ++ ERROR("Failed to get sandbox info for nri"); ++ return false; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto req = makeUniquePtrCStructWrapper<nri_stop_container_request>(free_nri_stop_container_request); ++ if (req == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ req->get()->pod = pod->move(); ++ req->get()->container = con->move(); ++ ++ pluginResult result; ++ result.Init(); ++ ++ if (!PluginsStopContainer(req->get(), conId, result)) { ++ ERROR("Failed to call stop container to all plugins"); ++ return false; ++ } ++ ++ RemoveClosedPlugins(); ++ ++ // TODO:how can i rollback on failure ++ std::vector<nri_container_update *> failed; ++ if (!ApplyUpdates(result.GetReplyUpdate(), failed, false, error)) { ++ ERROR("Failed to apply updates"); ++ NRIHelpers::FreeNriContainerUpdateVector(failed); ++ return false; ++ } ++ ++ NRIHelpers::FreeNriContainerUpdateVector(failed); ++ return true; ++} ++ ++auto NRIAdaptation::RemoveContainer(const std::string &conId, Errors &error) ->bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto con = NRIContainerByID(conId, error); ++ if (con == nullptr) { ++ ERROR("Failed to covert container to nri: %s", conId.c_str()); ++ return false; ++ } ++ ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(con->get()->pod_sandbox_id); ++ if (sandbox == nullptr) { ++ ERROR("Failed to get sandbox info for nri"); ++ return false; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto removeConEvent = makeUniquePtrCStructWrapper<nri_state_change_event>(free_nri_state_change_event); ++ if (removeConEvent == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ removeConEvent->get()->container = con->move(); ++ removeConEvent->get()->pod = pod->move(); ++ removeConEvent->get()->event = REMOVE_CONTAINER; ++ return StateChange(removeConEvent->get(), error); ++} ++ ++auto NRIAdaptation::StateChange(nri_state_change_event *evt, Errors &error) ->bool ++{ ++ if (evt->event == UNKNOWN) { ++ ERROR("invalid (unset) event in state change notification"); ++ error.SetError("invalid (unset) event in state change notification"); ++ return false; ++ } ++ ++ PluginsStateChange(evt); ++ ++ RemoveClosedPlugins(); ++ return true; ++} ++ ++void NRIAdaptation::PluginsStateChange(nri_state_change_event *evt) ++{ ++ ReadGuard<RWMutex> lock(m_mutex); ++ ++ for (const auto &pair : m_storeMap) { ++ auto plugin = pair.second; ++ Errors tmpError; ++ if (!plugin->StateChange(evt, tmpError)) { ++ ERROR("invalid (unset) event in state change notification: %s", plugin->GetName().c_str()); ++ plugin->shutdown(); ++ continue; ++ } ++ } ++} ++ ++// Perform a set of unsolicited container updates requested by a plugin. ++auto NRIAdaptation::updateContainers(const nri_update_containers_request *req, ++ nri_update_containers_response **resp) ->bool ++{ ++ std::vector<nri_container_update *> failed; ++ std::vector<nri_container_update *> vec; ++ size_t i; ++ Errors error; ++ bool ret = false; ++ ++ if (req == nullptr) { ++ ERROR("Invalid request"); ++ return false; ++ } ++ ++ for (i = 0; i < req->update_len; i++) { ++ vec.push_back(req->update[i]); ++ } ++ ++ if (!ApplyUpdates(vec, failed, false, error)) { ++ ERROR("Failed to apply updates: %s", error.GetCMessage()); ++ goto free_out; ++ } ++ ++ if (failed.size() == 0) { ++ ret = true; ++ goto free_out; ++ } ++ ++ (*resp)->failed = (nri_container_update **)util_common_calloc_s(failed.size() * sizeof(nri_container_update *)); ++ if ((*resp)->failed == nullptr) { ++ ERROR("Out of memory"); ++ goto free_out; ++ } ++ ++ for (i = 0; i < failed.size(); i++) { ++ (*resp)->failed[i] = failed[i]; ++ failed[i] = nullptr; ++ (*resp)->failed_len++; ++ } ++ ret = true; ++ ++free_out: ++ NRIHelpers::FreeNriContainerUpdateVector(vec); ++ NRIHelpers::FreeNriContainerUpdateVector(failed); ++ return ret; ++} ++ ++auto NRIAdaptation::StartPlugin() -> bool ++{ ++ std::map<std::string, std::shared_ptr<NRIPlugin>> tmp_storeMap; ++ ++ if (!DiscoverPlugins(tmp_storeMap) != 0) { ++ ERROR("Failed to do DiscoverPlugins"); ++ return false; ++ } ++ ++ for (const auto &pair : tmp_storeMap) { ++ const std::string &index = pair.first; ++ auto plugin = pair.second; ++ ++ if (!NewLaunchedPlugin(plugin)) { ++ ERROR("Failed to do NewLaunchedPlugin for %s", plugin->GetName().c_str()); ++ continue; ++ } ++ ++ AddPluginByIndex(index, plugin); ++ ++ if (!plugin->Start(m_plugin_registration_timeout, m_plugin_requst_timeout)) { ++ ERROR("Failed to start plugin %s ready", plugin->GetName().c_str()); ++ RemovePluginByIndex(index); ++ (void)plugin->shutdown(); ++ continue; ++ } ++ } ++ ++ return true; ++} ++ ++// return true always, failure to obtain one plugin does not affect other plugin ++static auto walk_plugin_dir_cb(const char *path_name, const struct dirent *sub_dir, void *context) -> bool ++{ ++ std::string full_path = std::string(path_name) + "/" + sub_dir->d_name; ++ struct stat file_stat = { 0 }; ++ ++ if (stat(full_path.c_str(), &file_stat) != 0) { ++ WARN("Failed to get NRI plugin %s stat", sub_dir->d_name); ++ return true; ++ } ++ ++ if (S_ISDIR(file_stat.st_mode)) { ++ INFO("Skip dir in plugin path %s", sub_dir->d_name); ++ return true; ++ } ++ ++ // 1. Verify plugin permissions for exec ++ if (!(file_stat.st_mode & S_IXUSR)) { ++ ERROR("NRI plugin %s has no permission for exec", sub_dir->d_name); ++ return true; ++ } ++ ++ // 2. Parse plugin name ++ __isula_auto_array_t char **arr = util_string_split_n(sub_dir->d_name, '-', 2); ++ if (arr == nullptr) { ++ ERROR("Invalid plugin name %s, idx-pluginname expected", sub_dir->d_name); ++ return true; ++ } ++ ++ if (!NRIHelpers::CheckPluginIndex(arr[0])) { ++ ERROR("Invalid plugin name %s, invalid idx", sub_dir->d_name); ++ return true; ++ } ++ ++ // 3. init plugin ++ std::string index(arr[0]); ++ std::string pluginName(arr[1]); ++ std::string config; ++ ++ std::map<std::string, std::shared_ptr<NRIPlugin>> &map = ++ *static_cast<std::map<std::string, std::shared_ptr<NRIPlugin>>*>(context); ++ if (!NRIHelpers::GetPluginConfig(index, pluginName, config)) { ++ ERROR("Failed to get plugin %s config", pluginName.c_str()); ++ return true; ++ } ++ ++ auto plugin = std::make_shared<NRIPlugin>(index, pluginName, config); ++ ++ // todo:use random id str ++ map[pluginName] = plugin; ++ return true; ++} ++ ++static void plugin_exec_func(nri_plugin_exec_args_t * plugin_args) ++{ ++ const char *params[PARAM_NUM] = {0}; ++ int i = 0; ++ std::string sock = std::to_string(plugin_args->sockFd); ++ ++ if (plugin_args == nullptr) { ++ ERROR("Missing plugin exec info"); ++ _exit(EXIT_FAILURE); ++ } ++ ++ if (chdir(plugin_args->workdir) < 0) { ++ ERROR("Failed to chdir to %s", plugin_args->workdir); ++ _exit(EXIT_FAILURE); ++ } ++ ++ if (setenv(PluginNameEnv.c_str(), plugin_args->name, 1) != 0) { ++ ERROR("%s: failed to set PluginNameEnv for process %d", plugin_args->name, getpid()); ++ exit(EXIT_FAILURE); ++ } ++ ++ if (setenv(PluginIdxEnv.c_str(), plugin_args->index, 1) != 0) { ++ ERROR("%s: failed to set PluginIdxEnv for process %d", plugin_args->name, getpid()); ++ exit(EXIT_FAILURE); ++ } ++ ++ if (setenv(PluginSocketEnv.c_str(), sock.c_str(), 1) != 0) { ++ ERROR("%s: failed to set PluginSocketEnv for process %d", plugin_args->name, getpid()); ++ exit(EXIT_FAILURE); ++ } ++ ++ if (util_check_inherited(true, plugin_args->sockFd) != 0) { ++ ERROR("Failed to close inherited fds"); ++ exit(EXIT_FAILURE); ++ } ++ ++ if (setsid() < 0) { ++ ERROR("Failed to setsid for nri plugin: %s", plugin_args->name); ++ exit(EXIT_FAILURE); ++ } ++ ++ params[i++] = plugin_args->name; ++ ++ execvp(plugin_args->cmd, (char * const *)params); ++ ERROR("Failed to exec %s", plugin_args->cmd); ++ _exit(EXIT_FAILURE); ++} ++ ++// create socket, and call plugin start ++auto NRIAdaptation::NewLaunchedPlugin(const std::shared_ptr<NRIPlugin> &plugin) -> bool ++{ ++ // 1. create socket for plugin ++ if (!plugin->CreateSocketPair()) { ++ ERROR("Failed to create socket pair"); ++ return false; ++ } ++ ++ std::string name = plugin->GetQualifiedName(); ++ std::string cmd = m_pluginPath + "/" + name; ++ ++ DEBUG("Plugin %s start", cmd.c_str()); ++ ++ // 2. exec plugin ++ nri_plugin_exec_args_t p_args = { ++ .workdir = m_pluginPath.c_str(), ++ .cmd = cmd.c_str(), ++ .name = name.c_str(), ++ .index = plugin->GetIndex().c_str(), ++ .sockFd = plugin->GetPeerSockFd(), ++ }; ++ ++ int pid = fork(); ++ if (pid == (pid_t) -1) { ++ SYSERROR("Failed to fork"); ++ return false; ++ } ++ ++ if (pid == (pid_t)0) { ++ set_child_process_pdeathsig(); ++ ++ plugin_exec_func(&p_args); ++ } ++ ++ close(plugin->GetPeerSockFd()); ++ ++ plugin->SetPid(pid); ++ ++ return true; ++} ++ ++// find plugin and create plugin ++auto NRIAdaptation::DiscoverPlugins(std::map<std::string, std::shared_ptr<NRIPlugin>> &map) -> bool ++{ ++ int nret = 0; ++ ++ // 1. get all plugin ++ nret = util_scan_subdirs(m_pluginPath.c_str(), walk_plugin_dir_cb, static_cast<void*>(&map)); ++ if (nret != 0) { ++ ERROR("Failed to scan nri plugin subdirs"); ++ } ++ return true; ++} ++ ++auto NRIAdaptation::SortPlugins() -> bool ++{ ++ RemoveClosedPlugins(); ++ ++ std::vector<std::pair<std::string, std::shared_ptr<NRIPlugin>>> sortedPlugins(m_storeMap.begin(), m_storeMap.end()); ++ ++ std::sort(sortedPlugins.begin(), sortedPlugins.end(), [](const auto & a, const auto & b) { ++ return a.first < b.first; ++ }); ++ ++ WriteGuard<RWMutex> lock(m_mutex); ++ m_storeMap.clear(); ++ for (const auto &pair : sortedPlugins) { ++ m_storeMap.insert(pair); ++ } ++ ++ return true; ++} ++ ++auto NRIAdaptation::GetNRIPluginConfigPath(void) -> std::string ++{ ++ __isula_auto_free char *config_path = nullptr; ++ std::string ret; ++ ++ config_path = conf_get_nri_plugin_config_path(); ++ if (config_path == nullptr) { ++ return ret; ++ } ++ ret = std::string(config_path); ++ return ret; ++} ++ ++auto NRIAdaptation::GetNRIPluginPath(void) -> std::string ++{ ++ __isula_auto_free char *plugin_path = nullptr; ++ std::string ret; ++ ++ plugin_path = conf_get_nri_plugin_path(); ++ if (plugin_path == nullptr) { ++ return ret; ++ } ++ ret = std::string(plugin_path); ++ return ret; ++} ++ ++auto NRIAdaptation::GetNRISockPath(void) -> std::string ++{ ++ __isula_auto_free char *sock_path = nullptr; ++ std::string ret; ++ ++ sock_path = conf_get_socket_path(); ++ if (sock_path == nullptr) { ++ return ret; ++ } ++ ret = std::string(sock_path); ++ return ret; ++} ++ ++auto NRIAdaptation::NRIPodSandbox(const std::shared_ptr<const sandbox::Sandbox> &sandbox, ++ Errors &error) -> std::unique_ptr<CStructWrapper<nri_pod_sandbox>> ++{ ++ auto pod = makeUniquePtrCStructWrapper<nri_pod_sandbox>(free_nri_pod_sandbox); ++ if (pod == nullptr) { ++ ERROR("Out of memory"); ++ return nullptr; ++ } ++ ++ if (!PodSandboxToNRI(sandbox, *pod->get())) { ++ error.Errorf("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return nullptr; ++ } ++ ++ return pod; ++} ++ ++auto NRIAdaptation::NRIContainerByID(const std::string &id, ++ Errors &error) -> std::unique_ptr<CStructWrapper<nri_container>> ++{ ++ auto con = makeUniquePtrCStructWrapper<nri_container>(free_nri_container); ++ if (con == nullptr) { ++ ERROR("Out of memory"); ++ return nullptr; ++ } ++ ++ if (!ContainerToNRIByID(id, *con->get())) { ++ error.Errorf("Failed to covert container to nri: %s", id.c_str()); ++ ERROR("Failed to covert container to nri: %s", id.c_str()); ++ return nullptr; ++ } ++ ++ return con; ++} ++ ++auto NRIAdaptation::NRIContainerByConConfig(const std::shared_ptr<const sandbox::Sandbox> &sandbox, ++ const runtime::v1::ContainerConfig &containerConfig, Errors &error) -> std::unique_ptr<CStructWrapper<nri_container>> ++{ ++ auto con = makeUniquePtrCStructWrapper<nri_container>(free_nri_container); ++ if (con == nullptr) { ++ ERROR("Out of memory"); ++ return nullptr; ++ } ++ ++ if (!ContainerToNRIByConConfig(containerConfig, *con->get())) { ++ error.Errorf("Failed to covert container to nri: %s", con->get()->name); ++ ERROR("Failed to covert container to nri: %s", con->get()->name); ++ return nullptr; ++ } ++ con->get()->pod_sandbox_id = isula_strdup_s(sandbox->GetId().c_str()); ++ ++ return con; ++} +\ No newline at end of file +diff --git a/src/daemon/nri/nri_adaption.h b/src/daemon/nri/nri_adaption.h +index 874662cf..27a6d93e 100644 +--- a/src/daemon/nri/nri_adaption.h ++++ b/src/daemon/nri/nri_adaption.h +@@ -46,10 +46,6 @@ public: + + auto GetSockpath(std::vector<std::string> &paths) -> bool; + +- auto StopPlugins() -> bool; +- +- void RemoveClosedPlugins(); +- + auto GetPluginByIndex(const std::string &index) -> std::shared_ptr<NRIPlugin>; + void AddPluginByIndex(const std::string &index, std::shared_ptr<NRIPlugin> plugin); + void RemovePluginByIndex(const std::string &index); +@@ -65,7 +61,8 @@ public: + Errors &error) -> bool; + auto StartContainer(const std::string &conId, Errors &error) -> bool; + auto PostStartContainer(const std::string &conId, Errors &error) -> bool; +- auto UpdateContainer(const std::string &conId, Errors &error) -> bool; ++ auto UpdateContainer(const std::string &conId, const runtime::v1::LinuxContainerResources &resources, ++ runtime::v1::LinuxContainerResources &adjust, Errors &error) -> bool; + auto PostUpdateContainer(const std::string &conId, Errors &error) -> bool; + auto StopContainer(const std::string &conId, Errors &error) -> bool; + auto RemoveContainer(const std::string &conId, Errors &error) -> bool; +@@ -87,6 +84,8 @@ private: + auto SyncPlugin() -> bool; + + auto SortPlugins() -> bool; ++ void RemoveClosedPlugins(); ++ + void GetClosedPlugins(std::vector<std::string> &closedPlugin); + + auto ApplyUpdates(const std::vector<nri_container_update *> &update, std::vector<nri_container_update *> &failed, +@@ -107,18 +106,16 @@ private: + void PluginsStateChange(nri_state_change_event *evt); + bool PluginsCreateContainer(nri_create_container_request *req, const std::string &conId, pluginResult &result); + bool PluginsUpdateContainer(nri_update_container_request *req, const std::string &conId, pluginResult &result); ++ bool PluginsStopContainer(nri_stop_container_request *req, const std::string &conId, pluginResult &result); + + private: + RWMutex m_mutex; + static std::atomic<NRIAdaptation *> m_instance; + bool m_support; + bool m_external_support; +- std::string m_version; + std::string m_sock_path; + std::string m_pluginConfigPath; + std::string m_pluginPath; +- std::vector<std::string> m_socketPathArr; +- std::string m_disableConnections; + // id --> NRIPlugin map + std::map<std::string, std::shared_ptr<NRIPlugin>> m_storeMap; + // TODO:plugin monitor thread id?? +diff --git a/src/daemon/nri/nri_helpers.cc b/src/daemon/nri/nri_helpers.cc +index ff9d67c1..b660e7a7 100644 +--- a/src/daemon/nri/nri_helpers.cc ++++ b/src/daemon/nri/nri_helpers.cc +@@ -90,4 +90,25 @@ bool CheckPluginIndex(const std::string &idx) + + return true; + } ++ ++void FreeNriContainerUpdateVector(std::vector<nri_container_update *> &vec) ++{ ++ for (auto ptr : vec) { ++ free_nri_container_update(ptr); ++ } ++} ++ ++void FreeNriContainerVector(std::vector<nri_container *> &vec) ++{ ++ for (auto ptr : vec) { ++ free_nri_container(ptr); ++ } ++} ++ ++void FreeNriPodVector(std::vector<nri_pod_sandbox *> &vec) ++{ ++ for (auto ptr : vec) { ++ free_nri_pod_sandbox(ptr); ++ } ++} + }// namespace NRIHelpers +\ No newline at end of file +diff --git a/src/daemon/nri/nri_helpers.h b/src/daemon/nri/nri_helpers.h +index 1a2f488e..e776987b 100644 +--- a/src/daemon/nri/nri_helpers.h ++++ b/src/daemon/nri/nri_helpers.h +@@ -41,6 +41,10 @@ void GenerateRandomExternalName(std::string &ret); + + bool CheckPluginIndex(const std::string &idx); + ++void FreeNriContainerUpdateVector(std::vector<nri_container_update *> &vec); ++void FreeNriContainerVector(std::vector<nri_container *> &vec); ++void FreeNriPodVector(std::vector<nri_pod_sandbox *> &vec); ++ + template <typename T> + void freeArray(T ** &arr, int size) + { +diff --git a/src/daemon/nri/nri_plugin_ops.cc b/src/daemon/nri/nri_plugin_ops.cc +new file mode 100644 +index 00000000..e2f88b63 +--- /dev/null ++++ b/src/daemon/nri/nri_plugin_ops.cc +@@ -0,0 +1,123 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-03-26 ++ * Description: provide nri plugin api definition ++ ******************************************************************************/ ++#include "nri_plugin_ops.h" ++ ++#include <isula_libutils/log.h> ++#include <isula_libutils/auto_cleanup.h> ++ ++#include "nri_adaption.h" ++#include "nri_plugin.h" ++#include "isulad_config.h" ++ ++static bool start_external_listener() ++{ ++ __isula_auto_free char *sock_path = NULL; ++ ++ sock_path = conf_get_socket_path(); ++ if (sock_path == NULL) { ++ ERROR("Failed to get socket path"); ++ return false; ++ } ++ ++ if (nri_external_service_start(sock_path, nri_external_plugin_connect) != 0) { ++ ERROR("Failed to lauch external service"); ++ return false; ++ } ++ return true; ++} ++ ++bool nri_adaption_init(void) ++{ ++ Errors error; ++ ++ if (!conf_get_nri_support()) { ++ return true; ++ } ++ ++ nri_runtime_callbacks callbacks; ++ callbacks.register_plugin = nri_registry_containers; ++ callbacks.update_containers = nri_update_containers; ++ if (nri_runtime_service_init(callbacks) != 0) { ++ ERROR("Failed to init runtime service\n"); ++ return false; ++ } ++ ++ if (conf_get_nri_external_support()) { ++ if (!start_external_listener()) { ++ ERROR("Failed to start external listener\n"); ++ goto clean_out; ++ } ++ } ++ ++ NRIAdaptation::GetInstance()->Init(error); ++ if (error.NotEmpty()) { ++ ERROR("Failed to init NRIAdaptation: %s", error.GetCMessage()); ++ goto clean_out; ++ } ++ return true; ++clean_out: ++ nri_runtime_service_destroy(); ++ return false; ++} ++ ++bool nri_adaption_shutdown(void) ++{ ++ nri_external_service_shutdown(); ++ nri_runtime_service_destroy(); ++ return true; ++} ++ ++int nri_update_containers(const char *plugin_id, const nri_update_containers_request *request, ++ nri_update_containers_response **response) ++{ ++ if (request == nullptr || response == nullptr || plugin_id == nullptr) { ++ ERROR("Invalid input arguments"); ++ return -1; ++ } ++ ++ if (!NRIAdaptation::GetInstance()->updateContainers(request, response)) { ++ ERROR("Failed to update containers by plugin %s", plugin_id); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++int nri_registry_containers(const char *plugin_id, const nri_register_plugin_request *request) ++{ ++ if (request == nullptr || plugin_id == nullptr) { ++ ERROR("Invalid input arguments"); ++ return -1; ++ } ++ ++ auto plugin = NRIAdaptation::GetInstance()->GetPluginByIndex(plugin_id); ++ if (plugin == nullptr) { ++ ERROR("Failed to get plugin by index %s", plugin_id); ++ return -1; ++ } ++ ++ plugin->SetReady(); ++ return 0; ++} ++ ++int nri_external_plugin_connect(int fd) ++{ ++ if (fd < 0) { ++ ERROR("Invalid input arguments"); ++ return -1; ++ } ++ ++ return NRIAdaptation::GetInstance()->NewExternalPlugin(fd) ? 0 : -1; ++} +\ No newline at end of file +diff --git a/src/daemon/nri/nri_plugin_ops.h b/src/daemon/nri/nri_plugin_ops.h +index 37d437d2..3a5393ba 100644 +--- a/src/daemon/nri/nri_plugin_ops.h ++++ b/src/daemon/nri/nri_plugin_ops.h +@@ -25,13 +25,15 @@ extern "C" { + #endif + + bool nri_adaption_init(void); ++bool nri_adaption_shutdown(void); + + #ifdef __cplusplus + } + #endif + +-int nri_update_containers(const nri_update_containers_request *request, nri_update_containers_response **response); +-int nri_registry_containers(const nri_register_plugin_request *request); ++int nri_update_containers(const char *plugin_id, const nri_update_containers_request *request, ++ nri_update_containers_response **response); ++int nri_registry_containers(const char *plugin_id, const nri_register_plugin_request *request); + + int nri_external_plugin_connect(int fd); + +diff --git a/src/daemon/nri/nri_result.cc b/src/daemon/nri/nri_result.cc +new file mode 100644 +index 00000000..7da4451d +--- /dev/null ++++ b/src/daemon/nri/nri_result.cc +@@ -0,0 +1,977 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-06-29 ++ * Description: provide nri result definition ++ *********************************************************************************/ ++ ++#include "nri_result.h" ++ ++#include <isula_libutils/log.h> ++#include <isula_libutils/nri_container_adjustment.h> ++ ++#include "cxxutils.h" ++#include "transform.h" ++#include "utils.h" ++ ++pluginResult::~pluginResult() ++{ ++ free_nri_linux_resources(m_update_req); ++ free_nri_container_adjustment(m_reply.adjust); ++ for (size_t i = 0; i < m_reply.update.size(); i++) { ++ free_nri_container_update(m_reply.update[i]); ++ } ++} ++ ++auto pluginResult::InitReply() -> bool ++{ ++ m_reply.adjust = (nri_container_adjustment *)util_common_calloc_s(sizeof(nri_container_adjustment)); ++ if (m_reply.adjust == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ return true; ++} ++ ++auto pluginResult::Init() -> bool ++{ ++ if (!InitReply()) { ++ ERROR("Failed to init reply"); ++ return false; ++ } ++ m_update_req = nullptr; ++ return true; ++} ++ ++auto pluginResult::InitByConId(std::string conId) -> bool ++{ ++ m_conId = conId; ++ ++ if (!InitReply()) { ++ ERROR("Failed to init reply"); ++ return false; ++ } ++ ++ m_update_req = nullptr; ++ return true; ++} ++ ++auto pluginResult::InitByUpdateReq(nri_update_container_request *req) -> bool ++{ ++ m_conId = req->container->id; ++ m_update_req = copy_nri_linux_resources(req->linux_resources); ++ if (m_update_req == nullptr) { ++ ERROR("Failed to copy nri linux resources"); ++ return false; ++ } ++ ++ if (!InitReply()) { ++ ERROR("Failed to init reply"); ++ return false; ++ } ++ m_update_req = nullptr; ++ return true; ++} ++ ++auto pluginResult::GetReplyUpdate() -> std::vector<nri_container_update*> ++{ ++ return m_reply.update; ++} ++ ++auto pluginResult::MoveReplyAdjust() -> nri_container_adjustment * ++{ ++ nri_container_adjustment *ret = m_reply.adjust; ++ m_reply.adjust = nullptr; ++ return ret; ++} ++ ++auto pluginResult::GetReplyResources(const std::string &id) -> const nri_linux_resources * ++{ ++ nri_linux_resources *ret = NULL; ++ nri_container_update *update = m_updates[id]; ++ ret = update->linux->resources; ++ return ret; ++} ++ ++auto pluginResult::Apply(int32_t event, const nri_container_adjustment *adjust, nri_container_update **update, ++ size_t update_len, const std::string &plugin) -> bool ++{ ++ if (plugin.length() == 0) { ++ ERROR("Empty plugin name"); ++ return false; ++ } ++ if (event == CREATE_CONTAINER) { ++ if (!Adjust(adjust, plugin)) { ++ ERROR("Failed to do adjust to plugin: %s", plugin.c_str()); ++ return false; ++ } ++ ++ if (!Update(update, update_len, plugin)) { ++ ERROR("Failed to do update to plugin: %s", plugin.c_str()); ++ return false; ++ } ++ return true; ++ } else if (event == UPDATE_CONTAINER) { ++ if (!Update(update, update_len, plugin)) { ++ ERROR("Failed to do update to plugin: %s", plugin.c_str()); ++ return false; ++ } ++ return true; ++ } else if (event == STOP_CONTAINER) { ++ if (!Update(update, update_len, plugin)) { ++ ERROR("Failed to do update to plugin: %s", plugin.c_str()); ++ return false; ++ } ++ return true; ++ } else { ++ ERROR("Cannot apply response of invalid type %d", event); ++ return false; ++ } ++ return true; ++} ++ ++auto pluginResult::Adjust(const nri_container_adjustment *adjust, const std::string &plugin) -> bool ++{ ++ if (adjust == nullptr) { ++ return true; ++ } ++ ++ if (!AdjustAnnotations(adjust->annotations, plugin)) { ++ ERROR("Cannot adajust annotations by plugin %s", plugin.c_str()); ++ return false; ++ } ++ ++ if (!AdjustMounts(adjust->mounts, adjust->mounts_len, plugin)) { ++ ERROR("Cannot adajust mounts by plugin %s", plugin.c_str()); ++ return false; ++ } ++ ++ if (!AdjustEnv(adjust->env, adjust->env_len, plugin)) { ++ ERROR("Cannot adajust mounts by plugin %s", plugin.c_str()); ++ return false; ++ } ++ ++ if (!AdjustHooks(adjust->hooks, plugin)) { ++ ERROR("Cannot adajust hooks by plugin %s", plugin.c_str()); ++ return false; ++ } ++ ++ if (adjust->linux != nullptr) { ++ if (m_reply.adjust->linux == nullptr) { ++ m_reply.adjust->linux = (nri_linux_container_adjustment *)util_common_calloc_s(sizeof(nri_linux_container_adjustment)); ++ if (m_reply.adjust->linux == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ } ++ ++ if (!AdjustDevices(adjust->linux->devices, adjust->linux->devices_len, plugin)) { ++ ERROR("Cannot adajust devices by plugin %s", plugin.c_str()); ++ return false; ++ } ++ ++ if (!AdjustResources(adjust->linux->resources, plugin)) { ++ ERROR("Cannot adajust devices by plugin %s", plugin.c_str()); ++ return false; ++ } ++ ++ if (!AdjustCgroupsPath(adjust->linux->cgroups_path, plugin)) { ++ ERROR("Cannot adajust cgroups path by plugin %s", plugin.c_str()); ++ return false; ++ } ++ } ++ ++ if (!AdjustRlimits(adjust->rlimits, adjust->rlimits_len, plugin)) { ++ ERROR("Cannot adajust rlimits path by plugin %s", plugin.c_str()); ++ return false; ++ } ++ ++ return true; ++} ++ ++auto pluginResult::AdjustAnnotations(json_map_string_string *annos, const std::string &plugin) -> bool ++{ ++ if (annos == nullptr || annos->len == 0) { ++ return true; ++ } ++ ++ if (m_reply.adjust->annotations == nullptr) { ++ m_reply.adjust->annotations = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); ++ if (m_reply.adjust->annotations == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ } ++ ++ google::protobuf::Map<std::string, std::string> del; ++ const char *id = m_conId.c_str(); ++ google::protobuf::Map<std::string, std::string> mapAnno; ++ Transform::JsonMapToProtobufMapForString(annos, mapAnno); ++ ++ // if key is marked for remove, add pair to del, and delete key from annos ++ for (auto it = mapAnno.begin(); it != mapAnno.end();) { ++ const std::string &key = it->first; ++ char *out = NULL; ++ if (is_marked_for_removal(key.c_str(), &out)) { ++ del[out] = ""; ++ it = mapAnno.erase(it); ++ } else { ++ ++it; ++ } ++ } ++ ++ for (const auto &iter : mapAnno) { ++ std::string key = iter.first; ++ std::string value = iter.second; ++ auto it = del.find(key); ++ if (it != del.end()) { ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ m_owners[id].annotations.erase(key); ++ } ++ append_json_map_string_string(m_reply.adjust->annotations, NRIHelpers::MarkForRemoval(key).c_str(), ""); ++ } ++ ++ // set annotations's owner plugin ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ auto anno = m_owners[id].annotations.find(key); ++ if (anno != m_owners[id].annotations.end()) { ++ ERROR("plugins %s and %s both tried to set annotation: %s", plugin.c_str(), anno->second.c_str(), key.c_str()); ++ return false; ++ } ++ m_owners[id].annotations[key] = plugin; ++ } ++ ++ // add pair to m_reply.adjust ++ append_json_map_string_string(m_reply.adjust->annotations, key.c_str(), value.c_str()); ++ del.erase(key); ++ } ++ ++ // add del to m_reply.adjust ++ for (auto &pair : del) { ++ append_json_map_string_string(m_reply.adjust->annotations, NRIHelpers::MarkForRemoval(pair.first).c_str(), ""); ++ } ++ ++ return true; ++} ++ ++auto pluginResult::AdjustMounts(nri_mount **mounts, size_t mounts_size, const std::string &plugin) -> bool ++{ ++ if (mounts == nullptr || mounts_size == 0) { ++ return true; ++ } ++ ++ size_t i; ++ std::vector<nri_mount *> add; ++ std::map<std::string, nri_mount *> del; ++ std::string id = m_conId.c_str(); ++ ++ // first split removals from the rest of adjustments ++ for (i = 0; i < mounts_size; i++) { ++ char *out = NULL; ++ if (is_marked_for_removal(mounts[i]->destination, &out)) { ++ del[out] = mounts[i]; ++ } else { ++ add.push_back(mounts[i]); ++ } ++ } ++ ++ // next remove marked mounts from collected adjustments ++ nri_mount** cleard = nullptr; ++ size_t clearLen = 0; ++ ++ if (m_reply.adjust->mounts_len > 0) { ++ cleard = (nri_mount **)util_common_calloc_s(m_reply.adjust->mounts_len * sizeof(nri_mount *)); ++ if (cleard == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ for (i = 0; i < m_reply.adjust->mounts_len; i++) { ++ auto removed = del.find(m_reply.adjust->mounts[i]->destination); ++ if (removed != del.end()) { ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ m_owners[id].mounts.erase(m_reply.adjust->mounts[i]->destination); ++ } ++ continue; ++ } ++ cleard[clearLen] = copy_nri_mount(m_reply.adjust->mounts[i]); ++ if (cleard[clearLen] == nullptr) { ++ ERROR("Failed to copy nri mounts to cleard"); ++ return false; ++ } ++ clearLen++; ++ } ++ ++ NRIHelpers::freeArray(m_reply.adjust->mounts, m_reply.adjust->mounts_len); ++ m_reply.adjust->mounts = cleard; ++ m_reply.adjust->mounts_len = clearLen; ++ } ++ ++ // finally, apply additions to collected adjustments ++ size_t oldSize, newSize; ++ oldSize = m_reply.adjust->mounts_len * sizeof(nri_mount *); ++ newSize = oldSize + add.size() * sizeof(nri_mount *); ++ int ret = util_mem_realloc((void **)(&m_reply.adjust->mounts), newSize, (void *)m_reply.adjust->mounts, oldSize); ++ if (ret != 0) { ++ ERROR("Failed to realloc and assign nri mounts array"); ++ return false; ++ } ++ for (i = 0; i < add.size(); i++) { ++ // set mounts's owner plugin ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ auto mount = m_owners[id].mounts.find(add[i]->destination); ++ if (mount != m_owners[id].mounts.end()) { ++ ERROR("plugins %s and %s both tried to set mount: %s", plugin.c_str(), mount->second.c_str(), add[i]->destination); ++ return false; ++ } ++ m_owners[id].mounts[add[i]->destination] = plugin; ++ } ++ m_reply.adjust->mounts[m_reply.adjust->mounts_len] = copy_nri_mount(add[i]); ++ if (m_reply.adjust->mounts[m_reply.adjust->mounts_len] == nullptr) { ++ ERROR("Failed to copy add nri mounts to reply adjust"); ++ return false; ++ } ++ m_reply.adjust->mounts_len++; ++ } ++ ++ return true; ++} ++ ++auto pluginResult::AdjustEnv(nri_key_value **envs, size_t envs_size, const std::string &plugin) -> bool ++{ ++ if (envs == nullptr || envs_size == 0) { ++ return true; ++ } ++ ++ size_t i; ++ std::vector<nri_key_value *> add; ++ std::map<std::string, nri_key_value *> del; ++ std::string id = m_conId.c_str(); ++ ++ // first split removals from the rest of adjustments ++ for (i = 0; i < envs_size; i++) { ++ char *out = NULL; ++ if (is_marked_for_removal(envs[i]->key, &out)) { ++ del[out] = envs[i]; ++ } else { ++ add.push_back(envs[i]); ++ } ++ } ++ ++ // next remove marked mounts from collected adjustments ++ nri_key_value** cleard; ++ size_t clearLen = 0; ++ ++ if(m_reply.adjust->env_len > 0) { ++ cleard = (nri_key_value **)util_common_calloc_s(m_reply.adjust->env_len * sizeof(nri_key_value *)); ++ if (cleard == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ for (i = 0; i < m_reply.adjust->env_len; i++) { ++ auto removed = del.find(m_reply.adjust->env[i]->key); ++ if (removed != del.end()) { ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ m_owners[id].env.erase(m_reply.adjust->env[i]->key); ++ } ++ continue; ++ } ++ cleard[clearLen] = copy_nri_key_value(m_reply.adjust->env[i]); ++ if (cleard[clearLen] == nullptr) { ++ ERROR("Failed to copy nri env key value to cleard"); ++ return false; ++ } ++ clearLen++; ++ } ++ ++ NRIHelpers::freeArray(m_reply.adjust->env, m_reply.adjust->env_len); ++ m_reply.adjust->env = cleard; ++ m_reply.adjust->env_len = clearLen; ++ } ++ ++ // finally, apply additions to collected adjustments ++ size_t oldSize, newSize; ++ oldSize = m_reply.adjust->env_len * sizeof(nri_key_value *); ++ newSize = oldSize + add.size() * sizeof(nri_key_value *); ++ int ret = util_mem_realloc((void **)(&m_reply.adjust->env), newSize, m_reply.adjust->env, oldSize); ++ if (ret != 0) { ++ ERROR("Failed to realloc and assign nri env array"); ++ return false; ++ } ++ for (i = 0; i < add.size(); i++) { ++ // set env's owner plugin ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ auto env = m_owners[id].env.find(add[i]->key); ++ if (env != m_owners[id].env.end()) { ++ ERROR("plugins %s and %s both tried to set env: %s", plugin.c_str(), env->second.c_str(), add[i]->key); ++ return false; ++ } ++ m_owners[id].env[add[i]->key] = plugin; ++ } ++ m_reply.adjust->env[m_reply.adjust->env_len] = copy_nri_key_value(add[i]); ++ if (m_reply.adjust->env[m_reply.adjust->env_len] == nullptr) { ++ ERROR("Failed to copy add nri env to reply adjust"); ++ return false; ++ } ++ m_reply.adjust->env_len++; ++ } ++ ++ return true; ++} ++ ++auto pluginResult::AdjustHooks(const nri_hooks *hooks, const std::string &plugin) -> bool ++{ ++ if (hooks == nullptr) { ++ return true; ++ } ++ ++ if (m_reply.adjust->hooks == nullptr) { ++ m_reply.adjust->hooks = (nri_hooks *)util_common_calloc_s(sizeof(nri_hooks)); ++ if (m_reply.adjust->hooks == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ } ++ ++ nri_hooks * reply = m_reply.adjust->hooks; ++ ++ if (!merge_nri_hooks(reply->prestart, reply->prestart_len, (const nri_hook**)hooks->prestart, hooks->prestart_len)) { ++ ERROR("Failed to realloc and copy prestart hooks"); ++ return false; ++ } ++ ++ if (!merge_nri_hooks(reply->poststart, reply->poststart_len, (const nri_hook**)hooks->poststart, ++ hooks->poststart_len)) { ++ ERROR("Failed to realloc and copy poststart hooks"); ++ return false; ++ } ++ ++ if (!merge_nri_hooks(reply->poststop, reply->poststop_len, (const nri_hook**)hooks->poststop, hooks->poststop_len)) { ++ ERROR("Failed to realloc and copy poststop hooks"); ++ return false; ++ } ++ ++ /* TODO:zhongtao ++ * The OCI being used by the iSulad not supportes ++ * createRuntime/createContainer/startContainer currently. ++ */ ++ if (!merge_nri_hooks(reply->create_runtime, reply->create_runtime_len, (const nri_hook**)hooks->create_runtime, ++ hooks->create_runtime_len)) { ++ ERROR("Failed to realloc and copy create_runtime hooks"); ++ return false; ++ } ++ ++ if (!merge_nri_hooks(reply->create_container, reply->create_container_len, (const nri_hook**)hooks->create_container, ++ hooks->create_container_len)) { ++ ERROR("Failed to realloc and copy create_container hooks"); ++ return false; ++ } ++ ++ if (!merge_nri_hooks(reply->start_container, reply->start_container_len, (const nri_hook**)hooks->start_container, ++ hooks->start_container_len)) { ++ ERROR("Failed to realloc and copy start_container hooks"); ++ return false; ++ } ++ ++ return false; ++} ++ ++auto pluginResult::AdjustDevices(nri_linux_device **devices, size_t devices_size, const std::string &plugin) -> bool ++{ ++ if (devices_size == 0) { ++ return true; ++ } ++ ++ size_t i; ++ std::vector<nri_linux_device *> add; ++ std::map<std::string, nri_linux_device *> del; ++ std::string id = m_conId.c_str(); ++ ++ // first split removals from the rest of adjustments ++ for (i = 0; i < devices_size; i++) { ++ char *out = NULL; ++ if (is_marked_for_removal(devices[i]->path, &out)) { ++ del[out] = devices[i]; ++ } else { ++ add.push_back(devices[i]); ++ } ++ } ++ ++ // next remove marked mounts from collected adjustments ++ nri_linux_device** cleard; ++ size_t clearLen = 0; ++ ++ if (m_reply.adjust->linux->devices_len > 0) { ++ cleard = (nri_linux_device **)util_common_calloc_s(m_reply.adjust->linux->devices_len * sizeof(nri_linux_device *)); ++ if (cleard == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ for (i = 0; i < m_reply.adjust->linux->devices_len; i++) { ++ auto removed = del.find(m_reply.adjust->linux->devices[i]->path); ++ if (removed != del.end()) { ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ m_owners[id].devices.erase(m_reply.adjust->linux->devices[i]->path); ++ } ++ continue; ++ } ++ cleard[clearLen] = copy_nri_device(m_reply.adjust->linux->devices[i]); ++ if (cleard[clearLen] == nullptr) { ++ ERROR("Failed to copy nri linux device to cleard"); ++ return false; ++ } ++ clearLen++; ++ } ++ ++ NRIHelpers::freeArray(m_reply.adjust->linux->devices, m_reply.adjust->linux->devices_len); ++ m_reply.adjust->linux->devices = cleard; ++ m_reply.adjust->linux->devices_len = clearLen; ++ } ++ ++ // finally, apply additions to collected adjustments ++ size_t oldSize, newSize; ++ oldSize = m_reply.adjust->linux->devices_len * sizeof(nri_linux_device *); ++ newSize = oldSize + add.size() * sizeof(nri_linux_device *); ++ int ret = util_mem_realloc((void **)(&m_reply.adjust->linux->devices), newSize, m_reply.adjust->linux->devices, oldSize); ++ if (ret != 0) { ++ ERROR("Failed to realloc and assign nri devices array"); ++ return false; ++ } ++ for (i = 0; i < add.size(); i++) { ++ // set mounts's owner plugin ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ auto device = m_owners[id].devices.find(add[i]->path); ++ if (device != m_owners[id].devices.end()) { ++ ERROR("plugins %s and %s both tried to set devices: %s", plugin.c_str(), device->second.c_str(), add[i]->path); ++ return false; ++ } ++ m_owners[id].devices[add[i]->path] = plugin; ++ } ++ m_reply.adjust->linux->devices[m_reply.adjust->linux->devices_len] = copy_nri_device(add[i]); ++ if (m_reply.adjust->linux->devices[m_reply.adjust->linux->devices_len] == nullptr) { ++ ERROR("Failed to copy add nri devices to reply adjust"); ++ return false; ++ } ++ m_reply.adjust->linux->devices_len++; ++ } ++ ++ return true; ++} ++ ++auto pluginResult::AdjustResources(nri_linux_resources *resources, const std::string &plugin) -> bool ++{ ++ if (resources == nullptr) { ++ return true; ++ } ++ ++ if (m_reply.adjust->linux->resources == nullptr) { ++ m_reply.adjust->linux->resources = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); ++ if (m_reply.adjust->linux->resources == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ } ++ ++ std::string id = m_conId.c_str(); ++ nri_linux_resources *reply = m_reply.adjust->linux->resources; ++ ++ return ClaimAndCopyResources(resources, id, plugin, reply); ++} ++ ++bool pluginResult::ClaimAndCopyResources(nri_linux_resources *src, std::string &id, const std::string &plugin, ++ nri_linux_resources *dest) ++{ ++ size_t i; ++ if (src->memory != nullptr) { ++ if (src->memory->limit != nullptr) { ++ auto memLimit = m_owners[id].memLimit; ++ if (!memLimit.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's memory limit", plugin.c_str(), memLimit.c_str()); ++ return false; ++ } ++ m_owners[id].memLimit = plugin; ++ *dest->memory->limit = *src->memory->limit; ++ } ++ ++ if (src->memory->reservation != nullptr) { ++ auto memReservation = m_owners[id].memReservation; ++ if (!memReservation.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's memory reservation", plugin.c_str(), ++ memReservation.c_str()); ++ return false; ++ } ++ m_owners[id].memReservation = plugin; ++ *dest->memory->reservation = *src->memory->reservation; ++ } ++ ++ if (src->memory->swap != nullptr) { ++ auto memSwapLimit = m_owners[id].memSwapLimit; ++ if (!memSwapLimit.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's memory swap limit", plugin.c_str(), ++ memSwapLimit.c_str()); ++ return false; ++ } ++ m_owners[id].memSwapLimit = plugin; ++ *dest->memory->swap = *src->memory->swap; ++ } ++ ++ if (src->memory->kernel != nullptr) { ++ auto memKernelLimit = m_owners[id].memKernelLimit; ++ if (!memKernelLimit.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's memory kernel limit", plugin.c_str(), ++ memKernelLimit.c_str()); ++ return false; ++ } ++ m_owners[id].memKernelLimit = plugin; ++ *dest->memory->kernel = *src->memory->kernel; ++ } ++ ++ if (src->memory->kernel_tcp != nullptr) { ++ auto memTCPLimit = m_owners[id].memTCPLimit; ++ if (!memTCPLimit.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's memory tcp limit", plugin.c_str(), ++ memTCPLimit.c_str()); ++ return false; ++ } ++ m_owners[id].memTCPLimit = plugin; ++ *dest->memory->kernel_tcp = *src->memory->kernel_tcp; ++ } ++ ++ if (src->memory->swappiness != nullptr) { ++ auto memSwappiness = m_owners[id].memSwappiness; ++ if (!memSwappiness.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's memory swappiness", plugin.c_str(), ++ memSwappiness.c_str()); ++ return false; ++ } ++ m_owners[id].memSwappiness = plugin; ++ *dest->memory->swappiness = *src->memory->swappiness; ++ } ++ ++ if (src->memory->disable_oom_killer != nullptr) { ++ auto memDisableOomKiller = m_owners[id].memDisableOomKiller; ++ if (!memDisableOomKiller.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's memory disable_oom_killer", plugin.c_str(), ++ memDisableOomKiller.c_str()); ++ return false; ++ } ++ m_owners[id].memDisableOomKiller = plugin; ++ *dest->memory->disable_oom_killer = *src->memory->disable_oom_killer; ++ } ++ ++ if (src->memory->use_hierarchy != nullptr) { ++ auto memUseHierarchy = m_owners[id].memUseHierarchy; ++ if (!memUseHierarchy.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's memory use_hierarchy", plugin.c_str(), ++ memUseHierarchy.c_str()); ++ return false; ++ } ++ m_owners[id].memUseHierarchy = plugin; ++ *dest->memory->use_hierarchy = *src->memory->use_hierarchy; ++ } ++ } ++ ++ if (src->cpu != nullptr) { ++ if (src->cpu->shares != nullptr) { ++ auto cpuShares = m_owners[id].cpuShares; ++ if (!cpuShares.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's cpu shares", plugin.c_str(), cpuShares.c_str()); ++ return false; ++ } ++ m_owners[id].cpuShares = plugin; ++ *dest->cpu->shares = *src->cpu->shares; ++ } ++ ++ if (src->cpu->quota != nullptr) { ++ auto cpuQuota = m_owners[id].cpuQuota; ++ if (!cpuQuota.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's cpu quota", plugin.c_str(), cpuQuota.c_str()); ++ return false; ++ } ++ m_owners[id].cpuQuota = plugin; ++ *dest->cpu->quota = *src->cpu->quota; ++ } ++ ++ if (src->cpu->period != nullptr) { ++ auto cpuPeriod = m_owners[id].cpuPeriod; ++ if (!cpuPeriod.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's cpu period", plugin.c_str(), cpuPeriod.c_str()); ++ return false; ++ } ++ m_owners[id].cpuPeriod = plugin; ++ *dest->cpu->period = *src->cpu->period; ++ } ++ ++ if (src->cpu->realtime_runtime != nullptr) { ++ auto cpuRealtimePeriod = m_owners[id].cpuRealtimePeriod; ++ if (!cpuRealtimePeriod.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's cpu realtime_runtime", plugin.c_str(), ++ cpuRealtimePeriod.c_str()); ++ return false; ++ } ++ m_owners[id].cpuRealtimePeriod = plugin; ++ *dest->cpu->realtime_runtime = *src->cpu->realtime_runtime; ++ } ++ ++ if (src->cpu->realtime_period != nullptr) { ++ auto cpuRealtimePeriod = m_owners[id].cpuRealtimePeriod; ++ if (!cpuRealtimePeriod.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's cpu realtime_period", plugin.c_str(), ++ cpuRealtimePeriod.c_str()); ++ return false; ++ } ++ m_owners[id].cpuRealtimePeriod = plugin; ++ *dest->cpu->realtime_period = *src->cpu->realtime_period; ++ } ++ ++ if (src->cpu->cpus != nullptr) { ++ auto cpusetCpus = m_owners[id].cpusetCpus; ++ if (!cpusetCpus.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's cpu cpus", plugin.c_str(), cpusetCpus.c_str()); ++ return false; ++ } ++ m_owners[id].cpusetCpus = plugin; ++ *dest->cpu->cpus = *src->cpu->cpus; ++ } ++ ++ if (src->cpu->mems != nullptr) { ++ auto cpusetMems = m_owners[id].cpusetMems; ++ if (!cpusetMems.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's cpu mems", plugin.c_str(), cpusetMems.c_str()); ++ return false; ++ } ++ m_owners[id].cpusetMems = plugin; ++ *dest->cpu->mems = *src->cpu->mems; ++ } ++ } ++ ++ for (i = 0; i < src->hugepage_limits_len; i++) { ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ auto find = m_owners[id].hugepageLimits.find(src->hugepage_limits[i]->page_size); ++ if (find != m_owners[id].hugepageLimits.end()) { ++ ERROR("plugins %s and %s both tried to set hugepageLimits: %s", plugin.c_str(), find->second.c_str(), ++ src->hugepage_limits[i]->page_size); ++ return false; ++ } ++ m_owners[id].hugepageLimits[src->hugepage_limits[i]->page_size] = plugin; ++ } ++ } ++ ++ if (src->unified->len != 0) { ++ google::protobuf::Map<std::string, std::string> mapAnno; ++ Transform::JsonMapToProtobufMapForString(src->unified, mapAnno); ++ for (const auto &iter : mapAnno) { ++ std::string key = iter.first; ++ std::string value = iter.second; ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ auto anno = m_owners[id].unified.find(key); ++ if (anno != m_owners[id].unified.end()) { ++ ERROR("plugins %s and %s both tried to set unified: %s", plugin.c_str(), anno->second.c_str(), ++ key.c_str()); ++ return false; ++ } ++ m_owners[id].unified[key] = plugin; ++ } ++ // add pair to m_reply.adjust ++ append_json_map_string_string(dest->unified, key.c_str(), value.c_str()); ++ } ++ } ++ ++ if (src->blockio_class != nullptr) { ++ auto blockioClass = m_owners[id].blockioClass; ++ if (!blockioClass.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's blockio_class", plugin.c_str(), blockioClass.c_str()); ++ return false; ++ } ++ m_owners[id].blockioClass = plugin; ++ dest->blockio_class = util_strdup_s(src->blockio_class); ++ } ++ ++ if (src->rdt_class != nullptr) { ++ auto rdtClass = m_owners[id].rdtClass; ++ if (!rdtClass.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's rdt_class", plugin.c_str(), rdtClass.c_str()); ++ return false; ++ } ++ m_owners[id].rdtClass = plugin; ++ dest->rdt_class = util_strdup_s(src->rdt_class); ++ } ++ return true; ++} ++ ++auto pluginResult::AdjustCgroupsPath(char *path, const std::string &plugin) -> bool ++{ ++ if (path == nullptr || strcmp(path, "") == 0) { ++ return true; ++ } ++ ++ std::string id = m_conId.c_str(); ++ ++ auto cgroupsPath = m_owners[id].cgroupsPath; ++ if (!cgroupsPath.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's cgroups path", plugin.c_str(), cgroupsPath.c_str()); ++ return false; ++ } ++ m_owners[id].cgroupsPath = plugin; ++ m_reply.adjust->linux->cgroups_path = util_strdup_s(path); ++ ++ return true; ++} ++ ++auto pluginResult::AdjustRlimits(nri_posix_rlimit **rlimits, size_t rlimits_len, const std::string &plugin) -> bool ++{ ++ if (rlimits_len == 0) { ++ return true; ++ } ++ ++ size_t i; ++ std::string id = m_conId.c_str(); ++ ++ size_t oldSize, newSize; ++ oldSize = m_reply.adjust->rlimits_len * sizeof(nri_posix_rlimit *); ++ newSize = oldSize + rlimits_len * sizeof(nri_posix_rlimit *); ++ int ret = util_mem_realloc((void **)(&m_reply.adjust->rlimits), newSize, m_reply.adjust->rlimits, oldSize); ++ if (ret != 0) { ++ ERROR("Failed to realloc and assign nri rlimits array"); ++ return false; ++ } ++ ++ for (i = 0; i < rlimits_len; i++) { ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ auto find = m_owners[id].rlimits.find(rlimits[i]->type); ++ if (find != m_owners[id].rlimits.end()) { ++ ERROR("plugins %s and %s both tried to set rlimits type: %s", plugin.c_str(), find->second.c_str(), rlimits[i]->type); ++ return false; ++ } ++ m_owners[id].rlimits[rlimits[i]->type] = plugin; ++ } ++ m_reply.adjust->rlimits[m_reply.adjust->rlimits_len] = copy_nri_posix_rlimit(rlimits[i]); ++ if (m_reply.adjust->rlimits[m_reply.adjust->rlimits_len] == nullptr) { ++ ERROR("Failed to copy add nri rlimits to reply adjust"); ++ return false; ++ } ++ m_reply.adjust->rlimits_len++; ++ } ++ return true; ++} ++ ++auto pluginResult::Update(nri_container_update **updates, size_t update_len, const std::string &plugin) -> bool ++{ ++ if (update_len == 0) { ++ return true; ++ } ++ ++ size_t i; ++ ++ for (i = 0; i < update_len; i++) { ++ nri_container_update *reply; ++ if (!GetContainerUpdate(updates[i], plugin, &reply)) { ++ ERROR("Failed to get container update in plugin result"); ++ return false; ++ } ++ ++ if (!UpdateResources(reply, updates[i], plugin)) { ++ ERROR("Failed to update container resources in plugin result"); ++ return false; ++ } ++ } ++ ++ return true; ++} ++ ++auto pluginResult::GetContainerUpdate(nri_container_update *update, const std::string &plugin, ++ nri_container_update **out) -> bool ++{ ++ if (update == nullptr || out == nullptr || plugin.empty()) { ++ ERROR("Empyt input args"); ++ return false; ++ } ++ ++ auto id = update->container_id; ++ ++ if (std::string(id) == m_conId) { ++ ERROR("Plugin %s asked update of %s during creation", plugin.c_str(), id); ++ return false; ++ } ++ ++ auto find = m_updates.find(id); ++ if (find != m_updates.end()) { ++ *out = m_updates[id]; ++ (*out)->ignore_failure = (*out)->ignore_failure && update->ignore_failure; ++ return true; ++ } ++ ++ *out = init_nri_container_update(id, update->ignore_failure); ++ if (*out == nullptr) { ++ ERROR("Failed to init nri container update"); ++ return false; ++ } ++ ++ m_updates[id] = *out; ++ ++ // for update requests delay appending the requested container (in the response getter) ++ if (m_conId != id) { ++ m_reply.update.push_back(*out); ++ } ++ ++ return true; ++} ++ ++auto pluginResult::UpdateResources(nri_container_update *reply, nri_container_update *u, ++ const std::string &plugin) -> bool ++{ ++ if (u->linux == nullptr || u->linux->resources == nullptr) { ++ return true; ++ } ++ ++ std::string id = u->container_id; ++ nri_linux_resources *resources; ++ ++ if (m_conId == id) { ++ resources = copy_nri_linux_resources(m_update_req); ++ if (resources == nullptr) { ++ ERROR("Failed to copy request's nri linux resources"); ++ return false; ++ } ++ } else { ++ resources = copy_nri_linux_resources(reply->linux->resources); ++ if (resources == nullptr) { ++ ERROR("Failed to copy reply's nri linux resources"); ++ return false; ++ } ++ } ++ ++ if (!ClaimAndCopyResources(u->linux->resources, id, plugin, resources)) { ++ ERROR("Failed to claim and copy resources in plugin result"); ++ return false; ++ } ++ ++ // update reply from copy on success ++ reply->linux->resources = copy_nri_linux_resources(resources); ++ if (reply->linux->resources == nullptr) { ++ ERROR("Failed to copy resources's nri linux resources to reply"); ++ return false; ++ } ++ ++ return true; ++} +\ No newline at end of file +diff --git a/src/daemon/nri/nri_result.h b/src/daemon/nri/nri_result.h +index f2896ea0..4f26385a 100644 +--- a/src/daemon/nri/nri_result.h ++++ b/src/daemon/nri/nri_result.h +@@ -63,7 +63,7 @@ struct owners { + + struct resultReply { + nri_container_adjustment* adjust; +- std::vector<nri_container_update*> update; ++ std::vector<nri_container_update *> update; + }; + + using resultOwners = std::map<std::string, owners>; +@@ -71,16 +71,19 @@ using resultOwners = std::map<std::string, owners>; + class pluginResult { + public: + pluginResult() = default; ++ pluginResult(std::string conId); + +- ~pluginResult() = default; ++ virtual ~pluginResult(); + ++ auto Init() -> bool; + auto InitByConId(std::string conId) -> bool; + auto InitByUpdateReq(nri_update_container_request *req) -> bool; + + auto GetReplyUpdate() -> std::vector<nri_container_update *>; +- auto GetReplyAdjust() -> nri_container_adjustment *; ++ auto MoveReplyAdjust() -> nri_container_adjustment *; ++ auto GetReplyResources(const std::string &id) -> const nri_linux_resources *; + +- auto Apply(int32_t event, nri_container_adjustment *adjust, nri_container_update **update, size_t update_len, ++ auto Apply(int32_t event, const nri_container_adjustment *adjust, nri_container_update **update, size_t update_len, + const std::string &plugin) -> bool; + auto Update(nri_container_update **updates, size_t update_len, const std::string &plugin) -> bool; + +@@ -90,12 +93,12 @@ private: + + auto InitReply(void) -> bool; + +- auto Adjust(nri_container_adjustment *adjust, const std::string &plugin) -> bool; ++ auto Adjust(const nri_container_adjustment *adjust, const std::string &plugin) -> bool; + + auto AdjustAnnotations(json_map_string_string *annos, const std::string &plugin) -> bool; + auto AdjustMounts(nri_mount **mounts, size_t mounts_size, const std::string &plugin) -> bool; + auto AdjustEnv(nri_key_value **envs, size_t envs_size, const std::string &plugin) -> bool; +- auto AdjustHooks(nri_hooks *hooks, const std::string &plugin) -> bool; ++ auto AdjustHooks(const nri_hooks *hooks, const std::string &plugin) -> bool; + auto AdjustDevices(nri_linux_device **devices, size_t devices_size, const std::string &plugin) -> bool; + auto AdjustResources(nri_linux_resources *resources, const std::string &plugin) -> bool; + bool ClaimAndCopyResources(nri_linux_resources *src, std::string &id, const std::string &plugin, +@@ -107,7 +110,9 @@ private: + std::string m_conId; + nri_linux_resources *m_update_req; + resultReply m_reply; ++ // update plugin -> update context + std::map<std::string, nri_container_update *> m_updates; ++ // adjust plugin -> adjust context + resultOwners m_owners; + }; + +diff --git a/src/daemon/nri/plugin.cc b/src/daemon/nri/plugin.cc +new file mode 100644 +index 00000000..904677c4 +--- /dev/null ++++ b/src/daemon/nri/plugin.cc +@@ -0,0 +1,417 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-03-15 ++ * Description: provide plugin class definition ++ *********************************************************************************/ ++ ++#include "plugin.h" ++ ++#include <string> ++#include <mutex> ++#include <google/protobuf/map.h> ++#include <iostream> ++#include <thread> ++#include <chrono> ++#include <sys/socket.h> ++#include <fcntl.h> ++ ++#include <nri_plugin.h> ++ ++#include <isula_libutils/log.h> ++#include <isula_libutils/nri_create_container_request.h> ++#include <isula_libutils/nri_create_container_response.h> ++#include <isula_libutils/nri_configure_request.h> ++#include <isula_libutils/nri_configure_response.h> ++#include <isula_libutils/nri_state_change_event.h> ++#include <isula_libutils/nri_stop_container_request.h> ++#include <isula_libutils/nri_stop_container_response.h> ++#include <isula_libutils/nri_synchronize_request.h> ++#include <isula_libutils/nri_synchronize_response.h> ++#include <isula_libutils/nri_update_container_request.h> ++#include <isula_libutils/nri_update_container_response.h> ++ ++#include "utils.h" ++#include "cstruct_wrapper.h" ++ ++// same as containerd ++std::string DefaultNRIVersion = "2.0.0-beta.2+unknown"; ++// same as containerd ++std::string DefaultNRIRuntimeName = "v2"; ++// defualt timeout for wait: 2s ++const int64_t DefaultWaitTimeout = 2000; ++const uint64_t SECOND_TO_NANOS = 1000000000; ++ ++// init client conn ++NRIPlugin::NRIPlugin(std::string &idx, std::string &name, std::string &config) ++{ ++ m_idx = idx; ++ m_name = name; ++ m_config = config; ++ m_closed = false; ++ m_external = false; ++ m_pid = -1; ++} ++ ++NRIPlugin::NRIPlugin(int fd, std::string &name) ++{ ++ m_sockFds.push_back(fd); ++ m_name = name; ++ m_closed = false; ++ m_external = true; ++ m_pid = -1; ++} ++ ++// wait for plugin to register, then configure it. ++auto NRIPlugin::Start(int64_t registry_timeout, int64_t request_timeout) -> bool ++{ ++ Errors error; ++ ++ // todo: what if timeout is 0 or other invalid value? ++ ++ if (!Connect(request_timeout)) { ++ ERROR("Failed to connect nri plugin %s", m_name.c_str()); ++ return false; ++ } ++ ++ if (!WaitForReady(registry_timeout)) { ++ ERROR("Failed to wait plugin %s ready with timeout %ld", m_name.c_str(), registry_timeout); ++ return false; ++ } ++ ++ if (!Configure(error)) { ++ ERROR("Failed to configure nri plugin %s", m_name.c_str()); ++ return false; ++ } ++ ++ return true; ++} ++ ++auto NRIPlugin::shutdown() -> void ++{ ++ if (!Close()) { ++ ERROR("Failed to close plugin %s", m_name.c_str()); ++ } ++ ++ if (!Stop()) { ++ ERROR("Failed to stop plugin %s", m_name.c_str()); ++ } ++} ++ ++// create client connect ++auto NRIPlugin::Connect(int64_t timeout) -> bool ++{ ++ if (m_name.empty()) { ++ ERROR("Empty nri plugin name"); ++ return false; ++ } ++ ++ if (nri_plugin_connect(m_name.c_str(), m_sockFds[0], timeout * SECOND_TO_NANOS) != 0) { ++ ERROR("Failed to create a new client for plugin %s", m_name.c_str()); ++ return false; ++ } ++ ++ return true; ++} ++ ++// close a plugin shutting down its multiplexed ttrpc connections. ++auto NRIPlugin::Close() -> bool ++{ ++ if (IsClose()) { ++ return true; ++ } ++ ++ if (nri_plugin_disconnect(m_name.c_str()) != 0) { ++ ERROR("Failed to close plugin %s", m_name.c_str()); ++ return false; ++ } ++ ++ SetClose(); ++ return true; ++} ++ ++// stop a plugin (if it was launched by us) ++auto NRIPlugin::Stop() -> bool ++{ ++ if (m_external) { ++ return true; ++ } ++ ++ if (m_pid <= 0) { ++ WARN("Invalid pid %d", m_pid); ++ return false; ++ } ++ ++ int nret = kill(m_pid, SIGKILL); ++ if (nret < 0 && errno != ESRCH) { ++ SYSWARN("Can not kill process (pid=%d) with SIGKILL", m_pid); ++ return false; ++ } ++ ++ if (util_waitpid_with_timeout(m_pid, DefaultWaitTimeout, NULL) != 0) { ++ WARN("Failed to wait for plugin %s to exit", m_name.c_str()); ++ return false; ++ } ++ return true; ++} ++ ++// Name returns a string indentication for the plugin. ++auto NRIPlugin::GetName() -> const std::string & ++{ ++ return m_name; ++} ++ ++auto NRIPlugin::GetIndex() -> const std::string & ++{ ++ return m_idx; ++} ++ ++auto NRIPlugin::GetPeerSockFd() -> uint32_t ++{ ++ return m_sockFds[1]; ++} ++ ++auto NRIPlugin::GetQualifiedName() -> std::string ++{ ++ return m_idx + "-" + m_name; ++} ++ ++void NRIPlugin::SetReady(void) ++{ ++ std::unique_lock<std::mutex> lock(m_readyMutex); ++ m_ready = true; ++ m_condition.notify_one(); ++} ++ ++void NRIPlugin::SetPid(int pid) ++{ ++ m_pid = pid; ++} ++ ++auto NRIPlugin::CreateSocketPair() -> bool ++{ ++ int fds[2]; ++ ++ if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) == -1) { ++ ERROR("Failed to create socketpair"); ++ return false; ++ } ++ ++ m_sockFds.push_back(fds[0]); ++ m_sockFds.push_back(fds[1]); ++ return true; ++} ++ ++auto NRIPlugin::Configure(Errors &error) -> bool ++{ ++ auto req = makeUniquePtrCStructWrapper<nri_configure_request>(free_nri_configure_request); ++ if (req == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ req->get()->config = isula_strdup_s(m_config.c_str()); ++ req->get()->runtime_name = isula_strdup_s(NRIRruntime.c_str()); ++ req->get()->runtime_version = isula_strdup_s(NRIVersion.c_str()); ++ ++ nri_configure_response *resp = nullptr; ++ if (nri_plugin_configure(m_name.c_str(), req->get(), &resp) != 0) { ++ ERROR("Failed to configure plugin %s", m_name.c_str()); ++ return false; ++ } ++ ++ auto resp_wrapper = makeUniquePtrCStructWrapper<nri_configure_response>(resp, free_nri_configure_response); ++ if (resp_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ EventMask events = resp_wrapper->get()->events; ++ if (events != 0) { ++ EventMask extra = events & ~ValidEvents; ++ if (extra != 0) { ++ ERROR("Invalid plugin events: %d", extra); ++ return false; ++ } ++ } else { ++ events = ValidEvents; ++ } ++ ++ m_events = events; ++ return true; ++} ++ ++auto NRIPlugin::Synchronize(std::vector<nri_pod_sandbox *> &pods, std::vector<nri_container *> &containers, ++ nri_container_update ***update, size_t update_len, Errors &error) -> bool ++{ ++ size_t i; ++ ++ auto req = makeUniquePtrCStructWrapper<nri_synchronize_request>(free_nri_synchronize_request); ++ if (req == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ if (pods.size() != 0) { ++ req->get()->pods = (nri_pod_sandbox **)util_common_calloc_s(pods.size() * sizeof(nri_pod_sandbox *)); ++ if (req->get()->pods == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ for (i = 0; i < pods.size(); i++) { ++ req->get()->pods[i] = pods[i]; ++ req->get()->pods_len++; ++ } ++ } ++ ++ if (containers.size() != 0) { ++ req->get()->containers = (nri_container **)util_common_calloc_s(containers.size() * sizeof(nri_container *)); ++ if (req->get()->containers == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ for (i = 0; i < containers.size(); i++) { ++ req->get()->containers[i] = containers[i]; ++ req->get()->containers_len++; ++ } ++ } ++ ++ nri_synchronize_response *resp = nullptr; ++ if (nri_plugin_synchronize(m_name.c_str(), req->get(), &resp) != 0) { ++ ERROR("Failed to synchronize plugin %s", m_name.c_str()); ++ return false; ++ } ++ ++ auto resp_wrapper = makeUniquePtrCStructWrapper<nri_synchronize_response>(resp, free_nri_synchronize_response); ++ if (resp_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ *update = resp->update; ++ resp->update = nullptr; ++ update_len = resp->update_len; ++ resp->update_len = 0; ++ return true; ++} ++ ++auto NRIPlugin::CreateContainer(nri_create_container_request *req, nri_create_container_response **resp, ++ Errors &error) -> bool ++{ ++ if (req == nullptr) { ++ ERROR("Invalid input"); ++ return false; ++ } ++ ++ if (IsSetEvent(CREATE_CONTAINER) == false) { ++ return true; ++ } ++ ++ if (nri_plugin_create_container(m_name.c_str(), req, resp) != 0) { ++ ERROR("Failed to create container by plugin %s", m_name.c_str()); ++ return false; ++ } ++ ++ return true; ++} ++ ++auto NRIPlugin::UpdateContainer(nri_update_container_request *req, nri_update_container_response **resp, ++ Errors &error) -> bool ++{ ++ if (req == nullptr) { ++ ERROR("Invalid input"); ++ return false; ++ } ++ ++ if (!IsSetEvent(UPDATE_CONTAINER)) { ++ return true; ++ } ++ ++ if (nri_plugin_update_container(m_name.c_str(), req, resp) != 0) { ++ ERROR("Failed to update container by plugin %s", m_name.c_str()); ++ return false; ++ } ++ return true; ++} ++ ++auto NRIPlugin::StopContainer(nri_stop_container_request *req, nri_stop_container_response **resp, ++ Errors &error) -> bool ++{ ++ if (req == nullptr) { ++ ERROR("Invalid input"); ++ return false; ++ } ++ ++ if (!IsSetEvent(STOP_CONTAINER)) { ++ return true; ++ } ++ ++ if (nri_plugin_stop_container(m_name.c_str(), req, resp) != 0) { ++ ERROR("Failed to stop container by plugin %s", m_name.c_str()); ++ return false; ++ } ++ return true; ++} ++ ++// do nothing with event ++auto NRIPlugin::StateChange(nri_state_change_event *evt, Errors &error) -> bool ++{ ++ if (evt == nullptr) { ++ ERROR("Invalid input"); ++ return false; ++ } ++ ++ if (!IsSetEvent(evt->event)) { ++ return true; ++ } ++ ++ if (nri_plugin_state_change(m_name.c_str(), evt) != 0) { ++ ERROR("Failed to state change by plugin %s", m_name.c_str()); ++ return false; ++ } ++ return true; ++} ++ ++auto NRIPlugin::WaitForReady(int64_t timeout) -> bool ++{ ++ auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(timeout * 1000); ++ std::unique_lock<std::mutex> readyMutex(m_readyMutex); ++ ++ if (timeout == 0) { ++ m_condition.wait(readyMutex); ++ return true; ++ } ++ ++ if (m_condition.wait_until(readyMutex, deadline) == std::cv_status::timeout) { ++ return false; ++ } ++ ++ return true; ++} ++ ++auto NRIPlugin::IsSetEvent(EventMask e) -> bool ++{ ++ return (m_events & (1 << (e - 1))) != 0; ++} ++ ++auto NRIPlugin::IsClose() -> bool ++{ ++ ReadGuard<RWMutex> lock(m_mutex); ++ return m_closed; ++} ++ ++void NRIPlugin::SetClose() ++{ ++ WriteGuard<RWMutex> lock(m_mutex); ++ m_closed = true; ++} +\ No newline at end of file +diff --git a/src/daemon/nri/plugin.h b/src/daemon/nri/plugin.h +index f60a9b3d..ed298be6 100644 +--- a/src/daemon/nri/plugin.h ++++ b/src/daemon/nri/plugin.h +@@ -38,17 +38,14 @@ public: + virtual ~NRIPlugin() = default; + // wait for plugin to register, then configure it. + auto Start(int64_t registry_timeout, int64_t request_timeout) -> bool; +- // close a plugin shutting down its multiplexed ttrpc connections. +- auto Close(void) -> bool; +- // stop a plugin (if it was launched by us) +- auto Stop(void) -> bool; ++ ++ auto shutdown() -> void; + + // Name returns a string indentication for the plugin. + auto GetName(void) -> const std::string &; + auto GetIndex(void) -> const std::string &; + auto GetPeerSockFd(void) -> uint32_t; + auto GetQualifiedName(void) -> std::string; +- + void SetReady(void); + void SetPid(int pid); + +@@ -58,8 +55,8 @@ public: + + auto Configure(Errors &error) -> bool; + // Only called in external plugin scenario +- auto Synchronize(std::vector<std::unique_ptr<nri_pod_sandbox>> pods, +- std::vector<std::unique_ptr<nri_container>> &containers, nri_container_update ***update, size_t update_len, ++ auto Synchronize(std::vector<nri_pod_sandbox *> &pods, ++ std::vector<nri_container *> &containers, nri_container_update ***update, size_t update_len, + Errors &error) -> bool; + auto CreateContainer(nri_create_container_request *req, nri_create_container_response **resp, Errors &error) -> bool; + auto UpdateContainer(nri_update_container_request *req, nri_update_container_response **resp, Errors &error) -> bool; +@@ -72,6 +69,13 @@ private: + auto WaitForReady(int64_t timeout) -> bool; + auto IsSetEvent(EventMask e) -> bool; + ++ // close a plugin shutting down its multiplexed ttrpc connections. ++ auto Close(void) -> bool; ++ // stop a plugin (if it was launched by us) ++ auto Stop(void) -> bool; ++ ++ void SetClose(void); ++ + private: + RWMutex m_mutex; + bool m_external; +@@ -83,7 +87,7 @@ private: + std::vector<uint32_t> m_sockFds; + std::string m_localFileName; + std::string m_peerFileName; +- // TODO:zhontao monitor? ++ // TODO: plugin monitor? + bool m_closed; + std::mutex m_readyMutex; + bool m_ready; +diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c +index f81a9141..69f6dbf0 100644 +--- a/src/utils/cutils/utils.c ++++ b/src/utils/cutils/utils.c +@@ -1705,4 +1705,4 @@ void set_child_process_pdeathsig(void) + if (prctl(PR_SET_PDEATHSIG, SIGKILL) < 0) { + SYSERROR("Failed to set child process pdeathsig"); + } +-} +\ No newline at end of file ++} +-- +2.25.1 + diff --git a/0123-code-improve-for-codecheck.patch b/0123-code-improve-for-codecheck.patch new file mode 100644 index 0000000..1801dff --- /dev/null +++ b/0123-code-improve-for-codecheck.patch @@ -0,0 +1,287 @@ +From 8e442712354a9d4f766d1f90b018fd1246cb9ef2 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Wed, 4 Sep 2024 16:26:59 +1400 +Subject: [PATCH 123/149] code improve for codecheck + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/common/sysinfo.h | 8 ++++---- + src/daemon/entry/cri/network_plugin.cc | 2 +- + src/daemon/mailbox/message_queue.h | 8 ++++---- + src/daemon/mailbox/message_subscriber.h | 8 ++++++++ + src/daemon/modules/runtime/shim/shim_rt_monitor.cc | 2 ++ + src/daemon/nri/nri_adaption.h | 1 - + src/daemon/sandbox/sandbox_manager.cc | 5 ++--- + src/utils/cutils/blocking_queue.h | 2 +- + src/utils/cutils/utils_aes.h | 2 +- + src/utils/cutils/utils_cap.h | 7 +++---- + src/utils/cutils/utils_fs.h | 2 +- + src/utils/cutils/utils_network.c | 2 ++ + src/utils/cutils/utils_string.h | 3 +-- + src/utils/tar/util_archive.h | 5 ++--- + src/utils/tar/util_gzip.h | 2 +- + 15 files changed, 33 insertions(+), 26 deletions(-) + +diff --git a/src/daemon/common/sysinfo.h b/src/daemon/common/sysinfo.h +index 6142487b..e6bb7f95 100644 +--- a/src/daemon/common/sysinfo.h ++++ b/src/daemon/common/sysinfo.h +@@ -15,16 +15,16 @@ + #ifndef DAEMON_COMMON_SYSINFO_H + #define DAEMON_COMMON_SYSINFO_H + +-#ifdef __cplusplus +-extern "C" { +-#endif +- + #include <stdbool.h> + #include <stdint.h> + #include <isula_libutils/auto_cleanup.h> + + #include "cgroup.h" + ++#ifdef __cplusplus ++extern "C" { ++#endif ++ + typedef struct { + // Number of processors currently online (i.e., available). + int ncpus; +diff --git a/src/daemon/entry/cri/network_plugin.cc b/src/daemon/entry/cri/network_plugin.cc +index f8f9c7e6..439d0224 100644 +--- a/src/daemon/entry/cri/network_plugin.cc ++++ b/src/daemon/entry/cri/network_plugin.cc +@@ -198,7 +198,7 @@ void InitNetworkPlugin(std::vector<std::shared_ptr<NetworkPlugin>> *plugins, std + + if (networkPluginName.empty()) { + DEBUG("network plugin name empty"); +- *result = std::shared_ptr<NetworkPlugin>(new (std::nothrow) NoopNetworkPlugin); ++ *result = std::make_shared<NoopNetworkPlugin>(); + if (*result == nullptr) { + ERROR("Out of memory"); + return; +diff --git a/src/daemon/mailbox/message_queue.h b/src/daemon/mailbox/message_queue.h +index 7905840f..c9bbc9e2 100644 +--- a/src/daemon/mailbox/message_queue.h ++++ b/src/daemon/mailbox/message_queue.h +@@ -16,10 +16,6 @@ + #ifndef DAEMON_MESSAGE_MESSAGE_QUEUE_H + #define DAEMON_MESSAGE_MESSAGE_QUEUE_H + +-#ifdef __cplusplus +-extern "C" { +-#endif +- + #include <pthread.h> + + #include "blocking_queue.h" +@@ -27,6 +23,10 @@ extern "C" { + #include "map.h" + #include "message_subscriber.h" + ++#ifdef __cplusplus ++extern "C" { ++#endif ++ + typedef struct message_queue { + blocking_queue *messages; + +diff --git a/src/daemon/mailbox/message_subscriber.h b/src/daemon/mailbox/message_subscriber.h +index de4574d9..2987b60d 100644 +--- a/src/daemon/mailbox/message_subscriber.h ++++ b/src/daemon/mailbox/message_subscriber.h +@@ -19,6 +19,10 @@ + #include "blocking_queue.h" + #include "mailbox_message.h" + ++#ifdef __cplusplus ++extern "C" { ++#endif ++ + typedef struct { + blocking_queue *queue; + } message_subscriber; +@@ -38,4 +42,8 @@ define_auto_cleanup_callback(message_subscriber_destroy, message_subscriber); + // define auto free macro for blocking queue + #define __isula_auto_subscriber auto_cleanup_tag(message_subscriber_destroy) + ++#ifdef __cplusplus ++} ++#endif ++ + #endif +diff --git a/src/daemon/modules/runtime/shim/shim_rt_monitor.cc b/src/daemon/modules/runtime/shim/shim_rt_monitor.cc +index 2547a206..97f5cd68 100644 +--- a/src/daemon/modules/runtime/shim/shim_rt_monitor.cc ++++ b/src/daemon/modules/runtime/shim/shim_rt_monitor.cc +@@ -30,6 +30,8 @@ + #include "utils.h" + #include "error.h" + ++// The shim v2 header file needs to be modified to ++// use extern "C" to wrap external functions. + extern "C" { + #include <shim_v2.h> + } +diff --git a/src/daemon/nri/nri_adaption.h b/src/daemon/nri/nri_adaption.h +index 27a6d93e..6bd41941 100644 +--- a/src/daemon/nri/nri_adaption.h ++++ b/src/daemon/nri/nri_adaption.h +@@ -16,7 +16,6 @@ + #ifndef DAEMON_NRI_PLUGIN_NRI_ADAPTION_H + #define DAEMON_NRI_PLUGIN_NRI_ADAPTION_H + +-// #include "read_write_lock.h" + #include <isula_libutils/nri_update_containers_request.h> + #include <isula_libutils/nri_update_containers_response.h> + +diff --git a/src/daemon/sandbox/sandbox_manager.cc b/src/daemon/sandbox/sandbox_manager.cc +index cee444f4..4159993f 100644 +--- a/src/daemon/sandbox/sandbox_manager.cc ++++ b/src/daemon/sandbox/sandbox_manager.cc +@@ -109,8 +109,7 @@ auto SandboxManager::CreateSandbox(const std::string &name, RuntimeInfo &info, s + return nullptr; + } + +- sandbox = std::shared_ptr<Sandbox>(new Sandbox(id, m_rootdir, m_statedir, name, info, netMode, netNsPath, +- sandboxConfig, image)); ++ sandbox = std::make_shared<Sandbox>(id, m_rootdir, m_statedir, name, info, netMode, netNsPath, sandboxConfig, image); + if (sandbox == nullptr) { + ERROR("Failed to malloc for sandbox: %s", name.c_str()); + error.Errorf("Failed to malloc for sandbox: %s", name.c_str()); +@@ -452,7 +451,7 @@ auto SandboxManager::LoadSandbox(std::string &id) -> std::shared_ptr<Sandbox> + return nullptr; + } + +- sandbox = std::shared_ptr<Sandbox>(new Sandbox(id, m_rootdir, m_statedir)); ++ sandbox = std::make_shared<Sandbox>(id, m_rootdir, m_statedir); + if (sandbox == nullptr) { + ERROR("Failed to malloc for sandboxes: %s", id.c_str()); + return nullptr; +diff --git a/src/utils/cutils/blocking_queue.h b/src/utils/cutils/blocking_queue.h +index 257779c3..e6931501 100644 +--- a/src/utils/cutils/blocking_queue.h ++++ b/src/utils/cutils/blocking_queue.h +@@ -26,7 +26,7 @@ + extern "C" { + #endif + +-#define BLOCKING_QUEUE_NO_TIMEOUT -1 ++#define BLOCKING_QUEUE_NO_TIMEOUT (-1) + + typedef struct blocking_node { + void *data; +diff --git a/src/utils/cutils/utils_aes.h b/src/utils/cutils/utils_aes.h +index bd2c2065..8ff6dad8 100644 +--- a/src/utils/cutils/utils_aes.h ++++ b/src/utils/cutils/utils_aes.h +@@ -26,7 +26,7 @@ extern "C" { + #define AES_256_CFB_KEY_LEN 32 + #define AES_256_CFB_IV_LEN 16 + +-int util_aes_key(const char *key_path, bool create, unsigned char *aeskey); ++int util_aes_key(const char *key_file, bool create, unsigned char *aeskey); + + // note: Input bytes is "IV+data", "bytes+AES_256_CFB_IV_LEN" is the real data to be encoded. + // The output length is the input "len" and add the '\0' after end of the length. +diff --git a/src/utils/cutils/utils_cap.h b/src/utils/cutils/utils_cap.h +index de63d070..c7e78ac2 100644 +--- a/src/utils/cutils/utils_cap.h ++++ b/src/utils/cutils/utils_cap.h +@@ -16,14 +16,13 @@ + #ifndef UTILS_CUTILS_UTILS_CAP_H + #define UTILS_CUTILS_UTILS_CAP_H + ++#include <stdbool.h> ++#include <stddef.h> ++ + #ifdef __cplusplus + extern "C" { + #endif + +-#include <stdbool.h> +-#include <stddef.h> +-#include <linux/capability.h> +- + bool util_valid_cap(const char *cap); + + /** +diff --git a/src/utils/cutils/utils_fs.h b/src/utils/cutils/utils_fs.h +index c44fed8c..438af416 100644 +--- a/src/utils/cutils/utils_fs.h ++++ b/src/utils/cutils/utils_fs.h +@@ -34,7 +34,7 @@ bool util_detect_mounted(const char *path); + int util_ensure_mounted_as(const char *dst, const char *mntopts); + int util_mount_from(const char *base, const char *src, const char *dst, const char *mtype, const char *mntopts); + typedef int (*mount_info_call_back_t)(const char *, const char *); +-bool util_deal_with_mount_info(mount_info_call_back_t cb, const char *); ++bool util_deal_with_mount_info(mount_info_call_back_t cb, const char *pattern); + bool util_check_readonly_fs(const char *path); + #ifdef __cplusplus + } +diff --git a/src/utils/cutils/utils_network.c b/src/utils/cutils/utils_network.c +index bb6a2f87..be33ec87 100644 +--- a/src/utils/cutils/utils_network.c ++++ b/src/utils/cutils/utils_network.c +@@ -801,6 +801,8 @@ static bool is_invalid_char(char c) + return true; + case ' ': + return true; ++ default: ++ return false; + } + return false; + } +diff --git a/src/utils/cutils/utils_string.h b/src/utils/cutils/utils_string.h +index 0de2266c..d37343d5 100644 +--- a/src/utils/cutils/utils_string.h ++++ b/src/utils/cutils/utils_string.h +@@ -17,7 +17,6 @@ + #define UTILS_CUTILS_UTILS_STRING_H + #include <stdbool.h> + #include <stddef.h> +-#include <stdint.h> + #include <sys/types.h> + + #ifdef __cplusplus +@@ -48,7 +47,7 @@ char **util_string_split(const char *src_str, char _sep); + // note that every delimiter bytes is considered to be a single delimiter + char **util_string_split_multi(const char *src_str, char delim); + +-char **util_string_split_n(const char *src_str, char delim, size_t n); ++char **util_string_split_n(const char *src, char sep, size_t n); + + const char *util_str_skip_str(const char *str, const char *skip); + +diff --git a/src/utils/tar/util_archive.h b/src/utils/tar/util_archive.h +index 8f0ab2a4..98597d53 100644 +--- a/src/utils/tar/util_archive.h ++++ b/src/utils/tar/util_archive.h +@@ -53,9 +53,8 @@ int archive_chroot_tar(const char *path, const char *file, const char *root_dir, + + int archive_chroot_tar_stream(const char *chroot_dir, const char *tar_path, const char *src_base, + const char *dst_base, const char *root_dir, struct io_read_wrapper *content); +-int archive_chroot_untar_stream(const struct io_read_wrapper *content, const char *chroot_dir, +- const char *untar_dir, const char *src_base, const char *dst_base, +- const char *root_dir, char **errmsg); ++int archive_chroot_untar_stream(const struct io_read_wrapper *context, const char *chroot_dir, const char *untar_dir, ++ const char *src_base, const char *dst_base, const char *root_dir, char **errmsg); + + int archive_copy_oci_tar_split_and_ret_size(int src_fd, const char *dist_file, int64_t *ret_size); + +diff --git a/src/utils/tar/util_gzip.h b/src/utils/tar/util_gzip.h +index 7d881e92..7797c5f9 100644 +--- a/src/utils/tar/util_gzip.h ++++ b/src/utils/tar/util_gzip.h +@@ -26,7 +26,7 @@ extern "C" { + int util_gzip_z(const char *srcfile, const char *dstfile, const mode_t mode); + + // Decompress +-int util_gzip_d(const char *srcfile, const FILE *destfp); ++int util_gzip_d(const char *srcfile, const FILE *dstfp); + + /* + * compress file. +-- +2.25.1 + diff --git a/0124-change-pull-registry-to-hub.oepkgs.net.patch b/0124-change-pull-registry-to-hub.oepkgs.net.patch new file mode 100644 index 0000000..75c8ca6 --- /dev/null +++ b/0124-change-pull-registry-to-hub.oepkgs.net.patch @@ -0,0 +1,137 @@ +From d6f7f7d3e2d644d2208ccc35f1de225b54c452a7 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Fri, 6 Sep 2024 17:45:58 +0800 +Subject: [PATCH 124/149] change pull registry to hub.oepkgs.net + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + CI/make-and-install.sh | 4 ++-- + .../container_cases/test_data/daemon.json | 2 +- + CI/test_cases/image_cases/image_digest.sh | 6 ++--- + CI/test_cases/image_cases/image_search.sh | 2 +- + CI/test_cases/image_cases/registry.sh | 22 +++++++++---------- + 5 files changed, 18 insertions(+), 18 deletions(-) + +diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh +index 61281965..2c2a4241 100755 +--- a/CI/make-and-install.sh ++++ b/CI/make-and-install.sh +@@ -95,7 +95,7 @@ cmake -DLIB_INSTALL_DIR=${restbuilddir}/lib -DCMAKE_INSTALL_PREFIX=${restbuilddi + make -j $(nproc) + make install + sed -i 's/"log-driver": "stdout"/"log-driver": "file"/g' ${restbuilddir}/etc/isulad/daemon.json +-sed -i "/registry-mirrors/a\ \"https://3laho3y3.mirror.aliyuncs.com\"" ${restbuilddir}/etc/isulad/daemon.json ++sed -i "/registry-mirrors/a\ \"https://hub.oepkgs.net\"" ${restbuilddir}/etc/isulad/daemon.json + + #build grpc version + cd $ISULAD_COPY_PATH +@@ -110,4 +110,4 @@ fi + make -j $(nproc) + make install + sed -i 's/"log-driver": "stdout"/"log-driver": "file"/g' ${builddir}/etc/isulad/daemon.json +-sed -i "/registry-mirrors/a\ \"https://3laho3y3.mirror.aliyuncs.com\"" ${builddir}/etc/isulad/daemon.json ++sed -i "/registry-mirrors/a\ \"https://hub.oepkgs.net\"" ${builddir}/etc/isulad/daemon.json +diff --git a/CI/test_cases/container_cases/test_data/daemon.json b/CI/test_cases/container_cases/test_data/daemon.json +index ab7d0360..20b001c0 100644 +--- a/CI/test_cases/container_cases/test_data/daemon.json ++++ b/CI/test_cases/container_cases/test_data/daemon.json +@@ -24,7 +24,7 @@ + "overlay2.override_kernel_check=true" + ], + "registry-mirrors": [ +- "https://3laho3y3.mirror.aliyuncs.com" ++ "https://hub.oepkgs.net" + ], + "insecure-registries": [ + ], +diff --git a/CI/test_cases/image_cases/image_digest.sh b/CI/test_cases/image_cases/image_digest.sh +index cc8b0e48..20774e07 100755 +--- a/CI/test_cases/image_cases/image_digest.sh ++++ b/CI/test_cases/image_cases/image_digest.sh +@@ -25,9 +25,9 @@ source ../helpers.sh + function test_image_with_digest() + { + local ret=0 +- local image="3laho3y3.mirror.aliyuncs.com/library/busybox" +- local image2="3laho3y3.mirror.aliyuncs.com/library/ubuntu" +- local image_digest="3laho3y3.mirror.aliyuncs.com/library/busybox@sha256:62ffc2ed7554e4c6d360bce40bbcf196573dd27c4ce080641a2c59867e732dee" ++ local image="hub.oepkgs.net/library/busybox" ++ local image2="hub.oepkgs.net/library/ubuntu" ++ local image_digest="hub.oepkgs.net/library/busybox@sha256:6066ca124f8c2686b7ae71aa1d6583b28c6dc3df3bdc386f2c89b92162c597d9" + local test="pull && inspect && tag image with digest test => (${FUNCNAME[@]})" + + msg_info "${test} starting..." +diff --git a/CI/test_cases/image_cases/image_search.sh b/CI/test_cases/image_cases/image_search.sh +index 4bf0e099..9ac680ce 100755 +--- a/CI/test_cases/image_cases/image_search.sh ++++ b/CI/test_cases/image_cases/image_search.sh +@@ -77,7 +77,7 @@ function test_image_search() + declare -i ans=0 + + # unable to pull image from docker.io without agent, skip this test +-# registry API v1 is not implemented in https://3laho3y3.mirror.aliyuncs.com and isula search cannot be tested ++# registry API v1 is not implemented in https://hub.oepkgs.net and isula search cannot be tested + # test_image_search || ((ans++)) + + show_result ${ans} "${curr_path}/${0}" +diff --git a/CI/test_cases/image_cases/registry.sh b/CI/test_cases/image_cases/registry.sh +index 7ea9a0c5..e56d99d3 100755 +--- a/CI/test_cases/image_cases/registry.sh ++++ b/CI/test_cases/image_cases/registry.sh +@@ -78,8 +78,8 @@ function isula_pull() + # isula pull docker.io/library/busybox:latest + # [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - --pull docker.io/library/busybox:latest failed" && ((ret++)) + +- isula pull 3laho3y3.mirror.aliyuncs.com/library/busybox +- fn_check_eq "$?" "0" "isula pull 3laho3y3.mirror.aliyuncs.com/library/busybox" ++ isula pull hub.oepkgs.net/library/busybox ++ fn_check_eq "$?" "0" "isula pull hub.oepkgs.net/library/busybox" + + rm -f /etc/isulad/daemon.json.bak + cp /etc/isulad/daemon.json /etc/isulad/daemon.json.bak +@@ -98,7 +98,7 @@ function isula_pull() + cp /etc/isulad/daemon.json.bak /etc/isulad/daemon.json + rm -f /etc/isulad/daemon.json.bak + +- isula rmi 3laho3y3.mirror.aliyuncs.com/library/busybox ++ isula rmi hub.oepkgs.net/library/busybox + + check_valgrind_log + fn_check_eq "$?" "0" "stop isulad with check valgrind" +@@ -109,12 +109,12 @@ function isula_pull() + + function isula_login() + { +- isula login -u test -p test 3laho3y3.mirror.aliyuncs.com +- fn_check_eq "$?" "0" "isula login -u test -p test 3laho3y3.mirror.aliyuncs.com" ++ isula login -u isulaci -p iSula123 hub.oepkgs.net ++ fn_check_eq "$?" "0" "isula login -u isulaci -p iSula123 hub.oepkgs.net" + + # double login for memory leak check +- isula login -u test -p test 3laho3y3.mirror.aliyuncs.com +- fn_check_eq "$?" "0" "isula login -u test -p test 3laho3y3.mirror.aliyuncs.com" ++ isula login -u isulaci -p iSula123 hub.oepkgs.net ++ fn_check_eq "$?" "0" "isula login -u isulaci -p iSula123 hub.oepkgs.net" + + # use username/password to pull busybox for memmory leak check + isula pull busybox +@@ -123,12 +123,12 @@ function isula_login() + + function isula_logout() + { +- isula logout 3laho3y3.mirror.aliyuncs.com +- fn_check_eq "$?" "0" "isula logout 3laho3y3.mirror.aliyuncs.com" ++ isula logout hub.oepkgs.net ++ fn_check_eq "$?" "0" "isula logout hub.oepkgs.net" + + # double logout for memory leak check +- isula logout 3laho3y3.mirror.aliyuncs.com +- fn_check_eq "$?" "0" "isula logout 3laho3y3.mirror.aliyuncs.com" ++ isula logout hub.oepkgs.net ++ fn_check_eq "$?" "0" "isula logout hub.oepkgs.net" + } + + function do_test_t() +-- +2.25.1 + diff --git a/0125-fix-clang-build-error.patch b/0125-fix-clang-build-error.patch new file mode 100644 index 0000000..ca8f9f6 --- /dev/null +++ b/0125-fix-clang-build-error.patch @@ -0,0 +1,216 @@ +From d141d8bfc7a602b0f139bef42a1c73dc673687de Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?=E6=AD=A6=E7=A7=AF=E8=B6=85?= <wujichao1@huawei.com> +Date: Mon, 21 Oct 2024 19:39:38 +0800 +Subject: [PATCH] fix-clang-build-error + +--- + src/daemon/common/cri/cri_helpers.cc | 4 ++-- + src/daemon/entry/cri/streams/stream_server.h | 4 ++-- + .../entry/cri/v1/v1_cri_container_manager_service.cc | 2 +- + .../entry/cri/v1/v1_cri_image_manager_service_impl.cc | 2 +- + .../entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc | 10 +++++----- + src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h | 2 +- + .../entry/cri/v1alpha/cri_container_manager_service.cc | 2 +- + .../cri/v1alpha/cri_pod_sandbox_manager_service.cc | 6 +++--- + src/daemon/sandbox/sandbox.cc | 2 +- + src/daemon/sandbox/sandbox_ops.cc | 2 +- + 10 files changed, 18 insertions(+), 18 deletions(-) + +diff --git a/src/daemon/common/cri/cri_helpers.cc b/src/daemon/common/cri/cri_helpers.cc +index 8117403c..a8cbd996 100644 +--- a/src/daemon/common/cri/cri_helpers.cc ++++ b/src/daemon/common/cri/cri_helpers.cc +@@ -525,8 +525,8 @@ void RemoveContainerLogSymlink(const std::string &containerID, Errors &error) + if (!path.empty()) { + // Only remove the symlink when container log path is specified. + if (util_path_remove(path.c_str()) != 0 && errno != ENOENT) { +- SYSERROR("Failed to remove container %s log symlink %s.", containerID.c_str(), path); +- error.Errorf("Failed to remove container %s log symlink %s.", containerID.c_str(), path); ++ SYSERROR("Failed to remove container %s log symlink %s.", containerID.c_str(), path.c_str()); ++ error.Errorf("Failed to remove container %s log symlink %s.", containerID.c_str(), path.c_str()); + } + } + } +diff --git a/src/daemon/entry/cri/streams/stream_server.h b/src/daemon/entry/cri/streams/stream_server.h +index 81aa9987..028dfc84 100644 +--- a/src/daemon/entry/cri/streams/stream_server.h ++++ b/src/daemon/entry/cri/streams/stream_server.h +@@ -17,6 +17,8 @@ + #include "errors.h" + #include "url.h" + ++url::URLDatum cri_stream_server_url(void); ++ + #ifdef __cplusplus + extern "C" { + #endif +@@ -27,8 +29,6 @@ void cri_stream_server_wait(void); + + void cri_stream_server_shutdown(void); + +-url::URLDatum cri_stream_server_url(void); +- + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +index d3fdd76a..1e84d14c 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +@@ -744,7 +744,7 @@ void ContainerManagerService::ListContainersToGRPC(container_list_response *resp + CRIHelpersV1::ContainerStatusToRuntime(Container_Status(response->containers[i]->status)); + container->set_state(state); + +- containers.push_back(move(container)); ++ containers.push_back(std::move(container)); + } + } + +diff --git a/src/daemon/entry/cri/v1/v1_cri_image_manager_service_impl.cc b/src/daemon/entry/cri/v1/v1_cri_image_manager_service_impl.cc +index 71918706..561a40d5 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_image_manager_service_impl.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_image_manager_service_impl.cc +@@ -149,7 +149,7 @@ void ImageManagerServiceImpl::list_images_to_grpc(im_list_response *response, + + imagetool_image_summary *element = list_images->images[i]; + conv_image_to_grpc(element, image); +- images.push_back(move(image)); ++ images.push_back(std::move(image)); + } + } + +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +index b629b1c3..a5f98619 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +@@ -536,7 +536,7 @@ auto PodSandboxManagerService::GetContainerListResponse(const std::string &readS + if (CRIHelpers::FiltersAddLabel(list_request->filters, CRIHelpers::Constants::SANDBOX_ID_LABEL_KEY, + readSandboxID) != 0) { + std::string tmp_errmsg = "Failed to add label in sandbox" + readSandboxID; +- ERROR(tmp_errmsg.c_str()); ++ ERROR("%s", tmp_errmsg.c_str()); + errors.push_back(tmp_errmsg); + return nullptr; + } +@@ -551,7 +551,7 @@ auto PodSandboxManagerService::GetContainerListResponse(const std::string &readS + } + if (ret != 0) { + if (list_response != nullptr && list_response->errmsg != nullptr) { +- ERROR(list_response->errmsg); ++ ERROR("%s", list_response->errmsg); + errors.push_back(list_response->errmsg); + } else { + ERROR("Failed to call list container callback"); +@@ -1218,7 +1218,7 @@ void PodSandboxManagerService::PodSandboxStatsToGRPC(const std::string &id, cons + return; + } + +- podStats = move(podStatsPtr); ++ podStats = std::move(podStatsPtr); + return; + } + +@@ -1227,7 +1227,7 @@ auto PodSandboxManagerService::PodSandboxStats(const std::string &podSandboxID, + Errors &error) -> std::unique_ptr<runtime::v1::PodSandboxStats> + { + Errors tmpErr; +- cgroup_metrics_t cgroupMetrics { 0 }; ++ cgroup_metrics_t cgroupMetrics {{ 0 }}; + std::vector<Network::NetworkInterfaceStats> netMetrics; + std::map<std::string, std::string> annotations; + std::unique_ptr<runtime::v1::PodSandboxStats> podStats { nullptr }; +@@ -1368,7 +1368,7 @@ void PodSandboxManagerService::ListPodSandboxStats(const runtime::v1::PodSandbox + continue; + } + +- podsStats.push_back(move(podStats)); ++ podsStats.push_back(std::move(podStats)); + } + } + +diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h +index 3d93c7bb..33539a32 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h ++++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h +@@ -104,7 +104,7 @@ protected: + private: + std::string m_podSandboxImage; + std::shared_ptr<Network::PluginManager> m_pluginManager { nullptr }; +- bool m_enablePodEvents; ++ [[maybe_unused]] bool m_enablePodEvents; + }; + } // namespace CRIV1 + #endif // DAEMON_ENTRY_CRI_V1_CRI_RUNTIME_SERVICE_IMPL_H +diff --git a/src/daemon/entry/cri/v1alpha/cri_container_manager_service.cc b/src/daemon/entry/cri/v1alpha/cri_container_manager_service.cc +index dbefa143..97acecd9 100644 +--- a/src/daemon/entry/cri/v1alpha/cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/v1alpha/cri_container_manager_service.cc +@@ -687,7 +687,7 @@ void ContainerManagerService::ListContainersToGRPC(container_list_response *resp + CRIHelpersV1Alpha::ContainerStatusToRuntime(Container_Status(response->containers[i]->status)); + container->set_state(state); + +- pods.push_back(move(container)); ++ pods.push_back(std::move(container)); + } + } + +diff --git a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +index 1c343cda..3c128645 100644 +--- a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +@@ -1573,7 +1573,7 @@ void PodSandboxManagerService::PodSandboxStatsToGRPC(const std::string &id, cons + return; + } + +- podStats = move(podStatsPtr); ++ podStats = std::move(podStatsPtr); + return; + } + +@@ -1583,7 +1583,7 @@ auto PodSandboxManagerService::PodSandboxStats(const std::string &podSandboxID, + { + Errors tmpErr; + container_inspect *inspectData { nullptr }; +- cgroup_metrics_t cgroupMetrics { 0 }; ++ cgroup_metrics_t cgroupMetrics {{ 0 }}; + std::vector<Network::NetworkInterfaceStats> netMetrics; + std::map<std::string, std::string> annotations; + std::unique_ptr<runtime::v1alpha2::PodSandboxStats> podStats { nullptr }; +@@ -1733,7 +1733,7 @@ void PodSandboxManagerService::ListPodSandboxStats(const runtime::v1alpha2::PodS + continue; + } + +- podsStats.push_back(move(podStats)); ++ podsStats.push_back(std::move(podStats)); + } + } + +diff --git a/src/daemon/sandbox/sandbox.cc b/src/daemon/sandbox/sandbox.cc +index d44abb99..dec082bc 100644 +--- a/src/daemon/sandbox/sandbox.cc ++++ b/src/daemon/sandbox/sandbox.cc +@@ -847,7 +847,7 @@ auto Sandbox::SaveState(Errors &error) -> bool + + nret = util_atomic_write_file(path.c_str(), stateJson.c_str(), stateJson.length(), CONFIG_FILE_MODE, false); + if (nret != 0) { +- SYSERROR("Failed to write file %s"); ++ SYSERROR("Failed to write file %s", path.c_str()); + error.Errorf("Failed to write file %s", path.c_str()); + return false; + } +diff --git a/src/daemon/sandbox/sandbox_ops.cc b/src/daemon/sandbox/sandbox_ops.cc +index b7fb40bf..22cfea95 100644 +--- a/src/daemon/sandbox/sandbox_ops.cc ++++ b/src/daemon/sandbox/sandbox_ops.cc +@@ -72,7 +72,7 @@ static int do_sandbox_prepare(const container_config_v2_common_config *config, + + params.containerId = config->id; + params.execId = (nullptr == exec_id) ? "" : exec_id; +- params.spec = std::move(std::unique_ptr<std::string>(new std::string(oci_spec))); ++ params.spec = std::unique_ptr<std::string>(new std::string(oci_spec)); + + if (generate_ctrl_rootfs(params, config) != 0) { + ERROR("Invalid rootfs"); +-- +2.25.1 + diff --git a/0126-add-a-new-registry-to-prevent-missing-mirrors.patch b/0126-add-a-new-registry-to-prevent-missing-mirrors.patch new file mode 100644 index 0000000..aba6256 --- /dev/null +++ b/0126-add-a-new-registry-to-prevent-missing-mirrors.patch @@ -0,0 +1,25 @@ +From a7a851f5be6c37665d948ec7587de062b6295bbe Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Sat, 7 Sep 2024 11:24:44 +0800 +Subject: [PATCH 133/149] add a new registry to prevent missing mirrors + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + CI/test_cases/container_cases/test_data/daemon.json | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/CI/test_cases/container_cases/test_data/daemon.json b/CI/test_cases/container_cases/test_data/daemon.json +index 20b001c0..cf7e0b9d 100644 +--- a/CI/test_cases/container_cases/test_data/daemon.json ++++ b/CI/test_cases/container_cases/test_data/daemon.json +@@ -24,6 +24,7 @@ + "overlay2.override_kernel_check=true" + ], + "registry-mirrors": [ ++ "https://docker.chenby.cn", + "https://hub.oepkgs.net" + ], + "insecure-registries": [ +-- +2.25.1 + diff --git a/0127-change-image-digest-ci-test-for-registry-change.patch b/0127-change-image-digest-ci-test-for-registry-change.patch new file mode 100644 index 0000000..f7c388b --- /dev/null +++ b/0127-change-image-digest-ci-test-for-registry-change.patch @@ -0,0 +1,47 @@ +From be8e1822b771576ef2f225da90dc6f0551477c0e Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Sat, 7 Sep 2024 14:49:33 +0800 +Subject: [PATCH 134/149] change image digest ci test for registry change + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + CI/make-and-install.sh | 5 ++--- + CI/test_cases/image_cases/image_digest.sh | 2 +- + 2 files changed, 3 insertions(+), 4 deletions(-) + +diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh +index 2c2a4241..599afcb9 100755 +--- a/CI/make-and-install.sh ++++ b/CI/make-and-install.sh +@@ -95,7 +95,7 @@ cmake -DLIB_INSTALL_DIR=${restbuilddir}/lib -DCMAKE_INSTALL_PREFIX=${restbuilddi + make -j $(nproc) + make install + sed -i 's/"log-driver": "stdout"/"log-driver": "file"/g' ${restbuilddir}/etc/isulad/daemon.json +-sed -i "/registry-mirrors/a\ \"https://hub.oepkgs.net\"" ${restbuilddir}/etc/isulad/daemon.json ++sed -i "/registry-mirrors/a\ \"https://hub.oepkgs.net\",\\n \"https://docker.chenby.cn\"" ${builddir}/etc/isulad/daemon.json + + #build grpc version + cd $ISULAD_COPY_PATH +@@ -109,5 +109,4 @@ else + fi + make -j $(nproc) + make install +-sed -i 's/"log-driver": "stdout"/"log-driver": "file"/g' ${builddir}/etc/isulad/daemon.json +-sed -i "/registry-mirrors/a\ \"https://hub.oepkgs.net\"" ${builddir}/etc/isulad/daemon.json ++sed -i "/registry-mirrors/a\ \"https://hub.oepkgs.net\",\\n \"https://docker.chenby.cn\"" ${builddir}/etc/isulad/daemon.json +diff --git a/CI/test_cases/image_cases/image_digest.sh b/CI/test_cases/image_cases/image_digest.sh +index 20774e07..ec1cbaa5 100755 +--- a/CI/test_cases/image_cases/image_digest.sh ++++ b/CI/test_cases/image_cases/image_digest.sh +@@ -26,7 +26,7 @@ function test_image_with_digest() + { + local ret=0 + local image="hub.oepkgs.net/library/busybox" +- local image2="hub.oepkgs.net/library/ubuntu" ++ local image2="ubuntu" + local image_digest="hub.oepkgs.net/library/busybox@sha256:6066ca124f8c2686b7ae71aa1d6583b28c6dc3df3bdc386f2c89b92162c597d9" + local test="pull && inspect && tag image with digest test => (${FUNCNAME[@]})" + +-- +2.25.1 + diff --git a/0128-bugfix-for-ci-make-and-install-shell.patch b/0128-bugfix-for-ci-make-and-install-shell.patch new file mode 100644 index 0000000..75d498d --- /dev/null +++ b/0128-bugfix-for-ci-make-and-install-shell.patch @@ -0,0 +1,26 @@ +From cc266a7c27cc40099f545b19d16fce49aee9a403 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Mon, 9 Sep 2024 10:51:43 +0800 +Subject: [PATCH 135/149] bugfix for ci make and install shell + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + CI/make-and-install.sh | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh +index 599afcb9..1498d700 100755 +--- a/CI/make-and-install.sh ++++ b/CI/make-and-install.sh +@@ -95,7 +95,7 @@ cmake -DLIB_INSTALL_DIR=${restbuilddir}/lib -DCMAKE_INSTALL_PREFIX=${restbuilddi + make -j $(nproc) + make install + sed -i 's/"log-driver": "stdout"/"log-driver": "file"/g' ${restbuilddir}/etc/isulad/daemon.json +-sed -i "/registry-mirrors/a\ \"https://hub.oepkgs.net\",\\n \"https://docker.chenby.cn\"" ${builddir}/etc/isulad/daemon.json ++sed -i "/registry-mirrors/a\ \"https://hub.oepkgs.net\",\\n \"https://docker.chenby.cn\"" ${restbuilddir}/etc/isulad/daemon.json + + #build grpc version + cd $ISULAD_COPY_PATH +-- +2.25.1 + diff --git a/0129-do-not-use-1000-as-the-test-gid-to-prevent-conflicts.patch b/0129-do-not-use-1000-as-the-test-gid-to-prevent-conflicts.patch new file mode 100644 index 0000000..d6f0533 --- /dev/null +++ b/0129-do-not-use-1000-as-the-test-gid-to-prevent-conflicts.patch @@ -0,0 +1,28 @@ +From 10af937fc2e095bce2da902c20e1f6b5e6178387 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Mon, 9 Sep 2024 17:18:52 +0800 +Subject: [PATCH 136/149] do not use 1000 as the test gid to prevent conflicts + with existing gids in the image + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + CI/test_cases/container_cases/exec_additional_gids.sh | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/CI/test_cases/container_cases/exec_additional_gids.sh b/CI/test_cases/container_cases/exec_additional_gids.sh +index 2edfd750..a62ab78c 100755 +--- a/CI/test_cases/container_cases/exec_additional_gids.sh ++++ b/CI/test_cases/container_cases/exec_additional_gids.sh +@@ -25,7 +25,8 @@ source ../helpers.sh + test_log=$(mktemp /tmp/additional_gids_test_XXX) + + USERNAME="user" +-USER_UID="1000" ++# Do not use 1000 as the test gid because "ubuntu:x:1000:" already exists in the ubuntu image ++USER_UID="1002" + USER_GID="$USER_UID" + ADDITIONAL_GID="1001" + ADDITIONAL_GROUP="additional" +-- +2.25.1 + diff --git a/0130-only-use-the-openeuler-mirror-registry-in-ci.patch b/0130-only-use-the-openeuler-mirror-registry-in-ci.patch new file mode 100644 index 0000000..d7adc89 --- /dev/null +++ b/0130-only-use-the-openeuler-mirror-registry-in-ci.patch @@ -0,0 +1,224 @@ +From d77740f686c90861198498ac760f0bb8a5bcc593 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Tue, 10 Sep 2024 15:16:37 +0800 +Subject: [PATCH 139/149] only use the openeuler mirror registry in ci + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + CI/make-and-install.sh | 4 ++-- + CI/test_cases/container_cases/exec_additional_gids.sh | 3 ++- + CI/test_cases/container_cases/runc_exec.sh | 2 +- + CI/test_cases/image_cases/image_digest.sh | 2 +- + CI/test_cases/image_cases/image_load.sh | 8 +++++--- + CI/test_cases/image_cases/image_tag.sh | 2 +- + CI/test_cases/image_cases/images_list.sh | 2 +- + CI/test_cases/image_cases/integration_check.sh | 2 +- + CI/test_cases/manual_cases/oom_monitor.sh | 4 ++-- + CI/test_cases/manual_cases/security_selinux.sh | 6 +++--- + 10 files changed, 19 insertions(+), 16 deletions(-) + +diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh +index 1498d700..09a68da6 100755 +--- a/CI/make-and-install.sh ++++ b/CI/make-and-install.sh +@@ -95,7 +95,7 @@ cmake -DLIB_INSTALL_DIR=${restbuilddir}/lib -DCMAKE_INSTALL_PREFIX=${restbuilddi + make -j $(nproc) + make install + sed -i 's/"log-driver": "stdout"/"log-driver": "file"/g' ${restbuilddir}/etc/isulad/daemon.json +-sed -i "/registry-mirrors/a\ \"https://hub.oepkgs.net\",\\n \"https://docker.chenby.cn\"" ${restbuilddir}/etc/isulad/daemon.json ++sed -i "/registry-mirrors/a\ \"https://hub.oepkgs.net\"" ${restbuilddir}/etc/isulad/daemon.json + + #build grpc version + cd $ISULAD_COPY_PATH +@@ -109,4 +109,4 @@ else + fi + make -j $(nproc) + make install +-sed -i "/registry-mirrors/a\ \"https://hub.oepkgs.net\",\\n \"https://docker.chenby.cn\"" ${builddir}/etc/isulad/daemon.json ++sed -i "/registry-mirrors/a\ \"https://hub.oepkgs.net\"" ${builddir}/etc/isulad/daemon.json +\ No newline at end of file +diff --git a/CI/test_cases/container_cases/exec_additional_gids.sh b/CI/test_cases/container_cases/exec_additional_gids.sh +index a62ab78c..a5eaf652 100755 +--- a/CI/test_cases/container_cases/exec_additional_gids.sh ++++ b/CI/test_cases/container_cases/exec_additional_gids.sh +@@ -38,13 +38,14 @@ function additional_gids_test() + { + local ret=0 + local runtime=$1 ++ local ubuntu_image="isulad/ubuntu" + test="exec additional gids test => test_exec_additional_gids => $runtime" + + msg_info "${test} starting..." + + isula rm -f `isula ps -a -q` + +- isula run -tid --runtime $runtime -n $cont_name ubuntu bash ++ isula run -tid --runtime $runtime -n $cont_name ${ubuntu_image} bash + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container" && ((ret++)) + + isula exec $cont_name bash -c "groupadd --gid $USER_GID $USERNAME \ +diff --git a/CI/test_cases/container_cases/runc_exec.sh b/CI/test_cases/container_cases/runc_exec.sh +index aa7020ee..32553694 100755 +--- a/CI/test_cases/container_cases/runc_exec.sh ++++ b/CI/test_cases/container_cases/runc_exec.sh +@@ -26,7 +26,7 @@ test="exec_runc_test => (${FUNCNAME[@]})" + function exec_runc_test() + { + local ret=0 +- local image="ubuntu" ++ local image="isulad/ubuntu" + local container_name="test_busybox" + + isula pull ${image} +diff --git a/CI/test_cases/image_cases/image_digest.sh b/CI/test_cases/image_cases/image_digest.sh +index ec1cbaa5..5036239a 100755 +--- a/CI/test_cases/image_cases/image_digest.sh ++++ b/CI/test_cases/image_cases/image_digest.sh +@@ -26,7 +26,7 @@ function test_image_with_digest() + { + local ret=0 + local image="hub.oepkgs.net/library/busybox" +- local image2="ubuntu" ++ local image2="isulad/ubuntu" + local image_digest="hub.oepkgs.net/library/busybox@sha256:6066ca124f8c2686b7ae71aa1d6583b28c6dc3df3bdc386f2c89b92162c597d9" + local test="pull && inspect && tag image with digest test => (${FUNCNAME[@]})" + +diff --git a/CI/test_cases/image_cases/image_load.sh b/CI/test_cases/image_cases/image_load.sh +index d50b3203..8a6c256a 100755 +--- a/CI/test_cases/image_cases/image_load.sh ++++ b/CI/test_cases/image_cases/image_load.sh +@@ -28,6 +28,7 @@ function test_image_load() + { + local ret=0 + local test="isula load image test => (${FUNCNAME[@]})" ++ local ubuntu_image="isulad/ubuntu" + + msg_info "${test} starting..." + +@@ -61,7 +62,7 @@ function test_image_load() + isula load -i $mult_image + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - load image failed: ${mult_image}" && ((ret++)) + +- ubuntu_id=`isula inspect -f '{{.image.id}}' ubuntu` ++ ubuntu_id=`isula inspect -f '{{.image.id}}' ${ubuntu_image}` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - fail to inspect image: ubuntu" && ((ret++)) + + busybox_id=`isula inspect -f '{{.image.id}}' busybox` +@@ -83,11 +84,12 @@ function test_concurrent_load() + { + local ret=0 + local test="isula load image test => (${FUNCNAME[@]})" ++ local ubuntu_image="isulad/ubuntu" + + msg_info "${test} starting..." + + # clean exist image +- ubuntu_id=`isula inspect -f '{{.image.id}}' ubuntu` ++ ubuntu_id=`isula inspect -f '{{.image.id}}' ${ubuntu_image}` + busybox_id=`isula inspect -f '{{.image.id}}' busybox` + isula rmi $ubuntu_id $busybox_id + +@@ -105,7 +107,7 @@ function test_concurrent_load() + + tail -n 50 /var/lib/isulad/isulad.log + +- ubuntu_id=`isula inspect -f '{{.image.id}}' ubuntu` ++ ubuntu_id=`isula inspect -f '{{.image.id}}' ${ubuntu_image}` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - fail to inspect image: ubuntu" && ((ret++)) + + top_layer_id=$(isula inspect -f '{{.image.top_layer}}' ${ubuntu_id}) +diff --git a/CI/test_cases/image_cases/image_tag.sh b/CI/test_cases/image_cases/image_tag.sh +index 551d8249..0b8dd3e8 100755 +--- a/CI/test_cases/image_cases/image_tag.sh ++++ b/CI/test_cases/image_cases/image_tag.sh +@@ -23,7 +23,7 @@ declare -r curr_path=$(dirname $(readlink -f "$0")) + source ../helpers.sh + + image_busybox="busybox" +-image_hello="hello-world" ++image_hello="isulad/hello-world" + + function test_tag_image() + { +diff --git a/CI/test_cases/image_cases/images_list.sh b/CI/test_cases/image_cases/images_list.sh +index 56cde5b6..a192a75c 100755 +--- a/CI/test_cases/image_cases/images_list.sh ++++ b/CI/test_cases/image_cases/images_list.sh +@@ -25,7 +25,7 @@ source ../helpers.sh + function test_image_list() + { + local ret=0 +- local image="hello-world" ++ local image="isulad/hello-world" + local image_busybox="busybox" + local INVALID_IMAGE="k~k" + local test="list images info test => (${FUNCNAME[@]})" +diff --git a/CI/test_cases/image_cases/integration_check.sh b/CI/test_cases/image_cases/integration_check.sh +index f340348d..6a55706c 100755 +--- a/CI/test_cases/image_cases/integration_check.sh ++++ b/CI/test_cases/image_cases/integration_check.sh +@@ -27,7 +27,7 @@ image="busybox" + function test_image_info() + { + local ret=0 +- local uimage="nats" ++ local uimage="isulad/nats" + local test="list && inspect image info test => (${FUNCNAME[@]})" + local lid + local cid +diff --git a/CI/test_cases/manual_cases/oom_monitor.sh b/CI/test_cases/manual_cases/oom_monitor.sh +index a1c2503d..8e991cc8 100755 +--- a/CI/test_cases/manual_cases/oom_monitor.sh ++++ b/CI/test_cases/manual_cases/oom_monitor.sh +@@ -26,7 +26,7 @@ test_data_path=$(realpath $curr_path/test_data) + function test_oom_monitor() + { + local ret=0 +- local ubuntu_image="ubuntu" ++ local ubuntu_image="isulad/ubuntu" + local test="container oom monitor test => (${FUNCNAME[@]})" + containername="oommonitor" + +@@ -35,7 +35,7 @@ function test_oom_monitor() + isula pull ${ubuntu_image} + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${ubuntu_image}" && return ${FAILURE} + +- isula images | grep ubuntu ++ isula images | grep ${ubuntu_image} + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${ubuntu_image}" && ((ret++)) + + # use more than 10m memory limit, otherwise it might fail to run +diff --git a/CI/test_cases/manual_cases/security_selinux.sh b/CI/test_cases/manual_cases/security_selinux.sh +index c558befe..cf94c809 100755 +--- a/CI/test_cases/manual_cases/security_selinux.sh ++++ b/CI/test_cases/manual_cases/security_selinux.sh +@@ -87,7 +87,7 @@ function daemon_disable_selinux() + function test_isulad_selinux_file_label() + { + local ret=0 +- local image="centos" ++ local image="isulad/centos" + local test="isulad selinux file label test => (${FUNCNAME[@]})" + + msg_info "${test} starting..." +@@ -144,7 +144,7 @@ function test_isulad_selinux_file_label() + function test_isulad_selinux_process_label() + { + local ret=0 +- local image="centos" ++ local image="isulad/centos" + local test="isulad selinux process label test => (${FUNCNAME[@]})" + + msg_info "${test} starting..." +@@ -172,7 +172,7 @@ function test_isulad_selinux_process_label() + function test_isulad_selinux_mount_mode() + { + local ret=0 +- local image="centos" ++ local image="isulad/centos" + local test="isulad selinux mount mode test => (${FUNCNAME[@]})" + + msg_info "${test} starting..." +-- +2.25.1 + diff --git a/0131-modify-alpine-image-source-to-isulad-alpine.patch b/0131-modify-alpine-image-source-to-isulad-alpine.patch new file mode 100644 index 0000000..52f5380 --- /dev/null +++ b/0131-modify-alpine-image-source-to-isulad-alpine.patch @@ -0,0 +1,25 @@ +From c90dab4057f73614537b3765ee06173c55d4d39c Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?=E6=AD=A6=E7=A7=AF=E8=B6=85?= <wujichao1@huawei.com> +Date: Wed, 11 Sep 2024 10:34:48 +0800 +Subject: [PATCH 140/149] modify alpine image source to isulad/alpine + +--- + CI/test_cases/image_cases/integration_check.sh | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/CI/test_cases/image_cases/integration_check.sh b/CI/test_cases/image_cases/integration_check.sh +index 6a55706c..f5ae94e9 100755 +--- a/CI/test_cases/image_cases/integration_check.sh ++++ b/CI/test_cases/image_cases/integration_check.sh +@@ -55,7 +55,7 @@ function test_image_info() + ucid=$(isula create ${uimage}) + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - create container failed" && ((ret++)) + +- isula run -tid --name checker alpine ++ isula run -tid --name checker isulad/alpine + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - run container failed" && ((ret++)) + + tmp_fname=$(echo -n "/var/run/isulad/storage" | sha256sum | awk '{print $1}') +-- +2.25.1 + diff --git a/0132-update-docs-design-README_zh.md.patch b/0132-update-docs-design-README_zh.md.patch new file mode 100644 index 0000000..cab3794 --- /dev/null +++ b/0132-update-docs-design-README_zh.md.patch @@ -0,0 +1,26 @@ +From a38db853128d0fe1c521829a4f2b17dc08f31aed Mon Sep 17 00:00:00 2001 +From: chen524 <chenkui_yewu@cmss.chinamobile.com> +Date: Wed, 11 Sep 2024 04:18:46 +0000 +Subject: [PATCH 141/149] update docs/design/README_zh.md. + +Signed-off-by: chen524 <chenkui_yewu@cmss.chinamobile.com> +--- + docs/design/README_zh.md | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/docs/design/README_zh.md b/docs/design/README_zh.md +index 0f4cf13e..1a23207d 100644 +--- a/docs/design/README_zh.md ++++ b/docs/design/README_zh.md +@@ -37,7 +37,7 @@ + + - 查看 image store 模块的设计文档： [image_store_design](./detailed/Image/image_store_design_zh.md) 。 + +-- 查看 layer store 模块的设计文档 [layer_store_degisn](./detailed/Image/layer_store_degisn_zh.md) 。 ++- 查看 layer store 模块的设计文档： [layer_store_degisn](./detailed/Image/layer_store_degisn_zh.md) 。 + + - 查看 registry 模块的设计文档： [registry_degisn](./detailed/Image/registry_degisn_zh.md) 。 + +-- +2.25.1 + diff --git a/0133-modify-the-image-name-isulad-ubuntu-to-ubuntu.patch b/0133-modify-the-image-name-isulad-ubuntu-to-ubuntu.patch new file mode 100644 index 0000000..faee164 --- /dev/null +++ b/0133-modify-the-image-name-isulad-ubuntu-to-ubuntu.patch @@ -0,0 +1,34 @@ +From 18a8120dfa71b50879b562692013a308b9508224 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?=E6=AD=A6=E7=A7=AF=E8=B6=85?= <wujichao1@huawei.com> +Date: Wed, 11 Sep 2024 17:03:08 +0800 +Subject: [PATCH 142/149] modify the image name: isulad/ubuntu to ubuntu + +--- + CI/test_cases/image_cases/image_load.sh | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/CI/test_cases/image_cases/image_load.sh b/CI/test_cases/image_cases/image_load.sh +index 8a6c256a..8a3fd7b3 100755 +--- a/CI/test_cases/image_cases/image_load.sh ++++ b/CI/test_cases/image_cases/image_load.sh +@@ -28,7 +28,7 @@ function test_image_load() + { + local ret=0 + local test="isula load image test => (${FUNCNAME[@]})" +- local ubuntu_image="isulad/ubuntu" ++ local ubuntu_image="ubuntu" + + msg_info "${test} starting..." + +@@ -84,7 +84,7 @@ function test_concurrent_load() + { + local ret=0 + local test="isula load image test => (${FUNCNAME[@]})" +- local ubuntu_image="isulad/ubuntu" ++ local ubuntu_image="ubuntu" + + msg_info "${test} starting..." + +-- +2.25.1 + diff --git a/0134-ignore-chdir-failed-errmsg-when-kill-and-delete.patch b/0134-ignore-chdir-failed-errmsg-when-kill-and-delete.patch new file mode 100644 index 0000000..cbd8381 --- /dev/null +++ b/0134-ignore-chdir-failed-errmsg-when-kill-and-delete.patch @@ -0,0 +1,116 @@ +From f3a8da522798e68a6ba5e8f00163c4a6d05a30d0 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Wed, 7 Aug 2024 23:44:12 +1400 +Subject: [PATCH 143/149] ignore chdir failed errmsg when kill and delete + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + .../modules/runtime/isula/isula_rt_ops.c | 44 ++++++++++++++----- + 1 file changed, 32 insertions(+), 12 deletions(-) + +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 62cff3cf..dc156154 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -69,7 +69,7 @@ + #define RESOURCE_FNAME_FORMATS "%s/resources.json" + + // handle string from stderr output. +-typedef int(*handle_output_callback_t)(const char *output); ++typedef int(*handle_output_callback_t)(const char *output, const char *workdir); + typedef struct { + bool fg; + const char *id; +@@ -757,7 +757,7 @@ static int runtime_call_simple(const char *workdir, const char *runtime, const c + // we consider the runtime call simple succeeded, + // even if the process exit with failure. + if (stderr_msg != NULL && cb != NULL) { +- ret = cb(stderr_msg); ++ ret = cb(stderr_msg, workdir); + } + } + +@@ -768,17 +768,37 @@ static int runtime_call_simple(const char *workdir, const char *runtime, const c + + // oci runtime return -1 if the container 'does not exist' + // if output contains 'does not exist', means nothing to kill or delete, return 0 ++// util_exec_cmd return -1 if chdir failed ++// if output contains 'chdir %s failed', means state dir damaged, return 0 + // this will change the exit status of kill or delete command +-static int non_existent_output_check(const char *output) ++static int shielded_output_check(const char *output, const char *workdir) + { +- char *pattern = "does not exist"; ++ int nret = 0; ++ const char *nonexist_pattern = "does not exist"; ++ char chdir_pattern[PATH_MAX] = { 0 }; + +- if (output == NULL) { ++ if (output == NULL || workdir == NULL) { + return -1; + } + + // container not exist, kill or delete success, return 0 +- if (util_strings_contains_word(output, pattern)) { ++ if (util_strings_contains_word(output, nonexist_pattern)) { ++ return 0; ++ } ++ ++ if (sizeof(chdir_pattern) > PATH_MAX - strlen("chdir ") - strlen(" failed")) { ++ INFO("chdir_pattern is too long"); ++ return -1; ++ } ++ ++ nret = snprintf(chdir_pattern, sizeof(chdir_pattern), "chdir %s failed", workdir); ++ if (nret < 0 || (size_t)nret >= sizeof(chdir_pattern)) { ++ INFO("Failed to make full chdir_pattern"); ++ return -1; ++ } ++ ++ // if output contains 'chdir ${workdir} failed', means state dir damaged, return 0 ++ if (util_strings_contains_word(output, chdir_pattern)) { + return 0; + } + +@@ -786,15 +806,15 @@ static int non_existent_output_check(const char *output) + return -1; + } + +-// kill success or non_existent_output_check succeed return 0, DO_RETRY_CALL will break; ++// kill success or shielded_output_check succeed return 0, DO_RETRY_CALL will break; + // if kill failed, recheck on shim alive, if not alive, kill succeed, still return 0; + // else, return -1, DO_RETRY_CALL will call this again; + static int runtime_call_kill_and_check(const char *workdir, const char *runtime, const char *id) + { + int ret = -1; + +- // kill succeed, return 0; non_existent_output_check succeed, return 0; +- ret = runtime_call_simple(workdir, runtime, "kill", NULL, 0, id, non_existent_output_check); ++ // kill succeed, return 0; shielded_output_check succeed, return 0; ++ ret = runtime_call_simple(workdir, runtime, "kill", NULL, 0, id, shielded_output_check); + if (ret == 0) { + return 0; + } +@@ -814,8 +834,8 @@ static int runtime_call_delete_force(const char *workdir, const char *runtime, c + // if the container does not exist when force deleting it, + // runc will report an error and isulad does not need to retry the deletion again. + // related PR ID:d1a743674a98e23d348b29f52c43436356f56b79 +- // non_existent_output_check succeed, return 0; +- return runtime_call_simple(workdir, runtime, "delete", opts, 1, id, non_existent_output_check); ++ // shielded_output_check succeed, return 0; ++ return runtime_call_simple(workdir, runtime, "delete", opts, 1, id, shielded_output_check); + } + + #define ExitSignalOffset 128 +@@ -1825,7 +1845,7 @@ static int create_resources_json_file(const char *workdir, const shim_client_cgr + } + + // show std error msg, always return -1. +-static int show_stderr(const char *err) ++static int show_stderr(const char *err, const char *workdir) + { + isulad_set_error_message(err); + return -1; +-- +2.25.1 + diff --git a/0135-followlocation-only-not-with-head.patch b/0135-followlocation-only-not-with-head.patch new file mode 100644 index 0000000..2fd2bb0 --- /dev/null +++ b/0135-followlocation-only-not-with-head.patch @@ -0,0 +1,28 @@ +From f526268e2b78330dfe6b63eb5f6ece7417f4c06e Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Sat, 27 Apr 2024 14:38:58 +0800 +Subject: [PATCH 144/149] followlocation only not with head + +Signed-off-by: jikai <jikai11@huawei.com> +--- + src/utils/http/http.c | 4 +++- + 1 file changed, 3 insertions(+), 1 deletion(-) + +diff --git a/src/utils/http/http.c b/src/utils/http/http.c +index 8b74f773..c9bb5959 100644 +--- a/src/utils/http/http.c ++++ b/src/utils/http/http.c +@@ -495,7 +495,9 @@ int http_request(const char *url, struct http_get_options *options, long *respon + if (options->resume) { + curl_easy_setopt(curl_handle, CURLOPT_RESUME_FROM_LARGE, (curl_off_t)fsize); + } +- curl_easy_setopt(curl_handle, CURLOPT_FOLLOWLOCATION, 1L); ++ if (options->with_head == 0) { ++ curl_easy_setopt(curl_handle, CURLOPT_FOLLOWLOCATION, 1L); ++ } + curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, pagefile); + curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, fwrite_file); + } else { +-- +2.25.1 + diff --git a/0136-update-docs-design-detailed-Image-image_storage_driv.patch b/0136-update-docs-design-detailed-Image-image_storage_driv.patch new file mode 100644 index 0000000..7ee7b61 --- /dev/null +++ b/0136-update-docs-design-detailed-Image-image_storage_driv.patch @@ -0,0 +1,28 @@ +From 06dca49eacebb94a6d38de5042e3608bce0e366f Mon Sep 17 00:00:00 2001 +From: Beans <gujiateng_yewu@cmss.chinamobile.com> +Date: Tue, 24 Sep 2024 03:15:14 +0000 +Subject: [PATCH 145/149] update + docs/design/detailed/Image/image_storage_driver_design_zh.md. This place is + spelled wrong. + +Signed-off-by: Beans <gujiateng_yewu@cmss.chinamobile.com> +--- + docs/design/detailed/Image/image_storage_driver_design_zh.md | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/docs/design/detailed/Image/image_storage_driver_design_zh.md b/docs/design/detailed/Image/image_storage_driver_design_zh.md +index ea82df14..9799fc31 100644 +--- a/docs/design/detailed/Image/image_storage_driver_design_zh.md ++++ b/docs/design/detailed/Image/image_storage_driver_design_zh.md +@@ -116,7 +116,7 @@ int graphdriver_cleanup(void) + + ## 3.1 Driver 初始化 + +-Driver 初始化初始化流程： ++Driver 初始化流程： + +  + Overlay 模块初始化流程： +-- +2.25.1 + diff --git a/0137-upgrade-isulad-compilation-script-install_iSulad_on_.patch b/0137-upgrade-isulad-compilation-script-install_iSulad_on_.patch new file mode 100644 index 0000000..1ee2323 --- /dev/null +++ b/0137-upgrade-isulad-compilation-script-install_iSulad_on_.patch @@ -0,0 +1,77 @@ +From b9b6e5bd6984db8ab33ea1f7d8650113d8c21fd1 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Fri, 27 Sep 2024 17:26:03 +1400 +Subject: [PATCH 146/149] upgrade isulad compilation script + install_iSulad_on_Ubuntu_20_04_LTS + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + CI/dockerfiles/Dockerfile-ubuntu | 2 +- + .../install_iSulad_on_Ubuntu_20_04_LTS.sh | 26 +++++++++++++++---- + 2 files changed, 22 insertions(+), 6 deletions(-) + +diff --git a/CI/dockerfiles/Dockerfile-ubuntu b/CI/dockerfiles/Dockerfile-ubuntu +index 09a20eb5..6420173a 100644 +--- a/CI/dockerfiles/Dockerfile-ubuntu ++++ b/CI/dockerfiles/Dockerfile-ubuntu +@@ -83,8 +83,8 @@ RUN apt update -y && apt upgrade -y && \ + patch \ + tcpdump + ++RUN apt install -y ninja-build meson + RUN apt install -y libncurses-dev && apt autoremove -y +-RUN pip3 install meson ninja + + RUN echo "export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH" >> /etc/bashrc && \ + echo "export LD_LIBRARY_PATH=/usr/local/lib:/usr/lib:$LD_LIBRARY_PATH" >> /etc/bashrc && \ +diff --git a/docs/build_docs/guide/script/install_iSulad_on_Ubuntu_20_04_LTS.sh b/docs/build_docs/guide/script/install_iSulad_on_Ubuntu_20_04_LTS.sh +index f44bddb4..35995ff4 100755 +--- a/docs/build_docs/guide/script/install_iSulad_on_Ubuntu_20_04_LTS.sh ++++ b/docs/build_docs/guide/script/install_iSulad_on_Ubuntu_20_04_LTS.sh +@@ -7,10 +7,25 @@ set -e + export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH + export LD_LIBRARY_PATH=/usr/local/lib:/usr/lib:/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH + echo "/usr/local/lib" >> /etc/ld.so.conf +-apt install -y g++ libprotobuf-dev protobuf-compiler protobuf-compiler-grpc libgrpc++-dev libgrpc-dev libtool automake autoconf cmake make pkg-config libyajl-dev zlib1g-dev libselinux1-dev libseccomp-dev libcap-dev libsystemd-dev git libarchive-dev libcurl4-gnutls-dev openssl libdevmapper-dev python3 libtar0 libtar-dev libwebsockets-dev ++ ++ ++if [ ! -e "/etc/timezone" ]; then ++ export TZ=Asia/Shanghai ++ ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone ++fi ++ ++apt update -y && apt upgrade -y ++apt install -y g++ systemd libprotobuf-dev protobuf-compiler protobuf-compiler-grpc libgrpc++-dev libgrpc-dev libtool automake autoconf cmake make pkg-config libyajl-dev zlib1g-dev libselinux1-dev libseccomp-dev libcap-dev libsystemd-dev git libarchive-dev libcurl4-gnutls-dev openssl libdevmapper-dev python3 libtar0 libtar-dev libwebsockets-dev ++ ++apt install -y runc ++ ++apt install -y docbook2x ninja-build meson ++apt install -y libncurses-dev + + BUILD_DIR=/tmp/build_isulad + ++git config --global http.sslverify false ++ + rm -rf $BUILD_DIR + mkdir -p $BUILD_DIR + +@@ -18,11 +33,12 @@ mkdir -p $BUILD_DIR + cd $BUILD_DIR + git clone https://gitee.com/src-openeuler/lxc.git + cd lxc ++git config --global --add safe.directory $BUILD_DIR/lxc/lxc-5.0.2 + ./apply-patches +-cd lxc-4.0.3 +-./autogen.sh +-./configure +-make -j $(nproc) ++cd lxc-5.0.2 ++meson setup -Disulad=true \ ++ -Dprefix=/usr build ++meson compile -C build + make install + + # build lcr +-- +2.25.1 + diff --git a/0138-bugfix-for-log-in-make_safedir_is_noexec.patch b/0138-bugfix-for-log-in-make_safedir_is_noexec.patch new file mode 100644 index 0000000..3007930 --- /dev/null +++ b/0138-bugfix-for-log-in-make_safedir_is_noexec.patch @@ -0,0 +1,26 @@ +From d0bfea1f4858672345d1b885bfb9ee1f6072c396 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Sun, 29 Sep 2024 17:22:42 +1400 +Subject: [PATCH 147/149] bugfix for log in make_safedir_is_noexec + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/utils/tar/util_archive.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/utils/tar/util_archive.c b/src/utils/tar/util_archive.c +index 985e0f16..b946dd39 100644 +--- a/src/utils/tar/util_archive.c ++++ b/src/utils/tar/util_archive.c +@@ -219,7 +219,7 @@ static int make_safedir_is_noexec(const char *flock_path, const char *dstdir, ch + } + + if (realpath(isulad_tmpdir_env, cleanpath) == NULL) { +- ERROR("Failed to get real path for %s", isula_tmpdir); ++ SYSERROR("Failed to get real path for %s", isulad_tmpdir_env); + return -1; + } + +-- +2.25.1 + diff --git a/0139-containers-in-paused-state-are-not-allowed-to-start.patch b/0139-containers-in-paused-state-are-not-allowed-to-start.patch new file mode 100644 index 0000000..7e0e549 --- /dev/null +++ b/0139-containers-in-paused-state-are-not-allowed-to-start.patch @@ -0,0 +1,29 @@ +From 5721206da703bf827cd125feb8f9bf518f10f69c Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Mon, 14 Oct 2024 19:27:57 +0800 +Subject: [PATCH 148/149] containers in paused state are not allowed to start + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/executor/container_cb/execution.c | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/src/daemon/executor/container_cb/execution.c b/src/daemon/executor/container_cb/execution.c +index f924a208..2e54f085 100644 +--- a/src/daemon/executor/container_cb/execution.c ++++ b/src/daemon/executor/container_cb/execution.c +@@ -584,6 +584,11 @@ static int container_start_cb(const container_start_request *request, container_ + + if (container_is_running(cont->state)) { + INFO("Container is already running"); ++ if (container_is_paused(cont->state)) { ++ cc = ISULAD_ERR_EXEC; ++ ERROR("cannot start a paused container, try unpause instead"); ++ isulad_set_error_message("cannot start a paused container, try unpause instead"); ++ } + goto pack_response; + } + +-- +2.25.1 + diff --git a/0140-remove-meaningless-code.patch b/0140-remove-meaningless-code.patch new file mode 100644 index 0000000..136e541 --- /dev/null +++ b/0140-remove-meaningless-code.patch @@ -0,0 +1,38 @@ +From e7778ed261d2550ea6e2179b856d0ee22241b858 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Mon, 14 Oct 2024 20:42:58 +0800 +Subject: [PATCH 149/149] remove meaningless code + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc | 1 - + src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h | 1 - + 2 files changed, 2 deletions(-) + +diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc +index 56c89c1e..23b620c3 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc +@@ -24,7 +24,6 @@ CRIRuntimeServiceImpl::CRIRuntimeServiceImpl(const std::string &podSandboxImage, + , m_containerManager(new ContainerManagerService(cb)) + , m_podSandboxManager(new PodSandboxManagerService(podSandboxImage, cb, pluginManager, enablePodEvents)) + , m_runtimeManager(new RuntimeManagerService(cb, pluginManager)) +- , m_enablePodEvents(enablePodEvents) + { + } + +diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h +index 33539a32..1d399a85 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h ++++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h +@@ -104,7 +104,6 @@ protected: + private: + std::string m_podSandboxImage; + std::shared_ptr<Network::PluginManager> m_pluginManager { nullptr }; +- [[maybe_unused]] bool m_enablePodEvents; + }; + } // namespace CRIV1 + #endif // DAEMON_ENTRY_CRI_V1_CRI_RUNTIME_SERVICE_IMPL_H +-- +2.25.1 + diff --git a/0141-fix-unqualified-call-to-std-move.patch b/0141-fix-unqualified-call-to-std-move.patch new file mode 100644 index 0000000..99cbc93 --- /dev/null +++ b/0141-fix-unqualified-call-to-std-move.patch @@ -0,0 +1,25 @@ +From f7e8abb13d1f1fd4b3c322853c91ef490da7141b Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?=E6=AD=A6=E7=A7=AF=E8=B6=85?= <wujichao1@huawei.com> +Date: Tue, 22 Oct 2024 15:57:55 +0800 +Subject: [PATCH] fix unqualified call to "std::move" + +--- + src/daemon/entry/cri/v1alpha/cri_image_manager_service_impl.cc | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/daemon/entry/cri/v1alpha/cri_image_manager_service_impl.cc b/src/daemon/entry/cri/v1alpha/cri_image_manager_service_impl.cc +index cf636428..a14dc626 100644 +--- a/src/daemon/entry/cri/v1alpha/cri_image_manager_service_impl.cc ++++ b/src/daemon/entry/cri/v1alpha/cri_image_manager_service_impl.cc +@@ -149,7 +149,7 @@ void ImageManagerServiceImpl::list_images_to_grpc(im_list_response *response, + + imagetool_image_summary *element = list_images->images[i]; + conv_image_to_grpc(element, image); +- images.push_back(move(image)); ++ images.push_back(std::move(image)); + } + } + +-- +2.25.1 + diff --git a/0142-pull-failure-shows-error-reason.patch b/0142-pull-failure-shows-error-reason.patch new file mode 100644 index 0000000..c156d61 --- /dev/null +++ b/0142-pull-failure-shows-error-reason.patch @@ -0,0 +1,27 @@ +From dd3f733949a0c6c9256a63b54079e9a7b8aa7fe6 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Fri, 25 Oct 2024 02:50:01 +1400 +Subject: [PATCH 142/156] pull failure shows error reason + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/entry/connect/grpc/grpc_images_service.cc | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/src/daemon/entry/connect/grpc/grpc_images_service.cc b/src/daemon/entry/connect/grpc/grpc_images_service.cc +index 32f0446e..6135945a 100644 +--- a/src/daemon/entry/connect/grpc/grpc_images_service.cc ++++ b/src/daemon/entry/connect/grpc/grpc_images_service.cc +@@ -691,6 +691,9 @@ Status ImagesServiceImpl::PullImage(ServerContext *context, const PullImageReque + stream.writer = (void *)writer; + + ret = cb->image.pull(image_req, &stream, &image_res); ++ if (image_res->errmsg != NULL) { ++ errmsg = image_res->errmsg; ++ } + free_image_pull_image_request(image_req); + free_image_pull_image_response(image_res); + if (ret == 0) { +-- +2.34.1 + diff --git a/0143-move-CGROUP2_SUPER_MAGIC-define-to-cgroup.c.patch b/0143-move-CGROUP2_SUPER_MAGIC-define-to-cgroup.c.patch new file mode 100644 index 0000000..01f25ae --- /dev/null +++ b/0143-move-CGROUP2_SUPER_MAGIC-define-to-cgroup.c.patch @@ -0,0 +1,44 @@ +From d0bb2c00769ab41ada532a04338b91032d7ada2e Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Sat, 26 Oct 2024 16:59:09 +1400 +Subject: [PATCH 143/156] move CGROUP2_SUPER_MAGIC define to cgroup.c + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/common/cgroup/cgroup.c | 4 ++++ + src/daemon/common/cgroup/cgroup_v2.c | 4 ---- + 2 files changed, 4 insertions(+), 4 deletions(-) + +diff --git a/src/daemon/common/cgroup/cgroup.c b/src/daemon/common/cgroup/cgroup.c +index 77fafdae..a6846c03 100644 +--- a/src/daemon/common/cgroup/cgroup.c ++++ b/src/daemon/common/cgroup/cgroup.c +@@ -36,6 +36,10 @@ + #define CGROUP_SUPER_MAGIC 0x27e0eb + #endif + ++#ifndef CGROUP2_SUPER_MAGIC ++#define CGROUP2_SUPER_MAGIC 0x63677270 ++#endif ++ + static cgroup_ops g_cgroup_ops; + + static int get_cgroup_version_for_init(void) +diff --git a/src/daemon/common/cgroup/cgroup_v2.c b/src/daemon/common/cgroup/cgroup_v2.c +index ce72e6c4..078425d2 100644 +--- a/src/daemon/common/cgroup/cgroup_v2.c ++++ b/src/daemon/common/cgroup/cgroup_v2.c +@@ -52,10 +52,6 @@ + #define CGROUP2_CPUSET_CPUS_EFFECTIVE_PATH CGROUP_MOUNTPOINT"/cpuset.cpus.effective" + #define CGROUP2_CPUSET_MEMS_EFFECTIVE_PATH CGROUP_MOUNTPOINT"/cpuset.mems.effective" + +-#ifndef CGROUP2_SUPER_MAGIC +-#define CGROUP2_SUPER_MAGIC 0x63677270 +-#endif +- + static int get_value_ull_v2(const char *content, const char *match, void *result) + { + uint64_t ull_result = 0; +-- +2.34.1 + diff --git a/0144-update-centos-build-script.patch b/0144-update-centos-build-script.patch new file mode 100644 index 0000000..4400afe --- /dev/null +++ b/0144-update-centos-build-script.patch @@ -0,0 +1,189 @@ +From 10aa1ba26eecdf044c1a97e05db53d286e19a3af Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Sat, 26 Oct 2024 18:24:21 +1400 +Subject: [PATCH 144/156] update centos build script + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + CI/dockerfiles/Dockerfile-centos | 10 ++- + docs/build_docs/README_zh.md | 4 + + .../script/install_iSulad_on_Centos_7.sh | 83 +++---------------- + 3 files changed, 23 insertions(+), 74 deletions(-) + +diff --git a/CI/dockerfiles/Dockerfile-centos b/CI/dockerfiles/Dockerfile-centos +index af3ce035..0b11014b 100644 +--- a/CI/dockerfiles/Dockerfile-centos ++++ b/CI/dockerfiles/Dockerfile-centos +@@ -24,6 +24,11 @@ + FROM centos:7.6.1810 + MAINTAINER LiFeng <lifeng68@huawei.com> + ++RUN sed -i 's/mirror.centos.org/archive.kernel.org\/centos-vault/g' /etc/yum.repos.d/*.repo ++RUN sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo ++RUN sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo ++RUN echo "sslverify=false" | tee -a /etc/yum.conf ++ + # Install dependency package + RUN yum clean all && yum makecache && yum install -y epel-release && yum swap -y fakesystemd systemd && \ + yum update -y && \ +@@ -113,6 +118,9 @@ RUN git config --global http.sslverify false + # install rust + RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + ++RUN mkdir -p /root/.cargo/ ++RUN touch ${HOME}/.cargo/config ++ + # configure rust + RUN echo "[source.crates-io]" >> ${HOME}/.cargo/config && \ + echo "[source.local-registry]" >> ${HOME}/.cargo/config && \ +@@ -286,7 +294,7 @@ RUN export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH && \ + make install && \ + ldconfig + +-RUN dnf install -y ncurses-devel && dnf clean all ++RUN yum install -y ncurses-devel && yum clean all + + VOLUME [ "/sys/fs/cgroup" ] + CMD ["/usr/sbin/init"] +diff --git a/docs/build_docs/README_zh.md b/docs/build_docs/README_zh.md +index f27fbc4f..6450bf7e 100644 +--- a/docs/build_docs/README_zh.md ++++ b/docs/build_docs/README_zh.md +@@ -40,6 +40,8 @@ $ cd iSulad/docs/build_docs/guide/script + $ sudo ./install_iSulad_on_Centos_7.sh + ``` + ++若需要在centos容器中编译iSulad，可参照iSulad编译可使用的镜像dockerfile[Dockerfile-ubuntu](../../CI/dockerfiles/Dockerfile-ubuntu)。 ++ + ### Ubuntu + + 我们同样在代码仓中提供了在Ubuntu上自动化安装的脚本: [install_iSulad_on_Ubuntu_20_04_LTS](./guide/script/install_iSulad_on_Ubuntu_20_04_LTS.sh)，您只需要执行这个脚本就可以自动编译安装iSulad以及其依赖的组件。 +@@ -51,6 +53,8 @@ $ sudo chmod +x ./install_iSulad_on_Ubuntu_20_04_LTS.sh + $ sudo ./install_iSulad_on_Ubuntu_20_04_LTS.sh + ``` + ++若需要在ubuntu容器中编译iSulad，可参照iSulad编译可使用的镜像dockerfile[Dockerfile-centos](../../CI/dockerfiles/)。 ++ + ## 构建指南 + + 我们提供了多种构建iSulad的方式： +diff --git a/docs/build_docs/guide/script/install_iSulad_on_Centos_7.sh b/docs/build_docs/guide/script/install_iSulad_on_Centos_7.sh +index b268d777..d83520e1 100755 +--- a/docs/build_docs/guide/script/install_iSulad_on_Centos_7.sh ++++ b/docs/build_docs/guide/script/install_iSulad_on_Centos_7.sh +@@ -5,93 +5,29 @@ set -x + set -e + + # install neccessary packages +-yum install -y patch automake autoconf libtool cmake make libcap libcap-devel libselinux libselinux-devel libseccomp libseccomp-devel yajl-devel git libcgroup tar python3 python3-pip device-mapper-devel libcurl-devel zlib-devel glibc-headers openssl-devel gcc gcc-c++ systemd-devel systemd-libs golang libtar libtar-devel which ++# yum install -y patch automake autoconf libtool cmake make libcap libcap-devel libselinux libselinux-devel libseccomp libseccomp-devel yajl-devel git libcgroup tar python3 python3-pip device-mapper-devel libcurl-devel zlib-devel glibc-headers openssl-devel gcc gcc-c++ systemd-devel systemd-libs golang libtar libtar-devel which + + # export LDFLAGS + export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH + export LD_LIBRARY_PATH=/usr/local/lib:/usr/lib:$LD_LIBRARY_PATH +- echo "/usr/local/lib" >> /etc/ld.so.conf ++echo "/usr/local/lib" >> /etc/ld.so.conf + + BUILD_DIR=/tmp/build_isulad + + rm -rf $BUILD_DIR + mkdir -p $BUILD_DIR + +-# build libarchive +-cd $BUILD_DIR +-git clone https://gitee.com/src-openeuler/libarchive.git +-cd libarchive +-git checkout -b openEuler-20.03-LTS-tag openEuler-20.03-LTS-tag +-tar -zxvf libarchive-3.4.1.tar.gz +-cd libarchive-3.4.1 +-patch -p1 -F1 -s < ../libarchive-uninitialized-value.patch +-cd build +-cmake -DCMAKE_USE_SYSTEM_LIBRARIES=ON ../ +-make -j $(nproc) +-make install +-ldconfig +- +-# build protobuf +-cd $BUILD_DIR +-git clone https://gitee.com/src-openeuler/protobuf.git +-cd protobuf +-git checkout openEuler-20.03-LTS-tag +-tar -xzvf protobuf-all-3.9.0.tar.gz +-cd protobuf-3.9.0 +-./autogen.sh +-./configure +-make -j $(nproc) +-make install +-ldconfig +- +-# build c-ares +-cd $BUILD_DIR +-git clone https://gitee.com/src-openeuler/c-ares.git +-cd c-ares +-git checkout openEuler-20.03-LTS-tag +-tar -xzvf c-ares-1.15.0.tar.gz +-cd c-ares-1.15.0 +-autoreconf -if +-./configure --enable-shared --disable-dependency-tracking +-make -j $(nproc) +-make install +-ldconfig +- +-# build grpc +-cd $BUILD_DIR +-git clone https://gitee.com/src-openeuler/grpc.git +-cd grpc +-git checkout openEuler-20.03-LTS-tag +-tar -xzvf grpc-1.22.0.tar.gz +-cd grpc-1.22.0 +-make -j $(nproc) +-make install +-ldconfig +- +-# build libwebsockets +-cd $BUILD_DIR +-git clone https://gitee.com/src-openeuler/libwebsockets.git +-cd libwebsockets +-git checkout openEuler-20.03-LTS-tag +-tar -xzvf libwebsockets-2.4.2.tar.gz +-cd libwebsockets-2.4.2 +-patch -p1 -F1 -s < ../libwebsockets-fix-coredump.patch +-mkdir build +-cd build +-cmake -DLWS_WITH_SSL=0 -DLWS_MAX_SMP=32 -DCMAKE_BUILD_TYPE=Debug ../ +-make -j $(nproc) +-make install +-ldconfig +- + # build lxc + cd $BUILD_DIR + git clone https://gitee.com/src-openeuler/lxc.git + cd lxc ++git config --global --add safe.directory $BUILD_DIR/lxc/lxc-5.0.2 + ./apply-patches +-cd lxc-4.0.3 +-./autogen.sh +-./configure +-make -j ++cd lxc-5.0.2 ++sed -i 's/return open(rpath, (int)((unsigned int)flags | O_CLOEXEC));/return open(rpath, (int)((unsigned int)flags | O_CLOEXEC), 0);/g' src/lxc/isulad_utils.c ++meson setup -Disulad=true \ ++ -Dprefix=/usr build ++meson compile -C build + make install + + # build lcr +@@ -118,8 +54,9 @@ make install + cd $BUILD_DIR + git clone https://gitee.com/openeuler/iSulad.git + cd iSulad ++sed -i 's/-O2 -Wall -fPIE/-O2 -Wall -fPIE -std=gnu99/g' cmake/set_build_flags.cmake + mkdir build + cd build +-cmake .. ++cmake -DDISABLE_WERROR=on ../ + make + make install +-- +2.34.1 + diff --git a/0145-cni-change-error-info.patch b/0145-cni-change-error-info.patch new file mode 100644 index 0000000..7f4e256 --- /dev/null +++ b/0145-cni-change-error-info.patch @@ -0,0 +1,97 @@ +From 445642718fd3dd7867aad9fc8e6eb5b7dafadcb6 Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Fri, 1 Nov 2024 12:56:38 +0800 +Subject: [PATCH 145/156] cni:change error info + +Signed-off-by: liuxu <liuxu156@huawei.com> +--- + .../modules/network/cni_operator/cni_operate.c | 8 ++++---- + .../network/cni_operator/libcni/libcni_api.c | 14 +++++++++++++- + .../network/cni_operator/libcni/libcni_api.h | 2 +- + 3 files changed, 18 insertions(+), 6 deletions(-) + +diff --git a/src/daemon/modules/network/cni_operator/cni_operate.c b/src/daemon/modules/network/cni_operator/cni_operate.c +index 6db6db51..80a2642d 100644 +--- a/src/daemon/modules/network/cni_operator/cni_operate.c ++++ b/src/daemon/modules/network/cni_operator/cni_operate.c +@@ -731,13 +731,13 @@ static int update_runtime_conf_cni_args_by_cached(cni_cached_info *info, struct + return 0; + } + +-static int get_configs_from_cached(const char *network, struct runtime_conf *rc, char **conf_list) ++static int get_configs_from_cached(const char *network, const char *cni_version, struct runtime_conf *rc, char **conf_list) + { + int ret = 0; + size_t i; + cni_cached_info *info = NULL; + +- info = cni_get_network_list_cached_info(network, rc); ++ info = cni_get_network_list_cached_info(network, cni_version, rc); + if (info == NULL) { + return 0; + } +@@ -857,7 +857,7 @@ int check_network_plane(const struct cni_manager *manager, const struct cni_netw + goto out; + } + +- ret = get_configs_from_cached(list->list->name, rc, NULL); ++ ret = get_configs_from_cached(list->list->name, list->list->cni_version, rc, NULL); + if (ret != 0) { + ERROR("Get cached info failed"); + ret = -1; +@@ -901,7 +901,7 @@ int detach_network_plane(const struct cni_manager *manager, const struct cni_net + goto out; + } + +- ret = get_configs_from_cached(list->list->name, rc, NULL); ++ ret = get_configs_from_cached(list->list->name, list->list->cni_version, rc, NULL); + if (ret != 0) { + ERROR("Get cached info failed"); + ret = -1; +diff --git a/src/daemon/modules/network/cni_operator/libcni/libcni_api.c b/src/daemon/modules/network/cni_operator/libcni/libcni_api.c +index 7ba983af..c55c8e9d 100644 +--- a/src/daemon/modules/network/cni_operator/libcni/libcni_api.c ++++ b/src/daemon/modules/network/cni_operator/libcni/libcni_api.c +@@ -81,13 +81,25 @@ struct cni_opt_result *cni_get_network_list_cached_result(const struct cni_netwo + return result; + } + +-cni_cached_info *cni_get_network_list_cached_info(const char *network, const struct runtime_conf *rc) ++cni_cached_info *cni_get_network_list_cached_info(const char *network, const char *cni_version, const struct runtime_conf *rc) + { ++ bool greater = false; + if (network == NULL) { + ERROR("Empty network"); + return NULL; + } + ++ if (util_version_greater_than_or_equal_to(cni_version, SUPPORT_CACHE_AND_CHECK_VERSION, &greater) != 0) { ++ ERROR("Invalid cni version %s", cni_version); ++ return NULL; ++ } ++ ++ // CACHE was added in CNI spec version 0.4.0 and higher ++ if (!greater) { ++ WARN("result version: %s is too old, do not save this cache", cni_version); ++ return NULL; ++ } ++ + return cni_cache_read(g_module_conf.cache_dir, network, rc); + } + +diff --git a/src/daemon/modules/network/cni_operator/libcni/libcni_api.h b/src/daemon/modules/network/cni_operator/libcni/libcni_api.h +index 2f10d6e9..bf16f168 100644 +--- a/src/daemon/modules/network/cni_operator/libcni/libcni_api.h ++++ b/src/daemon/modules/network/cni_operator/libcni/libcni_api.h +@@ -75,7 +75,7 @@ bool cni_module_init(const char *cache_dir, const char * const *paths, size_t pa + struct cni_opt_result *cni_get_network_list_cached_result(const struct cni_network_list_conf *list, + const struct runtime_conf *rc); + +-cni_cached_info *cni_get_network_list_cached_info(const char *network, const struct runtime_conf *rc); ++cni_cached_info *cni_get_network_list_cached_info(const char *network, const char *cni_version, const struct runtime_conf *rc); + + int cni_add_network_list(const struct cni_network_list_conf *list, const struct runtime_conf *rc, + struct cni_opt_result **pret); +-- +2.34.1 + diff --git a/0146-bugfix-for-sem_wait-call-when-errno-is-EINTR.patch b/0146-bugfix-for-sem_wait-call-when-errno-is-EINTR.patch new file mode 100644 index 0000000..38992e3 --- /dev/null +++ b/0146-bugfix-for-sem_wait-call-when-errno-is-EINTR.patch @@ -0,0 +1,233 @@ +From 70f5e98110b2c63755f283712eebbd075787081b Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Tue, 5 Nov 2024 03:15:04 +1400 +Subject: [PATCH 146/156] bugfix for sem_wait call when errno is EINTR + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/cmd/isula/base/start.c | 4 +++- + src/cmd/isula/client_console.c | 4 +++- + src/cmd/isula/stream/attach.c | 8 ++++++-- + src/cmd/isula/stream/exec.c | 4 +++- + src/cmd/isulad-shim/process.c | 4 +++- + src/cmd/isulad/main.c | 8 ++++++-- + src/daemon/entry/connect/grpc/grpc_containers_service.cc | 8 ++++++-- + src/daemon/entry/cri/streams/attach_serve.cc | 4 +++- + src/daemon/entry/cri/streams/websocket/ws_server.cc | 5 ++++- + src/daemon/modules/events/collector.c | 8 ++++++-- + src/daemon/modules/service/io_handler.c | 4 +++- + 11 files changed, 46 insertions(+), 15 deletions(-) + +diff --git a/src/cmd/isula/base/start.c b/src/cmd/isula/base/start.c +index 6a066638..3f8c13ee 100644 +--- a/src/cmd/isula/base/start.c ++++ b/src/cmd/isula/base/start.c +@@ -193,7 +193,9 @@ out: + void client_wait_fifo_exit(const struct client_arguments *args) + { + if (args->custom_conf.attach_stdin || args->custom_conf.attach_stdout || args->custom_conf.attach_stderr) { +- sem_wait(&g_console_waitexit_sem); ++ while(sem_wait(&g_console_waitexit_sem) == -1 && errno == EINTR) { ++ continue; ++ } + } + } + +diff --git a/src/cmd/isula/client_console.c b/src/cmd/isula/client_console.c +index 555f59b6..fb6f6c05 100644 +--- a/src/cmd/isula/client_console.c ++++ b/src/cmd/isula/client_console.c +@@ -259,7 +259,9 @@ int start_client_console_thread(struct command_fifo_config *console_fifos, bool + return -1; + } + +- sem_wait(console_fifos->wait_open); ++ while(sem_wait(console_fifos->wait_open) == -1 && errno == EINTR) { ++ continue; ++ } + + return 0; + } +diff --git a/src/cmd/isula/stream/attach.c b/src/cmd/isula/stream/attach.c +index b61c9350..bc3eb141 100644 +--- a/src/cmd/isula/stream/attach.c ++++ b/src/cmd/isula/stream/attach.c +@@ -285,7 +285,9 @@ static int container_wait_thread(struct client_arguments *args, uint32_t *exit_c + (void)sem_destroy(&sem_started); + return -1; + } +- (void)sem_wait(&sem_started); ++ while(sem_wait(&sem_started) == -1 && errno == EINTR) { ++ continue; ++ } + (void)sem_destroy(&sem_started); + return 0; + } +@@ -366,7 +368,9 @@ static int client_attach(struct client_arguments *args, uint32_t *exit_code) + } + + #ifndef GRPC_CONNECTOR +- sem_wait(&g_attach_waitexit_sem); ++ while(sem_wait(&g_attach_waitexit_sem) == -1 && errno == EINTR) { ++ continue; ++ } + #endif + + if (clock_gettime(CLOCK_REALTIME, &ts) == -1) { +diff --git a/src/cmd/isula/stream/exec.c b/src/cmd/isula/stream/exec.c +index cacb0278..6eab4d4f 100644 +--- a/src/cmd/isula/stream/exec.c ++++ b/src/cmd/isula/stream/exec.c +@@ -380,7 +380,9 @@ static int local_cmd_exec(struct client_arguments *args, uint32_t *exit_code) + ret = client_exec(args, command_fifos, exit_code); + if (ret == 0 && + (args->custom_conf.attach_stdin || args->custom_conf.attach_stdout || args->custom_conf.attach_stderr)) { +- sem_wait(&g_command_waitexit_sem); ++ while(sem_wait(&g_command_waitexit_sem) == -1 && errno == EINTR) { ++ continue; ++ } + } + out: + delete_command_fifo(command_fifos); +diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c +index 18fae03f..11903a5c 100644 +--- a/src/cmd/isulad-shim/process.c ++++ b/src/cmd/isulad-shim/process.c +@@ -1237,7 +1237,9 @@ int process_io_start(process_t *p, pthread_t *tid_epoll) + if (ret != SHIM_OK) { + return SHIM_SYS_ERR(errno); + } +- (void)sem_wait(&p->sem_mainloop); ++ while(sem_wait(&p->sem_mainloop) == -1 && errno == EINTR) { ++ continue; ++ } + (void)sem_destroy(&p->sem_mainloop); + + return SHIM_OK; +diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c +index 0228caa8..7c6148fd 100644 +--- a/src/cmd/isulad/main.c ++++ b/src/cmd/isulad/main.c +@@ -1648,7 +1648,9 @@ static void *do_shutdown_handler(void *arg) + + prctl(PR_SET_NAME, "Shutdown"); + +- sem_wait(&g_daemon_shutdown_sem); ++ while(sem_wait(&g_daemon_shutdown_sem) == -1 && errno == EINTR) { ++ continue; ++ } + + daemon_shutdown(); + +@@ -1868,7 +1870,9 @@ int main(int argc, char **argv) + + server_common_start(); + +- sem_wait(&g_daemon_wait_shutdown_sem); ++ while(sem_wait(&g_daemon_wait_shutdown_sem) == -1 && errno == EINTR) { ++ continue; ++ } + + DAEMON_CLEAR_ERRMSG(); + return 0; +diff --git a/src/daemon/entry/connect/grpc/grpc_containers_service.cc b/src/daemon/entry/connect/grpc/grpc_containers_service.cc +index c5e7c275..0a46b36c 100644 +--- a/src/daemon/entry/connect/grpc/grpc_containers_service.cc ++++ b/src/daemon/entry/connect/grpc/grpc_containers_service.cc +@@ -306,7 +306,9 @@ Status ContainerServiceImpl::RemoteStart(ServerContext *context, + // close pipe 1 first, make sure io copy thread exit + close(read_pipe_fd[1]); + if (container_req->attach_stderr && ret == 0) { +- (void)sem_wait(&sem); ++ while(sem_wait(&sem) == -1 && errno == EINTR) { ++ continue; ++ } + } + (void)sem_destroy(&sem); + close(read_pipe_fd[0]); +@@ -656,7 +658,9 @@ Status ContainerServiceImpl::Attach(ServerContext *context, ServerReaderWriter<A + close(pipefd[1]); + // Waiting sem, make sure the sem is posted always in attach callback. + if (container_req->attach_stderr && ret == 0) { +- (void)sem_wait(&sem_stderr); ++ while(sem_wait(&sem_stderr) == -1 && errno == EINTR) { ++ continue; ++ } + } + (void)sem_destroy(&sem_stderr); + close(pipefd[0]); +diff --git a/src/daemon/entry/cri/streams/attach_serve.cc b/src/daemon/entry/cri/streams/attach_serve.cc +index 3d59e539..9c0e56c4 100644 +--- a/src/daemon/entry/cri/streams/attach_serve.cc ++++ b/src/daemon/entry/cri/streams/attach_serve.cc +@@ -145,7 +145,9 @@ int AttachServe::ExecuteStreamCommand(SessionData *lwsCtx, void *request) + WsWriteStdoutToClient(lwsCtx, message.c_str(), message.length()); + } else { + // wait io copy thread complete +- (void)sem_wait(&attachSem); ++ while(sem_wait(&attachSem) == -1 && errno == EINTR) { ++ continue; ++ } + } + + (void)sem_destroy(&attachSem); +diff --git a/src/daemon/entry/cri/streams/websocket/ws_server.cc b/src/daemon/entry/cri/streams/websocket/ws_server.cc +index a8d89b36..7e3225b2 100644 +--- a/src/daemon/entry/cri/streams/websocket/ws_server.cc ++++ b/src/daemon/entry/cri/streams/websocket/ws_server.cc +@@ -193,7 +193,10 @@ void WebsocketServer::CloseWsSession(int socketID) + close(session->pipes.at(1)); + session->pipes.at(1) = -1; + } +- (void)sem_wait(session->syncCloseSem); ++ ++ while(sem_wait(session->syncCloseSem) == -1 && errno == EINTR) { ++ continue; ++ } + (void)sem_destroy(session->syncCloseSem); + delete session->syncCloseSem; + session->syncCloseSem = nullptr; +diff --git a/src/daemon/modules/events/collector.c b/src/daemon/modules/events/collector.c +index af688742..eb79bf81 100644 +--- a/src/daemon/modules/events/collector.c ++++ b/src/daemon/modules/events/collector.c +@@ -932,7 +932,9 @@ int add_monitor_client(char *name, const types_timestamp_t *since, const types_t + goto sem_free; + } + +- sem_wait(&context_info->context_sem); ++ while(sem_wait(&context_info->context_sem) == -1 && errno == EINTR) { ++ continue; ++ } + + sem_free: + sem_destroy(&context_info->context_sem); +@@ -1002,7 +1004,9 @@ static int start_monitored() + goto out; + } + +- sem_wait(msync.monitord_sem); ++ while(sem_wait(msync.monitord_sem) == -1 && errno == EINTR) { ++ continue; ++ } + sem_destroy(msync.monitord_sem); + if (monitored_exitcode) { + isulad_set_error_message("Monitored start failed"); +diff --git a/src/daemon/modules/service/io_handler.c b/src/daemon/modules/service/io_handler.c +index 474fa650..f3b47737 100644 +--- a/src/daemon/modules/service/io_handler.c ++++ b/src/daemon/modules/service/io_handler.c +@@ -485,7 +485,9 @@ static int start_io_copy_thread(int sync_fd, bool detach, struct io_copy_arg *co + return -1; + } + +- sem_wait(&thread_arg.wait_sem); ++ while(sem_wait(&thread_arg.wait_sem) == -1 && errno == EINTR) { ++ continue; ++ } + sem_destroy(&thread_arg.wait_sem); + return 0; + } +-- +2.34.1 + diff --git a/0147-add-no-pivot-root-support.patch b/0147-add-no-pivot-root-support.patch new file mode 100644 index 0000000..6e29081 --- /dev/null +++ b/0147-add-no-pivot-root-support.patch @@ -0,0 +1,229 @@ +From 3a925ead33267d44cafd182a85e75c9c3ac25d58 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Wed, 6 Nov 2024 15:24:30 +0800 +Subject: [PATCH 147/156] add no pivot root support + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + cmake/options.cmake | 7 +++++++ + src/cmd/isula/base/create.c | 7 +++++++ + src/cmd/isula/base/create.h | 11 ++++++++++ + src/cmd/isula/base/run.c | 3 +++ + src/cmd/isula/client_arguments.h | 4 ++++ + src/cmd/isula/isula_host_spec.c | 5 +++++ + src/cmd/isula/isula_host_spec.h | 4 ++++ + src/cmd/isulad-shim/process.c | 6 ++++++ + src/daemon/modules/api/runtime_api.h | 1 + + .../modules/runtime/isula/isula_rt_ops.c | 3 +++ + .../modules/service/service_container.c | 20 +++++++++++++++++++ + 11 files changed, 71 insertions(+) + +diff --git a/cmake/options.cmake b/cmake/options.cmake +index 41177fe0..15d7044a 100644 +--- a/cmake/options.cmake ++++ b/cmake/options.cmake +@@ -168,6 +168,13 @@ if (ENABLE_NATIVE_NETWORK OR ENABLE_GRPC) + set(ENABLE_NETWORK 1) + endif() + ++option(ENABLE_NO_PIVOT_ROOT "Enable no pivot root" ON) ++if (ENABLE_NO_PIVOT_ROOT STREQUAL "ON") ++ add_definitions(-DENABLE_NO_PIVOT_ROOT) ++ set(ENABLE_NO_PIVOT_ROOT 1) ++ message("${Green}-- Enable no pivot root${ColourReset}") ++endif() ++ + option(ENABLE_PLUGIN "enable plugin module" OFF) + if (ENABLE_PLUGIN STREQUAL "ON") + add_definitions(-DENABLE_PLUGIN=1) +diff --git a/src/cmd/isula/base/create.c b/src/cmd/isula/base/create.c +index b04dddb5..7331676a 100644 +--- a/src/cmd/isula/base/create.c ++++ b/src/cmd/isula/base/create.c +@@ -1292,6 +1292,10 @@ static isula_host_config_t *request_pack_host_config(const struct client_argumen + hostconfig->publish_all = args->custom_conf.publish_all; + #endif + ++#ifdef ENABLE_NO_PIVOT_ROOT ++ hostconfig->no_pivot_root = args->custom_conf.no_pivot_root; ++#endif ++ + return hostconfig; + + error_out: +@@ -1750,6 +1754,9 @@ int cmd_create_main(int argc, const char **argv) + COMMON_OPTIONS(g_cmd_create_args) + #ifdef ENABLE_NATIVE_NETWORK + CREATE_NETWORK_OPTIONS(g_cmd_create_args) ++#endif ++#ifdef ENABLE_NO_PIVOT_ROOT ++ NO_PIVOT_ROOT_OPTIONS(g_cmd_create_args) + #endif + }; + +diff --git a/src/cmd/isula/base/create.h b/src/cmd/isula/base/create.h +index 9eb471b4..b4205a88 100644 +--- a/src/cmd/isula/base/create.h ++++ b/src/cmd/isula/base/create.h +@@ -39,6 +39,17 @@ extern "C" { + #define USERNS_OPT(cmdargs) + #endif + ++#ifdef ENABLE_NO_PIVOT_ROOT ++#define NO_PIVOT_ROOT_OPTIONS(cmdargs) \ ++ { CMD_OPT_TYPE_BOOL, \ ++ false, \ ++ "no-pivot", \ ++ 0, \ ++ &(cmdargs).custom_conf.no_pivot_root, \ ++ "disable use of pivot-root (oci runtime only)", \ ++ NULL }, ++#endif ++ + #define CREATE_OPTIONS(cmdargs) \ + { \ + CMD_OPT_TYPE_BOOL, \ +diff --git a/src/cmd/isula/base/run.c b/src/cmd/isula/base/run.c +index 8d48244c..15e035c1 100644 +--- a/src/cmd/isula/base/run.c ++++ b/src/cmd/isula/base/run.c +@@ -150,6 +150,9 @@ int cmd_run_main(int argc, const char **argv) + CREATE_EXTEND_OPTIONS(g_cmd_run_args) RUN_OPTIONS(g_cmd_run_args) + #ifdef ENABLE_NATIVE_NETWORK + CREATE_NETWORK_OPTIONS(g_cmd_run_args) ++#endif ++#ifdef ENABLE_NO_PIVOT_ROOT ++ NO_PIVOT_ROOT_OPTIONS(g_cmd_run_args) + #endif + }; + isula_libutils_default_log_config(argv[0], &lconf); +diff --git a/src/cmd/isula/client_arguments.h b/src/cmd/isula/client_arguments.h +index 76d01122..debcc903 100644 +--- a/src/cmd/isula/client_arguments.h ++++ b/src/cmd/isula/client_arguments.h +@@ -237,6 +237,10 @@ struct custom_configs { + /* publish a container's port to the host */ + char **publish; + #endif ++ ++#ifdef ENABLE_NO_PIVOT_ROOT ++ bool no_pivot_root; ++#endif + }; + + struct args_cgroup_resources { +diff --git a/src/cmd/isula/isula_host_spec.c b/src/cmd/isula/isula_host_spec.c +index 9e902ed9..4c2fefa4 100644 +--- a/src/cmd/isula/isula_host_spec.c ++++ b/src/cmd/isula/isula_host_spec.c +@@ -1729,6 +1729,11 @@ int generate_hostconfig(const isula_host_config_t *srcconfig, char **hostconfigs + #ifdef ENABLE_NATIVE_NETWORK + dstconfig->port_bindings = srcconfig->port_bindings; + #endif ++ ++#ifdef ENABLE_NO_PIVOT_ROOT ++ dstconfig->no_pivot_root = srcconfig->no_pivot_root; ++#endif ++ + *hostconfigstr = host_config_generate_json(dstconfig, &ctx, &err); + #ifdef ENABLE_NATIVE_NETWORK + dstconfig->port_bindings = NULL; +diff --git a/src/cmd/isula/isula_host_spec.h b/src/cmd/isula/isula_host_spec.h +index 25a54236..f00526e4 100644 +--- a/src/cmd/isula/isula_host_spec.h ++++ b/src/cmd/isula/isula_host_spec.h +@@ -163,6 +163,10 @@ typedef struct isula_host_config { + bool publish_all; + defs_map_string_object_port_bindings *port_bindings; + #endif ++ ++#ifdef ENABLE_NO_PIVOT_ROOT ++ bool no_pivot_root; ++#endif + } isula_host_config_t; + + int generate_hostconfig(const isula_host_config_t *srcconfig, char **hostconfigstr); +diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c +index 11903a5c..10d21565 100644 +--- a/src/cmd/isulad-shim/process.c ++++ b/src/cmd/isulad-shim/process.c +@@ -1298,6 +1298,12 @@ static void get_runtime_cmd(process_t *p, const char *log_path, const char *pid_ + params[i++] = "create"; + params[i++] = "--bundle"; + params[i++] = p->bundle; ++#ifdef ENABLE_NO_PIVOT_ROOT ++ if (getenv("ISULAD_RAMDISK") != NULL || p->state->no_pivot_root) { ++ params[i++] = "--no-pivot"; ++ } ++#endif ++ + } + params[i++] = "--pid-file"; + params[i++] = pid_path; +diff --git a/src/daemon/modules/api/runtime_api.h b/src/daemon/modules/api/runtime_api.h +index bd170c30..930710ca 100644 +--- a/src/daemon/modules/api/runtime_api.h ++++ b/src/daemon/modules/api/runtime_api.h +@@ -84,6 +84,7 @@ typedef struct _rt_create_params_t { + bool tty; + bool open_stdin; + const char *task_addr; ++ bool no_pivot_root; + } rt_create_params_t; + + typedef struct _rt_start_params_t { +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index dc156154..e628c3fe 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -1209,6 +1209,9 @@ int rt_isula_create(const char *id, const char *runtime, const rt_create_params_ + p.runtime_args_len = runtime_args_len; + p.attach_socket = attach_socket; + p.systemd_cgroup = conf_get_systemd_cgroup(); ++#ifdef ENABLE_NO_PIVOT_ROOT ++ p.no_pivot_root = params->no_pivot_root; ++#endif + copy_process(&p, config->process); + copy_annotations(&p, config->annotations); + +diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c +index 4157c631..754c28ac 100644 +--- a/src/daemon/modules/service/service_container.c ++++ b/src/daemon/modules/service/service_container.c +@@ -743,6 +743,23 @@ static int do_oci_spec_update(const char *id, oci_runtime_spec *oci_spec, contai + return 0; + } + ++static bool pack_no_pivot_root(const container_t *cont) ++{ ++ size_t i = 0; ++ bool ret = false; ++ ++ ret = cont->hostconfig->no_pivot_root; ++ if (cont->common_config->config->annotations != NULL) { ++ for (i = 0; i < cont->common_config->config->annotations->len; i++) { ++ if (strcmp(cont->common_config->config->annotations->keys[i], "ISULAD_RAMDISK") == 0) { ++ ret = true; ++ break; ++ } ++ } ++ } ++ return ret; ++} ++ + static int do_start_container(container_t *cont, const char *console_fifos[], bool reset_rm, pid_ppid_info_t *pid_info) + { + int ret = 0; +@@ -906,6 +923,9 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo + create_params.exit_fifo = exit_fifo; + create_params.tty = tty; + create_params.open_stdin = open_stdin; ++#ifdef ENABLE_NO_PIVOT_ROOT ++ create_params.no_pivot_root = pack_no_pivot_root(cont); ++#endif + #ifdef ENABLE_CRI_API_V1 + if (cont->common_config->sandbox_info != NULL) { + create_params.task_addr = cont->common_config->sandbox_info->task_address; +-- +2.34.1 + diff --git a/0148-fix-issues-Isula-ps-cannot-display-port-mapping.patch b/0148-fix-issues-Isula-ps-cannot-display-port-mapping.patch new file mode 100644 index 0000000..1fee14a --- /dev/null +++ b/0148-fix-issues-Isula-ps-cannot-display-port-mapping.patch @@ -0,0 +1,227 @@ +From 6d5121eacdb42a4dd7c2ea498c534b893b80e405 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?=E6=AD=A6=E7=A7=AF=E8=B6=85?= <wujichao1@huawei.com> +Date: Thu, 7 Nov 2024 20:17:41 +0800 +Subject: [PATCH 148/156] fix issues: Isula ps cannot display port mapping + +--- + src/api/services/containers/container.proto | 1 + + .../connect/grpc/grpc_containers_client.cc | 2 + + src/client/connect/protocol_type.c | 4 ++ + src/client/connect/protocol_type.h | 1 + + src/cmd/isula/information/ps.c | 9 ++- + .../connect/grpc/container/list_service.cc | 3 + + src/daemon/executor/container_cb/list.c | 2 + + src/daemon/modules/api/container_api.h | 2 + + src/daemon/modules/container/container_unix.c | 56 ++++++++++++++++++- + 9 files changed, 77 insertions(+), 3 deletions(-) + +diff --git a/src/api/services/containers/container.proto b/src/api/services/containers/container.proto +index 8d05681e..113fcb10 100644 +--- a/src/api/services/containers/container.proto ++++ b/src/api/services/containers/container.proto +@@ -65,6 +65,7 @@ message Container { + string name = 16; + string health_state = 17; + int64 created = 18; ++ string ports = 19; + } + + message Container_info { +diff --git a/src/client/connect/grpc/grpc_containers_client.cc b/src/client/connect/grpc/grpc_containers_client.cc +index adeac0ba..f042290e 100644 +--- a/src/client/connect/grpc/grpc_containers_client.cc ++++ b/src/client/connect/grpc/grpc_containers_client.cc +@@ -1241,6 +1241,8 @@ private: + util_strdup_s("none"); + response->container_summary[index]->command = !in.command().empty() ? util_strdup_s(in.command().c_str()) : + util_strdup_s("-"); ++ response->container_summary[index]->ports = !in.ports().empty() ? util_strdup_s(in.ports().c_str()) : ++ nullptr; + const char *starttime = !in.startat().empty() ? in.startat().c_str() : "-"; + response->container_summary[index]->startat = util_strdup_s(starttime); + +diff --git a/src/client/connect/protocol_type.c b/src/client/connect/protocol_type.c +index 57dcf753..d0ab9732 100644 +--- a/src/client/connect/protocol_type.c ++++ b/src/client/connect/protocol_type.c +@@ -531,6 +531,10 @@ void isula_list_response_free(struct isula_list_response *response) + free(response->container_summary[i]->command); + response->container_summary[i]->command = NULL; + } ++ if (response->container_summary[i]->ports != NULL) { ++ free(response->container_summary[i]->ports); ++ response->container_summary[i]->ports = NULL; ++ } + if (response->container_summary[i]->startat != NULL) { + free(response->container_summary[i]->startat); + response->container_summary[i]->startat = NULL; +diff --git a/src/client/connect/protocol_type.h b/src/client/connect/protocol_type.h +index 2b445c5a..02e315bf 100644 +--- a/src/client/connect/protocol_type.h ++++ b/src/client/connect/protocol_type.h +@@ -204,6 +204,7 @@ struct isula_container_summary_info { + char *id; + char *image; + char *command; ++ char *ports; + char *name; + Container_Status status; + uint32_t exit_code; +diff --git a/src/cmd/isula/information/ps.c b/src/cmd/isula/information/ps.c +index 13c35dc7..6bdc157f 100644 +--- a/src/cmd/isula/information/ps.c ++++ b/src/cmd/isula/information/ps.c +@@ -332,7 +332,8 @@ static void print_basic_container_info_item(const struct isula_container_summary + } else if (strcmp(name, "Created") == 0) { + print_created_field(in->created, length->created_length); + } else if (strcmp(name, "Ports") == 0) { +- printf("%-*s", (int)length->ports_length, " "); ++ const char *ports = (in->ports != NULL ? in->ports : "N/A"); ++ printf("%-*s", (int)length->ports_length, ports); + } + } + +@@ -515,6 +516,12 @@ static void list_field_width(struct isula_container_summary_info **info, const s + l->command_length = (unsigned int)cmd_len; + } + } ++ if (in->ports != NULL) { ++ size_t ports_len = strlen(in->ports); ++ if (ports_len > l->ports_length) { ++ l->ports_length = (unsigned int)ports_len; ++ } ++ } + + calculate_str_length(in->name, &l->name_length); + calculate_str_length(in->runtime, &l->runtime_length); +diff --git a/src/daemon/entry/connect/grpc/container/list_service.cc b/src/daemon/entry/connect/grpc/container/list_service.cc +index e831acbc..dfd48191 100644 +--- a/src/daemon/entry/connect/grpc/container/list_service.cc ++++ b/src/daemon/entry/connect/grpc/container/list_service.cc +@@ -121,6 +121,9 @@ void ContainerListService::FillResponseTogRPC(void *containerRes, containers::Li + if (response->containers[i]->command != nullptr) { + container->set_command(response->containers[i]->command); + } ++ if (response->containers[i]->ports != nullptr) { ++ container->set_ports(response->containers[i]->ports); ++ } + container->set_exit_code(response->containers[i]->exit_code); + container->set_restartcount(response->containers[i]->restartcount); + if (response->containers[i]->startat != nullptr) { +diff --git a/src/daemon/executor/container_cb/list.c b/src/daemon/executor/container_cb/list.c +index 243eb4d2..2e8c7a79 100644 +--- a/src/daemon/executor/container_cb/list.c ++++ b/src/daemon/executor/container_cb/list.c +@@ -609,6 +609,8 @@ static int fill_container_info(container_container *container_info, const contai + + container_info->status = (int)container_state_judge_status(cont_state); + ++ container_info->ports = container_get_ports(cont); ++ + container_info->command = container_get_command(cont); + image = container_get_image(cont); + container_info->image = image ? image : util_strdup_s("none"); +diff --git a/src/daemon/modules/api/container_api.h b/src/daemon/modules/api/container_api.h +index a6ec8e22..e4d7fc25 100644 +--- a/src/daemon/modules/api/container_api.h ++++ b/src/daemon/modules/api/container_api.h +@@ -185,6 +185,8 @@ int container_v2_spec_merge_container_spec(container_config_v2_common_config *v2 + + char *container_get_command(const container_t *cont); + ++char *container_get_ports(const container_t *cont); ++ + char *container_get_image(const container_t *cont); + + int container_exit_on_next(container_t *cont); +diff --git a/src/daemon/modules/container/container_unix.c b/src/daemon/modules/container/container_unix.c +index 6f345ea0..812ab9e1 100644 +--- a/src/daemon/modules/container/container_unix.c ++++ b/src/daemon/modules/container/container_unix.c +@@ -20,6 +20,7 @@ + #include <isula_libutils/container_config_v2.h> + #include <isula_libutils/host_config.h> + #include <isula_libutils/json_common.h> ++#include <isula_libutils/auto_cleanup.h> + #include <limits.h> + #include <pthread.h> + #include <stdbool.h> +@@ -1253,7 +1254,7 @@ char *container_get_command(const container_t *cont) + if (cont->common_config->path != NULL) { + nret = util_array_append(&args, cont->common_config->path); + if (nret < 0) { +- ERROR("Appned string failed"); ++ ERROR("Append string failed"); + goto cleanup; + } + } +@@ -1268,7 +1269,7 @@ char *container_get_command(const container_t *cont) + nret = util_array_append(&args, arg); + free(arg); + if (nret < 0) { +- ERROR("Appned string failed"); ++ ERROR("Append string failed"); + goto cleanup; + } + } +@@ -1280,6 +1281,57 @@ cleanup: + return cmd; + } + ++/* container get ports */ ++char *container_get_ports(const container_t *cont) ++{ ++ int nret; ++ size_t i, j, ports_num = 0; ++ __isula_auto_array_t char **args = NULL; ++ char *ports = NULL; ++ ++ if (cont == NULL || cont->hostconfig == NULL || cont->hostconfig->port_bindings == NULL) { ++ return NULL; ++ } ++ ++ for (i = 0; cont->hostconfig->port_bindings->keys != NULL && i < cont->hostconfig->port_bindings->len; i++) { ++ for (j = 0; j < cont->hostconfig->port_bindings->values[i]->element->host_len; j++) { ++ __isula_auto_free char *arg = NULL; ++ char *host_ip = cont->hostconfig->port_bindings->values[i]->element->host[j]->host_ip; ++ if (host_ip == NULL) { ++ host_ip = "0.0.0.0"; ++ } ++ char *host_port = cont->hostconfig->port_bindings->values[i]->element->host[j]->host_port; ++ if (host_port == NULL) { ++ host_port = ""; ++ } ++ char *cont_port_type = cont->hostconfig->port_bindings->keys[i]; ++ if (cont_port_type == NULL) { ++ cont_port_type = ""; ++ } ++ size_t total_len = strlen(host_ip) + strlen(host_port) + strlen(cont_port_type) + 4; // 4 for ":->" and null terminator ++ arg = util_common_calloc_s(total_len); ++ if (arg == NULL) { ++ ERROR("Out of memory"); ++ return ports; ++ } ++ nret = snprintf(arg, total_len, "%s:%s->%s", host_ip, host_port, cont_port_type); ++ if (nret < 0 || (size_t)nret >= total_len) { ++ ERROR("Failed to print string"); ++ return ports; ++ } ++ nret = util_array_append(&args, arg); ++ if (nret < 0) { ++ ERROR("Append string failed"); ++ return ports; ++ } ++ ports_num++; ++ } ++ } ++ ports = util_string_join(", ", (const char **)args, ports_num); ++ ++ return ports; ++} ++ + /* container get image */ + char *container_get_image(const container_t *cont) + { +-- +2.34.1 + diff --git a/0149-move-nri-call-in-stop-and-remove-con.patch b/0149-move-nri-call-in-stop-and-remove-con.patch new file mode 100644 index 0000000..8c73f2d --- /dev/null +++ b/0149-move-nri-call-in-stop-and-remove-con.patch @@ -0,0 +1,87 @@ +From db60c64138b45539fe70282c853ac2dae5954924 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Thu, 14 Nov 2024 17:34:49 +0800 +Subject: [PATCH 149/156] move nri call in stop and remove con + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/common/cri/cri_helpers.cc | 18 ++++++++++++++++++ + .../cri/v1/v1_cri_container_manager_service.cc | 14 -------------- + 2 files changed, 18 insertions(+), 14 deletions(-) + +diff --git a/src/daemon/common/cri/cri_helpers.cc b/src/daemon/common/cri/cri_helpers.cc +index a8cbd996..aa8e3c19 100644 +--- a/src/daemon/common/cri/cri_helpers.cc ++++ b/src/daemon/common/cri/cri_helpers.cc +@@ -32,6 +32,10 @@ + #include "isulad_config.h" + #include "sha256.h" + ++#ifdef ENABLE_NRI ++#include "nri_adaption.h" ++#endif ++ + namespace CRIHelpers { + const std::string Constants::POD_NETWORK_ANNOTATION_KEY { "network.alpha.kubernetes.io/network" }; + const std::string Constants::CONTAINER_TYPE_LABEL_KEY { "cri.isulad.type" }; +@@ -660,6 +664,13 @@ void RemoveContainerHelper(service_executor_t *cb, const std::string &containerI + goto cleanup; + } + ++#ifdef ENABLE_NRI ++ if (!NRIAdaptation::GetInstance()->RemoveContainer(containerID, error)) { ++ ERROR("NRI RemoveContainer notification failed: %s", error.GetCMessage()); ++ } ++ error.Clear(); ++#endif ++ + if (cb->container.remove(request, &response) != 0) { + if (response != nullptr && response->errmsg != nullptr) { + error.SetError(response->errmsg); +@@ -719,6 +730,13 @@ void StopContainerHelper(service_executor_t *cb, const std::string &containerID, + error.SetError(msg); + } + ++#ifdef ENABLE_NRI ++ if (!NRIAdaptation::GetInstance()->StopContainer(containerID, error)) { ++ ERROR("NRI StopContainer notification failed: %s", error.GetCMessage()); ++ } ++ error.Clear(); ++#endif ++ + free_container_stop_request(request); + free_container_stop_response(response); + } +diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +index 1e84d14c..b585b49c 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +@@ -618,25 +618,11 @@ cleanup: + + void ContainerManagerService::StopContainer(const std::string &containerID, int64_t timeout, Errors &error) + { +-#ifdef ENABLE_NRI +- Errors nriErr; +-#endif + CRIHelpers::StopContainer(m_cb, containerID, timeout, error); +-#ifdef ENABLE_NRI +- if (!NRIAdaptation::GetInstance()->StopContainer(containerID, nriErr)) { +- ERROR("NRI StopContainer notification failed: %s", nriErr.GetCMessage()); +- } +-#endif + } + + void ContainerManagerService::RemoveContainer(const std::string &containerID, Errors &error) + { +-#ifdef ENABLE_NRI +- Errors nriErr; +- if (!NRIAdaptation::GetInstance()->RemoveContainer(containerID, nriErr)) { +- ERROR("NRI RemoveContainer notification failed: %s", nriErr.GetCMessage()); +- } +-#endif + CRIHelpers::RemoveContainer(m_cb, containerID, error); + if (error.NotEmpty()) { + WARN("Failed to remove container %s", containerID.c_str()); +-- +2.34.1 + diff --git a/0150-add-missing-con-linux-info-for-nri-module.patch b/0150-add-missing-con-linux-info-for-nri-module.patch new file mode 100644 index 0000000..7717193 --- /dev/null +++ b/0150-add-missing-con-linux-info-for-nri-module.patch @@ -0,0 +1,548 @@ +From 140166e0f385a2f23502efeeba4113536736c3c8 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Thu, 14 Nov 2024 16:45:08 +0800 +Subject: [PATCH 150/156] add missing con linux info for nri module + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/common/nri/nri_convert.cc | 482 ++++++++++++++++++++++++++- + 1 file changed, 480 insertions(+), 2 deletions(-) + +diff --git a/src/daemon/common/nri/nri_convert.cc b/src/daemon/common/nri/nri_convert.cc +index 30caf1dd..d862d992 100644 +--- a/src/daemon/common/nri/nri_convert.cc ++++ b/src/daemon/common/nri/nri_convert.cc +@@ -15,12 +15,16 @@ + + #include "nri_convert.h" + ++#include <sys/stat.h> ++#include <sys/sysmacros.h> ++ + #include "container_api.h" + #include "v1_cri_helpers.h" + #include "path.h" + #include "transform.h" + #include "nri_utils.h" + #include "cstruct_wrapper.h" ++#include "specs_api.h" + + static int64_t DefaultOOMScoreAdj = 0; + +@@ -385,6 +389,462 @@ error_out: + return false; + } + ++static int ConvertDevice(const char *host_path, const char *container_path, const char *permissions, ++ nri_linux_device &device, nri_linux_device_cgroup &deviceCgroup) ++{ ++ int ret = 0; ++ struct stat st; ++ const char *dev_type = NULL; ++ unsigned int file_mode = 0; ++ ++ if (host_path == NULL) { ++ return -1; ++ } ++ ++ ret = stat(host_path, &st); ++ if (ret < 0) { ++ ERROR("device %s no exists", host_path); ++ return -1; ++ } ++ ++ file_mode = st.st_mode & 0777; ++ ++ /* check device type first */ ++ if (S_ISBLK(st.st_mode)) { ++ file_mode |= S_IFBLK; ++ dev_type = "b"; ++ } else if (S_ISCHR(st.st_mode)) { ++ file_mode |= S_IFCHR; ++ dev_type = "c"; ++ } else { ++ ERROR("Cannot determine the device number for device %s", host_path); ++ return -1; ++ } ++ ++ /* fill spec dev */ ++ device.major = (int64_t)major(st.st_rdev); ++ device.minor = (int64_t)minor(st.st_rdev); ++ device.uid = (uint32_t *)util_common_calloc_s(sizeof(uint32_t*)); ++ if (device.uid == nullptr) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ *(device.uid) = st.st_uid; ++ device.gid = (uint32_t *)util_common_calloc_s(sizeof(uint32_t*)); ++ if (device.gid == nullptr) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ *(device.gid) = st.st_gid; ++ device.file_mode = (uint32_t *)util_common_calloc_s(sizeof(uint32_t)); ++ if (device.file_mode == nullptr) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ *(device.file_mode) = (int)file_mode; ++ device.type = util_strdup_s(dev_type); ++ device.path = util_strdup_s(container_path); ++ ++ /* fill spec cgroup dev */ ++ deviceCgroup.allow = true; ++ deviceCgroup.access = util_strdup_s(permissions); ++ deviceCgroup.type = util_strdup_s(dev_type); ++ deviceCgroup.major = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (deviceCgroup.major == nullptr) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ *(deviceCgroup.major) = (int64_t)major(st.st_rdev); ++ deviceCgroup.minor = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (deviceCgroup.minor == nullptr) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ *(deviceCgroup.minor) = (int64_t)minor(st.st_rdev); ++ ++ return 0; ++} ++ ++static int ConvertHostConfigDevices(const host_config_devices_element *dev_map, nri_linux_device &device, ++ nri_linux_device_cgroup &deviceCgroup) ++{ ++ return ConvertDevice(dev_map->path_on_host, dev_map->path_in_container, ++ dev_map->cgroup_permissions, device, deviceCgroup); ++} ++ ++static int ConLinuxDeviceToNRI(const host_config *config, nri_container &con) ++{ ++ size_t i; ++ ++ if (config->devices_len == 0 && config->nri_devices_len == 0) { ++ return 0; ++ } ++ con.linux->devices = (nri_linux_device **)util_smart_calloc_s(sizeof(nri_linux_device *), ++ config->devices_len + config->nri_devices_len); ++ if (con.linux->devices == nullptr) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ ++ con.linux->resources->devices = (nri_linux_device_cgroup **)util_smart_calloc_s(sizeof(nri_linux_device_cgroup *), ++ config->devices_len); ++ if (con.linux->resources->devices == nullptr) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ ++ for (i = 0; i < config->devices_len; i++) { ++ nri_linux_device *device = (nri_linux_device *)util_common_calloc_s(sizeof(nri_linux_device)); ++ if (device == nullptr) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ ++ nri_linux_device_cgroup *deviceCgroup = (nri_linux_device_cgroup *)util_common_calloc_s(sizeof( ++ nri_linux_device_cgroup)); ++ if (deviceCgroup == nullptr) { ++ ERROR("Out of memory"); ++ free_nri_linux_device(device); ++ return -1; ++ } ++ ++ if (ConvertHostConfigDevices(config->devices[i], *device, *deviceCgroup) != 0) { ++ ERROR("Failed to convert host config devices"); ++ free_nri_linux_device(device); ++ free_nri_linux_device_cgroup(deviceCgroup); ++ return -1; ++ } ++ ++ con.linux->devices[i] = device; ++ con.linux->resources->devices[i] = deviceCgroup; ++ con.linux->devices_len++; ++ con.linux->resources->devices_len++; ++ } ++ ++ for (i = 0; i < config->nri_devices_len; i++) { ++ nri_linux_device *device = (nri_linux_device *)util_common_calloc_s(sizeof(nri_linux_device)); ++ if (device == nullptr) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ ++ device->file_mode = (uint32_t *)util_common_calloc_s(sizeof(uint32_t)); ++ if (device->file_mode == nullptr) { ++ ERROR("Out of memory"); ++ free_nri_linux_device(device); ++ return -1; ++ } ++ *(device->file_mode) = config->nri_devices[i]->file_mode; ++ ++ device->path = util_strdup_s(config->nri_devices[i]->path); ++ device->type = util_strdup_s(config->nri_devices[i]->type); ++ device->major = config->nri_devices[i]->major; ++ device->minor = config->nri_devices[i]->minor; ++ ++ device->uid = (uint32_t *)util_common_calloc_s(sizeof(uint32_t)); ++ if (device->uid == nullptr) { ++ ERROR("Out of memory"); ++ free_nri_linux_device(device); ++ return -1; ++ } ++ *(device->uid) = config->nri_devices[i]->uid; ++ ++ device->gid = (uint32_t *)util_common_calloc_s(sizeof(uint32_t)); ++ if (device->gid == nullptr) { ++ ERROR("Out of memory"); ++ free_nri_linux_device(device); ++ return -1; ++ } ++ *(device->gid) = config->nri_devices[i]->gid; ++ con.linux->devices[i + config->devices_len] = device; ++ con.linux->devices_len++; ++ } ++ ++ return 0; ++} ++ ++static int ConvertCRIV1Devices(const ::runtime::v1::Device &dev_map, nri_linux_device &device, ++ nri_linux_device_cgroup &deviceCgroup) ++{ ++ return ConvertDevice(dev_map.host_path().c_str(), dev_map.container_path().c_str(), ++ dev_map.permissions().c_str(), device, deviceCgroup); ++} ++ ++static bool ConLinuxResourcesCpuToNRI(const host_config *config, nri_linux_cpu &cpu) ++{ ++ cpu.shares = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if (cpu.shares == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(cpu.shares) = config->cpu_shares; ++ ++ cpu.quota = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (cpu.quota == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(cpu.quota) = config->cpu_quota; ++ ++ cpu.period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if (cpu.period == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(cpu.period) = config->cpu_period; ++ ++ cpu.cpus = util_strdup_s(config->cpuset_cpus); ++ cpu.mems = util_strdup_s(config->cpuset_mems); ++ ++ return true; ++} ++ ++static bool ConLinuxResourcesMemoryToNRI(const host_config *config, nri_linux_memory &memory) ++{ ++ memory.limit = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (memory.limit == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(memory.limit) = config->memory; ++ ++ memory.reservation = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (memory.reservation == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ *(memory.reservation) = config->memory_reservation; ++ ++ memory.swap = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (memory.swap == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(memory.swap) = config->memory_swap; ++ ++ memory.kernel = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (memory.kernel == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(memory.kernel) = config->kernel_memory; ++ ++ // isulad has not set kernel_tcp ++ memory.kernel_tcp = nullptr; ++ ++ if (config->memory_swappiness != nullptr) { ++ memory.swappiness = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if (memory.swappiness == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(memory.swappiness) = *(config->memory_swappiness); ++ } ++ ++ memory.disable_oom_killer = (uint8_t *)util_common_calloc_s(sizeof(uint8_t)); ++ if (memory.disable_oom_killer == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(memory.disable_oom_killer) = config->oom_kill_disable; ++ ++ // isulad has not set use_hierarchy ++ memory.use_hierarchy = (uint8_t *)util_common_calloc_s(sizeof(uint8_t)); ++ if (memory.use_hierarchy == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(memory.use_hierarchy) = false; ++ return true; ++} ++ ++auto ConLinuxResourcesToNRI(const host_config *config) -> nri_linux_resources * ++{ ++ nri_linux_resources *resources = nullptr; ++ size_t i; ++ ++ resources = init_nri_linux_resources(); ++ if (resources == nullptr) { ++ ERROR("Failed to init nri linux resources"); ++ return nullptr; ++ } ++ ++ if (!ConLinuxResourcesCpuToNRI(config, *resources->cpu)) { ++ ERROR("Failed to transform cpu to nri"); ++ goto error_out; ++ } ++ ++ if (!ConLinuxResourcesMemoryToNRI(config, *resources->memory)) { ++ ERROR("Failed to transform memory to nri"); ++ goto error_out; ++ } ++ ++ resources->hugepage_limits = (nri_hugepage_limit **)util_smart_calloc_s(sizeof(nri_hugepage_limit *), ++ config->hugetlbs_len); ++ if (resources->hugepage_limits == nullptr) { ++ ERROR("Out of memory"); ++ goto error_out; ++ } ++ ++ for (i = 0; i < config->hugetlbs_len; i++) { ++ resources->hugepage_limits[i] = (nri_hugepage_limit *)util_common_calloc_s(sizeof(nri_hugepage_limit)); ++ if (resources->hugepage_limits[i] == nullptr) { ++ ERROR("Out of memory"); ++ goto error_out; ++ } ++ resources->hugepage_limits[i]->page_size = util_strdup_s(config->hugetlbs[i]->page_size); ++ resources->hugepage_limits[i]->limit = config->hugetlbs[i]->limit; ++ resources->hugepage_limits_len++; ++ } ++ ++ // resources.blockio_class is not support ++ // resources.rdt_class is not support ++ // They are not standard fields in oci spec ++ ++ if (dup_json_map_string_string(config->unified, resources->unified) != 0) { ++ ERROR("Failed to copy unified map"); ++ goto error_out; ++ } ++ ++ // resources.devices is set in ConLinuxDeviceToNRI ++ ++ return resources; ++ ++error_out: ++ free_nri_linux_resources(resources); ++ resources = nullptr; ++ return resources; ++} ++ ++static bool ConLinuxToNRI(const char *id, const host_config *config, nri_container &con) ++{ ++ con.linux = (nri_linux_container *)util_common_calloc_s(sizeof(nri_linux_container)); ++ if (con.linux == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ con.linux->resources = ConLinuxResourcesToNRI(config); ++ if (con.linux->resources == nullptr) { ++ ERROR("Failed to transform resources to nri for con : %s", id); ++ return false; ++ } ++ ++ if (ConLinuxDeviceToNRI(config, con) != 0) { ++ ERROR("Failed to transform devices to nri for con : %s", id); ++ return false; ++ } ++ ++ con.linux->oom_score_adj = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (con.linux->oom_score_adj == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ *(con.linux->oom_score_adj) = config->oom_score_adj; ++ ++ con.linux->cgroups_path = merge_container_cgroups_path(id, config); ++ if (con.linux->cgroups_path == NULL) { ++ WARN("nri container cgroups path is NULL"); ++ } ++ return true; ++} ++ ++static int ConConfigLinuxDeviceToNRI(const runtime::v1::ContainerConfig &containerConfig, nri_container &con) ++{ ++ int i; ++ int conConfigDevicesSize = containerConfig.devices_size(); ++ ++ if (conConfigDevicesSize == 0) { ++ return 0; ++ } ++ con.linux->devices = (nri_linux_device **)util_smart_calloc_s(sizeof(nri_linux_device *), conConfigDevicesSize); ++ if (con.linux->devices == nullptr) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ ++ if (con.linux->resources == nullptr) { ++ con.linux->resources = init_nri_linux_resources(); ++ if (con.linux->resources == nullptr) { ++ ERROR("Failed to init nri linux resources"); ++ return -1; ++ } ++ } ++ ++ con.linux->resources->devices = (nri_linux_device_cgroup **)util_smart_calloc_s(sizeof(nri_linux_device_cgroup *), ++ conConfigDevicesSize); ++ if (con.linux->resources->devices == nullptr) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ ++ for (i = 0; i < conConfigDevicesSize; i++) { ++ nri_linux_device *device = (nri_linux_device *)util_common_calloc_s(sizeof(nri_linux_device)); ++ if (device == nullptr) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ ++ nri_linux_device_cgroup *deviceCgroup = (nri_linux_device_cgroup *)util_common_calloc_s(sizeof( ++ nri_linux_device_cgroup)); ++ if (deviceCgroup == nullptr) { ++ ERROR("Out of memory"); ++ free_nri_linux_device(device); ++ return -1; ++ } ++ ++ if (ConvertCRIV1Devices(containerConfig.devices(i), *device, *deviceCgroup) != 0) { ++ ERROR("Failed to convert CRI v1 devices"); ++ free_nri_linux_device(device); ++ free_nri_linux_device_cgroup(deviceCgroup); ++ return -1; ++ } ++ ++ con.linux->devices[i] = device; ++ con.linux->resources->devices[i] = deviceCgroup; ++ con.linux->devices_len++; ++ con.linux->resources->devices_len++; ++ } ++ ++ return 0; ++} ++ ++static bool ConConfigLinuxToNRI(const runtime::v1::ContainerConfig &containerConfig, nri_container &con) ++{ ++ const char *name = containerConfig.metadata().name().c_str(); ++ con.linux = (nri_linux_container *)util_common_calloc_s(sizeof(nri_linux_container)); ++ if (con.linux == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ if (containerConfig.has_linux() && containerConfig.linux().has_resources()) { ++ con.linux->resources = LinuxResourcesToNRI(containerConfig.linux().resources()); ++ if (con.linux->resources == nullptr) { ++ ERROR("Failed to transform resources to nri for con : %s", name); ++ return false; ++ } ++ ++ con.linux->oom_score_adj = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (con.linux->oom_score_adj == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(con.linux->oom_score_adj) = containerConfig.linux().resources().oom_score_adj(); ++ } ++ ++ if (ConConfigLinuxDeviceToNRI(containerConfig, con) != 0) { ++ ERROR("Failed to convert devices to nri for con : %s", name); ++ return false; ++ } ++ ++ // ContainerToNRIByConfig is called when CreateContainer, and cannot get pid at this time ++ con.linux->cgroups_path = NULL; ++ return true; ++} ++ + // container info is incomplete because container in excution is not created + auto ContainerToNRIByConConfig(const runtime::v1::ContainerConfig &containerConfig, nri_container &con) -> bool + { +@@ -395,6 +855,9 @@ auto ContainerToNRIByConConfig(const runtime::v1::ContainerConfig &containerConf + + Errors tmpError; + ++ // ContainerToNRIByConfig is called when CreateConatiner, and the status is 0(CONTAINER_UNKNOWN) at this time ++ con.state = 0; ++ + con.labels = Transform::ProtobufMapToJsonMapForString(containerConfig.labels(), tmpError); + if (con.labels == nullptr) { + ERROR("Failed to transform labels to nri for con : %s, : %s", con.name, tmpError.GetMessage().c_str()); +@@ -426,9 +889,18 @@ auto ContainerToNRIByConConfig(const runtime::v1::ContainerConfig &containerConf + ERROR("Failed to transform mounts to nri for con : %s", con.name); + return false; + } +- return true; + +- // todo: can not get container hooks and pid from containerConfig ++ if (!ConConfigLinuxToNRI(containerConfig, con)) { ++ ERROR("Failed to convert conatiner linux info to nri for con : %s", con.name); ++ return false; ++ } ++ ++ // todo: CRI module can not get container hooks from containerConfig ++ // ContainerToNRIByConfig is called when CreateConatiner, and cannot get pid at this time ++ ++ // rlimit not support in containerd ++ ++ return true; + } + + // container info is incomplete because container in excution is not created +@@ -486,6 +958,11 @@ auto ContainerToNRIByID(const std::string &id, nri_container &con) -> bool + goto out; + } + ++ if (!ConLinuxToNRI(cont->common_config->id, cont->hostconfig, con)) { ++ ERROR("Failed to transform conatiner linux info to nri for con : %s", con.name); ++ goto out; ++ } ++ + // todo: can convert hostconfig's hook_spec to nri spec + + con.pid = container_state_get_pid(cont->state); +@@ -644,6 +1121,7 @@ auto ContainersToNRI(std::vector<std::unique_ptr<runtime::v1::Container>> &conta + } + if (!ContainerToNRIByID(containers[i].get()->id(), *con)) { + ERROR("Failed to transform container to nri for container : %s", containers[i]->metadata().name().c_str()); ++ free_nri_container(con); + return false; + } + cons.push_back(con); +-- +2.34.1 + diff --git a/0151-sandbox-sandbox-api-update.patch b/0151-sandbox-sandbox-api-update.patch new file mode 100644 index 0000000..f925ccc --- /dev/null +++ b/0151-sandbox-sandbox-api-update.patch @@ -0,0 +1,1600 @@ +From b994e99a4188bef549e5fa1f944eb3546be43201 Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Wed, 18 Sep 2024 11:40:15 +0800 +Subject: [PATCH 151/156] sandbox:sandbox api update + +Signed-off-by: liuxu <liuxu156@huawei.com> +--- + cmake/protoc.cmake | 1 + + src/api/services/sandbox/sandbox.proto | 107 +++--- + .../sandbox/sandbox/types/metrics.proto | 30 ++ + .../sandbox/sandbox/types/platform.proto | 3 +- + .../sandbox/sandbox/types/sandbox.proto | 15 +- + src/daemon/sandbox/controller/controller.h | 12 +- + .../sandboxer/client/grpc_sandboxer_client.cc | 111 ++---- + .../sandboxer/client/grpc_sandboxer_client.h | 18 +- + .../sandboxer/sandboxer_controller.cc | 19 +- + .../sandboxer/sandboxer_controller.h | 9 +- + .../controller/shim/shim_controller.cc | 11 +- + .../sandbox/controller/shim/shim_controller.h | 9 +- + src/daemon/sandbox/sandbox.cc | 221 +++++++++++- + src/daemon/sandbox/sandbox.h | 24 ++ + src/daemon/sandbox/sandbox_ops.cc | 333 ++++++++++++++---- + src/daemon/sandbox/sandbox_task.cc | 99 ++++++ + src/daemon/sandbox/sandbox_task.h | 48 +++ + 17 files changed, 808 insertions(+), 262 deletions(-) + create mode 100644 src/api/services/sandbox/sandbox/types/metrics.proto + create mode 100644 src/daemon/sandbox/sandbox_task.cc + create mode 100644 src/daemon/sandbox/sandbox_task.h + +diff --git a/cmake/protoc.cmake b/cmake/protoc.cmake +index 6e2d1b84..6343fe3e 100644 +--- a/cmake/protoc.cmake ++++ b/cmake/protoc.cmake +@@ -72,6 +72,7 @@ if (ENABLE_CRI_API_V1 AND ENABLE_SANDBOXER) + PROTOC_CPP_GEN(sandbox ${SANDBOX_PROTOS_OUT_PATH} ${PROTOS_PATH}/sandbox/sandbox/types/sandbox.proto) + PROTOC_CPP_GEN(sandbox ${SANDBOX_PROTOS_OUT_PATH} ${PROTOS_PATH}/sandbox/sandbox/types/mount.proto) + PROTOC_CPP_GEN(sandbox ${SANDBOX_PROTOS_OUT_PATH} ${PROTOS_PATH}/sandbox/sandbox/types/platform.proto) ++ PROTOC_CPP_GEN(sandbox ${SANDBOX_PROTOS_OUT_PATH} ${PROTOS_PATH}/sandbox/sandbox/types/metrics.proto) + PROTOC_CPP_GEN(sandbox ${SANDBOX_PROTOS_OUT_PATH} ${PROTOS_PATH}/sandbox/sandbox.proto) + PROTOC_GRPC_GEN(sandbox ${SANDBOX_PROTOS_OUT_PATH} ${PROTOS_PATH}/sandbox/sandbox.proto) + endif() +diff --git a/src/api/services/sandbox/sandbox.proto b/src/api/services/sandbox/sandbox.proto +index 87c7e27c..dcc78444 100644 +--- a/src/api/services/sandbox/sandbox.proto ++++ b/src/api/services/sandbox/sandbox.proto +@@ -31,6 +31,7 @@ import "google/protobuf/timestamp.proto"; + import "sandbox/types/sandbox.proto"; + import "sandbox/types/mount.proto"; + import "sandbox/types/platform.proto"; ++import "sandbox/types/metrics.proto"; + + option go_package = "github.com/containerd/containerd/api/services/sandbox/v1;sandbox"; + +@@ -89,21 +90,22 @@ service Controller { + rpc Create(ControllerCreateRequest) returns (ControllerCreateResponse); + rpc Start(ControllerStartRequest) returns (ControllerStartResponse); + rpc Platform(ControllerPlatformRequest) returns (ControllerPlatformResponse); +- rpc Prepare(PrepareRequest) returns (PrepareResponse); +- rpc Purge(PurgeRequest) returns (PurgeResponse); +- rpc UpdateResources(UpdateResourcesRequest) returns (UpdateResourcesResponse); + rpc Stop(ControllerStopRequest) returns (ControllerStopResponse); + rpc Wait(ControllerWaitRequest) returns (ControllerWaitResponse); + rpc Status(ControllerStatusRequest) returns (ControllerStatusResponse); + rpc Shutdown(ControllerShutdownRequest) returns (ControllerShutdownResponse); ++ rpc Metrics(ControllerMetricsRequest) returns (ControllerMetricsResponse); ++ rpc Update(ControllerUpdateRequest) returns (ControllerUpdateResponse); + } + + message ControllerCreateRequest { +- string sandboxer = 1; +- string sandbox_id = 2; +- repeated containerd.types.Mount rootfs = 3; +- google.protobuf.Any options = 4; +- string netns_path = 5; ++ string sandbox_id = 1; ++ repeated containerd.types.Mount rootfs = 2; ++ google.protobuf.Any options = 3; ++ string netns_path = 4; ++ map<string, string> annotations = 5; ++ containerd.types.Sandbox sandbox = 6; ++ string sandboxer = 10; + } + + message ControllerCreateResponse { +@@ -111,8 +113,8 @@ message ControllerCreateResponse { + } + + message ControllerStartRequest { +- string sandboxer = 1; +- string sandbox_id = 2; ++ string sandbox_id = 1; ++ string sandboxer = 10; + } + + message ControllerStartResponse { +@@ -120,12 +122,16 @@ message ControllerStartResponse { + uint32 pid = 2; + google.protobuf.Timestamp created_at = 3; + map<string, string> labels = 4; +- string task_address = 5; ++ // Address of the sandbox for containerd to connect, ++ // for calling Task or other APIs serving in the sandbox. ++ // it is in the form of ttrpc+unix://path/to/uds or grpc+vsock://<vsock cid>:<port>. ++ string address = 5; ++ uint32 version = 6; + } + + message ControllerPlatformRequest { +- string sandboxer = 1; +- string sandbox_id = 2; ++ string sandbox_id = 1; ++ string sandboxer = 10; + } + + message ControllerPlatformResponse { +@@ -133,16 +139,16 @@ message ControllerPlatformResponse { + } + + message ControllerStopRequest { +- string sandboxer = 1; +- string sandbox_id = 2; +- uint32 timeout_secs = 3; ++ string sandbox_id = 1; ++ uint32 timeout_secs = 2; ++ string sandboxer = 10; + } + + message ControllerStopResponse {} + + message ControllerWaitRequest { +- string sandboxer = 1; +- string sandbox_id = 2; ++ string sandbox_id = 1; ++ string sandboxer = 10; + } + + message ControllerWaitResponse { +@@ -151,61 +157,48 @@ message ControllerWaitResponse { + } + + message ControllerStatusRequest { +- string sandboxer = 1; +- string sandbox_id = 2; +- bool verbose = 3; ++ string sandbox_id = 1; ++ bool verbose = 2; ++ string sandboxer = 10; + } + + message ControllerStatusResponse { + string sandbox_id = 1; + uint32 pid = 2; + string state = 3; +- string task_address = 4; +- map<string, string> info = 5; +- google.protobuf.Timestamp created_at = 6; +- google.protobuf.Timestamp exited_at = 7; +- google.protobuf.Any extra = 8; ++ map<string, string> info = 4; ++ google.protobuf.Timestamp created_at = 5; ++ google.protobuf.Timestamp exited_at = 6; ++ google.protobuf.Any extra = 7; ++ // Address of the sandbox for containerd to connect, ++ // for calling Task or other APIs serving in the sandbox. ++ // it is in the form of ttrpc+unix://path/to/uds or grpc+vsock://<vsock cid>:<port>. ++ string address = 8; ++ uint32 version = 9; + } + + message ControllerShutdownRequest { +- string sandboxer = 1; +- string sandbox_id = 2; ++ string sandbox_id = 1; ++ string sandboxer = 10; + } + + message ControllerShutdownResponse {} + +-message PrepareRequest { +- string sandboxer = 1; +- string sandbox_id = 2; +- string container_id = 3; +- string exec_id = 4; +- google.protobuf.Any spec = 5; +- repeated containerd.types.Mount rootfs = 6; +- string stdin = 7; +- string stdout = 8; +- string stderr = 9; +- bool terminal = 10; +-} +- +-message PrepareResponse { +- string bundle = 1; ++message ControllerMetricsRequest { ++ string sandbox_id = 1; ++ string sandboxer = 10; + } + +-message PurgeRequest { +- string sandboxer = 1; +- string sandbox_id = 2; +- string container_id = 3; +- string exec_id = 4; ++message ControllerMetricsResponse { ++ types.Metric metrics = 1; + } + +-message PurgeResponse {} +- +-message UpdateResourcesRequest { +- string sandboxer = 1; +- string sandbox_id = 2; +- string container_id = 3; +- google.protobuf.Any resources = 4; +- map<string, string> annotations = 5; ++message ControllerUpdateRequest { ++ string sandbox_id = 1; ++ string sandboxer = 2; ++ containerd.types.Sandbox sandbox = 3; ++ repeated string fields = 4; + } + +-message UpdateResourcesResponse {} ++message ControllerUpdateResponse { ++} +\ No newline at end of file +diff --git a/src/api/services/sandbox/sandbox/types/metrics.proto b/src/api/services/sandbox/sandbox/types/metrics.proto +new file mode 100644 +index 00000000..61185939 +--- /dev/null ++++ b/src/api/services/sandbox/sandbox/types/metrics.proto +@@ -0,0 +1,30 @@ ++/* ++ Copyright The containerd Authors. ++ ++ Licensed under the Apache License, Version 2.0 (the "License"); ++ you may not use this file except in compliance with the License. ++ You may obtain a copy of the License at ++ ++ http://www.apache.org/licenses/LICENSE-2.0 ++ ++ Unless required by applicable law or agreed to in writing, software ++ distributed under the License is distributed on an "AS IS" BASIS, ++ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ See the License for the specific language governing permissions and ++ limitations under the License. ++*/ ++ ++syntax = "proto3"; ++ ++package containerd.types; ++ ++import "google/protobuf/any.proto"; ++import "google/protobuf/timestamp.proto"; ++ ++option go_package = "github.com/containerd/containerd/api/types;types"; ++ ++message Metric { ++ google.protobuf.Timestamp timestamp = 1; ++ string id = 2; ++ google.protobuf.Any data = 3; ++} +\ No newline at end of file +diff --git a/src/api/services/sandbox/sandbox/types/platform.proto b/src/api/services/sandbox/sandbox/types/platform.proto +index b6088251..102e6e2b 100644 +--- a/src/api/services/sandbox/sandbox/types/platform.proto ++++ b/src/api/services/sandbox/sandbox/types/platform.proto +@@ -26,4 +26,5 @@ message Platform { + string os = 1; + string architecture = 2; + string variant = 3; +-} ++ string os_version = 4; ++} +\ No newline at end of file +diff --git a/src/api/services/sandbox/sandbox/types/sandbox.proto b/src/api/services/sandbox/sandbox/types/sandbox.proto +index 7b9d196b..6fe08d40 100644 +--- a/src/api/services/sandbox/sandbox/types/sandbox.proto ++++ b/src/api/services/sandbox/sandbox/types/sandbox.proto +@@ -40,14 +40,15 @@ message Sandbox { + // Spec is sandbox configuration (kin of OCI runtime spec), spec's data will be written to a config.json file in the + // bundle directory (similary to OCI spec). + google.protobuf.Any spec = 3; +- // Sandboxer is the name of the sandbox controller who manages the sandbox. +- string sandboxer = 4; + // Labels provides an area to include arbitrary data on containers. +- map<string, string> labels = 5; ++ map<string, string> labels = 4; + // CreatedAt is the time the container was first created. +- google.protobuf.Timestamp created_at = 6; ++ google.protobuf.Timestamp created_at = 5; + // UpdatedAt is the last time the container was mutated. +- google.protobuf.Timestamp updated_at = 7; ++ google.protobuf.Timestamp updated_at = 6; + // Extensions allow clients to provide optional blobs that can be handled by runtime. +- map<string, google.protobuf.Any> extensions = 8; +-} ++ map<string, google.protobuf.Any> extensions = 7; ++ // Sandboxer is the name of the sandbox controller who manages the sandbox. ++ string sandboxer = 10; ++ ++} +\ No newline at end of file +diff --git a/src/daemon/sandbox/controller/controller.h b/src/daemon/sandbox/controller/controller.h +index 9ad45855..60d2dee5 100644 +--- a/src/daemon/sandbox/controller/controller.h ++++ b/src/daemon/sandbox/controller/controller.h +@@ -24,6 +24,7 @@ + + #include "errors.h" + #include "api_v1.pb.h" ++#include "sandbox.pb.h" + + namespace sandbox { + +@@ -65,6 +66,7 @@ struct ControllerSandboxInfo { + uint32_t pid; + uint64_t createdAt; + std::string taskAddress; ++ std::string version; + google::protobuf::Map<std::string, std::string> labels; + }; + +@@ -78,6 +80,7 @@ struct ControllerSandboxStatus { + uint32_t pid; + std::string state; + std::string taskAddress; ++ std::string version; + google::protobuf::Map<std::string, std::string> info; + uint64_t createdAt; + uint64_t exitedAt; +@@ -123,11 +126,10 @@ public: + Errors &error) = 0; + virtual std::unique_ptr<ControllerSandboxInfo> Start(const std::string &sandboxId, Errors &error) = 0 ; + virtual std::unique_ptr<ControllerPlatformInfo> Platform(const std::string &sandboxId, Errors &error) = 0; +- virtual std::string Prepare(const std::string &sandboxId, +- const ControllerPrepareParams ¶ms, +- Errors &error) = 0; +- virtual bool Purge(const std::string &sandboxId, const std::string &containerId, +- const std::string &execId, Errors &error) = 0; ++ virtual bool Prepare(containerd::types::Sandbox &apiSandbox, ++ std::vector<std::string> &fields, Errors &error) = 0; ++ virtual bool Purge(containerd::types::Sandbox &apiSandbox, ++ std::vector<std::string> &fields, Errors &error) = 0; + virtual bool UpdateResources(const std::string &sandboxId, + const ControllerUpdateResourcesParams ¶ms, + Errors &error) = 0; +diff --git a/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_client.cc b/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_client.cc +index 11c2b014..e042ad45 100644 +--- a/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_client.cc ++++ b/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_client.cc +@@ -74,6 +74,7 @@ auto SandboxerClient::InitCreateRequest(containerd::services::sandbox::v1::Contr + } + } + request.set_netns_path(params.netNSPath); ++ // The arg sandbox is useless for now + return true; + } + +@@ -117,7 +118,8 @@ void SandboxerClient::StartResponseToSandboxInfo(const containerd::services::san + sandboxInfo.id = response.sandbox_id(); + sandboxInfo.pid = response.pid(); + sandboxInfo.createdAt = TimestampToNanos(response.created_at()); +- sandboxInfo.taskAddress = response.task_address(); ++ sandboxInfo.taskAddress = response.address(); ++ sandboxInfo.version = response.version(); + sandboxInfo.labels = response.labels(); + } + +@@ -144,53 +146,27 @@ auto SandboxerClient::Start(const std::string &sandboxId, ControllerSandboxInfo + return true; + } + +-auto SandboxerClient::InitPrepareRequest(containerd::services::sandbox::v1::PrepareRequest &request, +- const std::string &sandboxId, const ControllerPrepareParams ¶ms) -> bool ++void SandboxerClient::InitUpdateRequest(containerd::services::sandbox::v1::ControllerUpdateRequest &request, ++ containerd::types::Sandbox &apiSandbox, ++ std::vector<std::string> &fields) + { +- if (params.spec == nullptr) { +- ERROR("Sandboxer controller prepare request failed, spec is null"); +- return false; +- } +- request.mutable_spec()->set_value(*(params.spec)); +- request.set_sandboxer(m_sandboxer); +- request.set_sandbox_id(sandboxId); +- request.set_container_id(params.containerId); +- request.set_exec_id(params.execId); +- for (const auto &entry : params.rootfs) { +- if (entry != nullptr) { +- Mount* mount = request.add_rootfs(); +- InitMountInfo(*mount, *entry); +- } +- } +- if (params.streamInfo != nullptr) { +- request.set_stdin(params.streamInfo->stdin); +- request.set_stdout(params.streamInfo->stdout); +- request.set_stderr(params.streamInfo->stderr); +- request.set_terminal(params.streamInfo->terminal); +- } else { +- request.set_stdin(""); +- request.set_stdout(""); +- request.set_stderr(""); +- request.set_terminal(false); +- } +- +- return true; ++ request.set_sandbox_id(apiSandbox.sandbox_id()); ++ request.set_sandboxer(apiSandbox.sandboxer()); ++ *(request.mutable_sandbox()) = apiSandbox; ++ *(request.mutable_fields()) = {fields.begin(), fields.end()}; + } + +-auto SandboxerClient::Prepare(const std::string &sandboxId, const ControllerPrepareParams ¶ms, std::string &bundle, +- Errors &error) -> bool ++auto SandboxerClient::Prepare(containerd::types::Sandbox &apiSandbox, ++ std::vector<std::string> &fields, Errors &error) -> bool + { + grpc::ClientContext context; +- containerd::services::sandbox::v1::PrepareRequest request; +- containerd::services::sandbox::v1::PrepareResponse response; ++ containerd::services::sandbox::v1::ControllerUpdateRequest request; ++ containerd::services::sandbox::v1::ControllerUpdateResponse response; + grpc::Status status; + +- if (!InitPrepareRequest(request, sandboxId, params)) { +- error.SetError("Failed to init prepare request for sandboxer prepare request, sandbox id: " + sandboxId); +- return false; +- } ++ InitUpdateRequest(request, apiSandbox, fields); + +- status = m_stub->Prepare(&context, request, &response); ++ status = m_stub->Update(&context, request, &response); + if (!status.ok()) { + error.SetError(status.error_message()); + ERROR("Sandboxer controller prepare request failed, error_code: %d: %s", status.error_code(), +@@ -198,25 +174,20 @@ auto SandboxerClient::Prepare(const std::string &sandboxId, const ControllerPrep + return false; + } + +- bundle = response.bundle(); +- + return true; + } + +-auto SandboxerClient::Purge(const std::string &sandboxId, const std::string &containerId, +- const std::string &execId, Errors &error) -> bool ++auto SandboxerClient::Purge(containerd::types::Sandbox &apiSandbox, ++ std::vector<std::string> &fields, Errors &error) -> bool + { + grpc::ClientContext context; +- containerd::services::sandbox::v1::PurgeRequest request; +- containerd::services::sandbox::v1::PurgeResponse response; ++ containerd::services::sandbox::v1::ControllerUpdateRequest request; ++ containerd::services::sandbox::v1::ControllerUpdateResponse response; + grpc::Status status; + +- request.set_sandboxer(m_sandboxer); +- request.set_sandbox_id(sandboxId); +- request.set_container_id(containerId); +- request.set_exec_id(execId); ++ InitUpdateRequest(request, apiSandbox, fields); + +- status = m_stub->Purge(&context, request, &response); ++ status = m_stub->Update(&context, request, &response); + if (!status.ok()) { + error.SetError(status.error_message()); + ERROR("Sandboxer controller purge request failed, error_code: %d: %s", status.error_code(), +@@ -227,44 +198,9 @@ auto SandboxerClient::Purge(const std::string &sandboxId, const std::string &con + return true; + } + +-auto SandboxerClient::InitUpdateResourcesRequest(containerd::services::sandbox::v1::UpdateResourcesRequest &request, +- const std::string &sandboxId, +- const ControllerUpdateResourcesParams ¶ms) -> bool +-{ +- if (params.resources == nullptr) { +- ERROR("Sandboxer controller update resources request failed, resources is null"); +- return false; +- } +- request.mutable_resources()->set_value(*(params.resources)); +- request.set_sandboxer(m_sandboxer); +- request.set_sandbox_id(sandboxId); +- request.set_container_id(params.containerId); +- request.mutable_annotations()->insert(params.annotations.begin(), params.annotations.end()); +- return true; +-} +- + auto SandboxerClient::UpdateResources(const std::string &sandboxId, const ControllerUpdateResourcesParams ¶ms, + Errors &error) -> bool + { +- grpc::ClientContext context; +- containerd::services::sandbox::v1::UpdateResourcesRequest request; +- containerd::services::sandbox::v1::UpdateResourcesResponse response; +- grpc::Status status; +- +- if (!InitUpdateResourcesRequest(request, sandboxId, params)) { +- error.SetError("Failed to init update-resources request for sandboxer update-resources request, sandbox id: " + +- sandboxId); +- return false; +- } +- +- status = m_stub->UpdateResources(&context, request, &response); +- if (!status.ok()) { +- error.SetError(status.error_message()); +- ERROR("Sandboxer controller update resources request failed, error_code: %d: %s", status.error_code(), +- status.error_message().c_str()); +- return false; +- } +- + return true; + } + +@@ -331,7 +267,8 @@ void SandboxerClient::StatusResponseToSandboxStatus(const containerd::services:: + sandboxStatus.id = response.sandbox_id(); + sandboxStatus.pid = response.pid(); + sandboxStatus.state = response.state(); +- sandboxStatus.taskAddress = response.task_address(); ++ sandboxStatus.taskAddress = response.address(); ++ sandboxStatus.version = response.version(); + sandboxStatus.info.insert(response.info().begin(), response.info().end()); + sandboxStatus.createdAt = TimestampToNanos(response.created_at()); + sandboxStatus.exitedAt = TimestampToNanos(response.exited_at()); +diff --git a/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_client.h b/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_client.h +index accca16b..eeb5d7f2 100644 +--- a/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_client.h ++++ b/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_client.h +@@ -48,11 +48,11 @@ public: + + auto Platform(const std::string &sandboxId, ControllerPlatformInfo &platformInfo, Errors &error) -> bool; + +- auto Prepare(const std::string &sandboxId, const ControllerPrepareParams ¶ms, std::string &bundle, +- Errors &error) -> bool; ++ auto Prepare(containerd::types::Sandbox &apiSandbox, ++ std::vector<std::string> &fields, Errors &error) -> bool; + +- auto Purge(const std::string &sandboxId, const std::string &containerId, +- const std::string &execId, Errors &error) -> bool; ++ auto Purge(containerd::types::Sandbox &apiSandbox, ++ std::vector<std::string> &fields, Errors &error) -> bool; + + auto UpdateResources(const std::string &sandboxId, const ControllerUpdateResourcesParams ¶ms, + Errors &error) -> bool; +@@ -72,11 +72,11 @@ private: + const ControllerCreateParams ¶ms) -> bool; + void StartResponseToSandboxInfo(const containerd::services::sandbox::v1::ControllerStartResponse &response, + ControllerSandboxInfo &sandboxInfo); +- auto InitPrepareRequest(containerd::services::sandbox::v1::PrepareRequest &request, +- const std::string &sandboxId, const ControllerPrepareParams ¶ms) -> bool; +- auto InitUpdateResourcesRequest(containerd::services::sandbox::v1::UpdateResourcesRequest &request, +- const std::string &sandboxId, +- const ControllerUpdateResourcesParams ¶ms) -> bool; ++ void InitUpdateRequest(containerd::services::sandbox::v1::ControllerUpdateRequest &request, ++ containerd::types::Sandbox &apiSandbox, std::vector<std::string> &fields); ++ // auto InitUpdateResourcesRequest(containerd::services::sandbox::v1::UpdateResourcesRequest &request, ++ // const std::string &sandboxId, ++ // const ControllerUpdateResourcesParams ¶ms) -> bool; + void PlatformResponseToPlatformInfo(const containerd::services::sandbox::v1::ControllerPlatformResponse &response, + ControllerPlatformInfo &platformInfo); + void StatusResponseToSandboxStatus(const containerd::services::sandbox::v1::ControllerStatusResponse &response, +diff --git a/src/daemon/sandbox/controller/sandboxer/sandboxer_controller.cc b/src/daemon/sandbox/controller/sandboxer/sandboxer_controller.cc +index d35f1118..70cab015 100644 +--- a/src/daemon/sandbox/controller/sandboxer/sandboxer_controller.cc ++++ b/src/daemon/sandbox/controller/sandboxer/sandboxer_controller.cc +@@ -61,21 +61,18 @@ std::unique_ptr<ControllerPlatformInfo> SandboxerController::Platform(const std: + return platformInfo; + } + +-std::string SandboxerController::Prepare(const std::string &sandboxId, +- const ControllerPrepareParams ¶ms, +- Errors &error) ++bool SandboxerController::Prepare(containerd::types::Sandbox &apiSandbox, ++ std::vector<std::string> &fields, ++ Errors &error) + { +- std::string bundle; +- if (!m_client->Prepare(sandboxId, params, bundle, error)) { +- return ""; +- } +- return bundle; ++ return m_client->Prepare(apiSandbox, fields, error); + } + +-bool SandboxerController::Purge(const std::string &sandboxId, const std::string &containerId, +- const std::string &execId, Errors &error) ++bool SandboxerController::Purge(containerd::types::Sandbox &apiSandbox, ++ std::vector<std::string> &fields, ++ Errors &error) + { +- return m_client->Purge(sandboxId, containerId, execId, error); ++ return m_client->Purge(apiSandbox, fields, error); + } + + bool SandboxerController::UpdateResources(const std::string &sandboxId, +diff --git a/src/daemon/sandbox/controller/sandboxer/sandboxer_controller.h b/src/daemon/sandbox/controller/sandboxer/sandboxer_controller.h +index ff66d3d8..8cb7fe7c 100644 +--- a/src/daemon/sandbox/controller/sandboxer/sandboxer_controller.h ++++ b/src/daemon/sandbox/controller/sandboxer/sandboxer_controller.h +@@ -37,11 +37,10 @@ public: + Errors &error) override; + std::unique_ptr<ControllerSandboxInfo> Start(const std::string &sandboxId, Errors &error) override; + std::unique_ptr<ControllerPlatformInfo> Platform(const std::string &sandboxId, Errors &error) override; +- std::string Prepare(const std::string &sandboxId, +- const ControllerPrepareParams ¶ms, +- Errors &error) override; +- bool Purge(const std::string &sandboxId, const std::string &containerId, +- const std::string &execId, Errors &error) override; ++ bool Prepare(containerd::types::Sandbox &apiSandbox, std::vector<std::string> &fields, ++ Errors &error) override; ++ bool Purge(containerd::types::Sandbox &apiSandbox, std::vector<std::string> &fields, ++ Errors &error) override; + bool UpdateResources(const std::string &sandboxId, + const ControllerUpdateResourcesParams ¶ms, + Errors &error) override; +diff --git a/src/daemon/sandbox/controller/shim/shim_controller.cc b/src/daemon/sandbox/controller/shim/shim_controller.cc +index ce09c076..14c99168 100644 +--- a/src/daemon/sandbox/controller/shim/shim_controller.cc ++++ b/src/daemon/sandbox/controller/shim/shim_controller.cc +@@ -340,15 +340,14 @@ std::unique_ptr<ControllerPlatformInfo> ShimController::Platform(const std::stri + return nullptr; + } + +-std::string ShimController::Prepare(const std::string &sandboxId, +- const ControllerPrepareParams ¶ms, +- Errors &error) ++bool ShimController::Prepare(containerd::types::Sandbox &apiSandbox, ++ std::vector<std::string> &fields, Errors &error) + { +- return std::string(""); ++ return true; + } + +-bool ShimController::Purge(const std::string &sandboxId, const std::string &containerId, +- const std::string &execId, Errors &error) ++bool ShimController::Purge(containerd::types::Sandbox &apiSandbox, ++ std::vector<std::string> &fields, Errors &error) + { + return true; + } +diff --git a/src/daemon/sandbox/controller/shim/shim_controller.h b/src/daemon/sandbox/controller/shim/shim_controller.h +index 5d097bac..1985ddc0 100644 +--- a/src/daemon/sandbox/controller/shim/shim_controller.h ++++ b/src/daemon/sandbox/controller/shim/shim_controller.h +@@ -45,11 +45,10 @@ public: + Errors &error) override; + std::unique_ptr<ControllerSandboxInfo> Start(const std::string &sandboxId, Errors &error) override; + std::unique_ptr<ControllerPlatformInfo> Platform(const std::string &sandboxId, Errors &error) override; +- std::string Prepare(const std::string &sandboxId, +- const ControllerPrepareParams ¶ms, +- Errors &error) override; +- bool Purge(const std::string &sandboxId, const std::string &containerId, +- const std::string &execId, Errors &error) override; ++ bool Prepare(containerd::types::Sandbox &apiSandbox, std::vector<std::string> &fields, ++ Errors &error) override; ++ bool Purge(containerd::types::Sandbox &apiSandbox, std::vector<std::string> &fields, ++ Errors &error) override; + bool UpdateResources(const std::string &sandboxId, + const ControllerUpdateResourcesParams ¶ms, + Errors &error) override; +diff --git a/src/daemon/sandbox/sandbox.cc b/src/daemon/sandbox/sandbox.cc +index dec082bc..97b77f22 100644 +--- a/src/daemon/sandbox/sandbox.cc ++++ b/src/daemon/sandbox/sandbox.cc +@@ -494,6 +494,7 @@ auto Sandbox::Load(Errors &error) -> bool + } + + LoadNetworkSetting(); ++ LoadSandboxTasks(); + + // When the sandbox status acquisition fails or wait fails, the sandbox status is set to not ready, + // and the user decides whether to delete the sandbox. +@@ -698,6 +699,7 @@ auto Sandbox::Start(Errors &error) -> bool + m_state.pid = info->pid; + m_state.createdAt = info->createdAt; + m_taskAddress = info->taskAddress; ++ m_version = info->version; + m_state.status = SANDBOX_STATUS_RUNNING; + + if (!SaveState(error)) { +@@ -814,7 +816,7 @@ void Sandbox::Status(runtime::v1::PodSandboxStatus &status) + + auto Sandbox::GenerateSandboxStateJson(sandbox_state *state) -> std::string + { +- __isula_auto_free parser_error error; ++ __isula_auto_free parser_error error = NULL; + std::string ret; + __isula_auto_free char *state_json = NULL; + state_json = sandbox_state_generate_json(state, NULL, &(error)); +@@ -874,7 +876,7 @@ auto Sandbox::SaveNetworkSetting(Errors &error) -> bool + + auto Sandbox::GenerateSandboxMetadataJson(sandbox_metadata *metadata) -> std::string + { +- __isula_auto_free parser_error error; ++ __isula_auto_free parser_error error = NULL; + std::string ret; + __isula_auto_free char *metadata_json = NULL; + metadata_json = sandbox_metadata_generate_json(metadata, NULL, &(error)); +@@ -1096,6 +1098,11 @@ auto Sandbox::GetNetworkSettingsPath() -> std::string + return m_rootdir + std::string("/") + NETWORK_SETTINGS_JSON; + } + ++auto Sandbox::GetTasksJsonPath() -> std::string ++{ ++ return m_rootdir + std::string("/") + SANDBOX_TASKS_JSON; ++} ++ + void Sandbox::FillSandboxMetadata(sandbox_metadata* metadata, Errors &error) + { + std::string jsonStr; +@@ -1116,4 +1123,214 @@ void Sandbox::FillSandboxMetadata(sandbox_metadata* metadata, Errors &error) + + metadata->sandbox_config_json = util_strdup_s(jsonStr.c_str()); + } ++ ++auto Sandbox::AddTaskById(const char *task_id, sandbox_task *task) -> bool ++{ ++ ++ std::string taskId = std::string(task_id); ++ auto iter = m_tasks.find(taskId); ++ ++ if (iter != m_tasks.end()) { ++ ERROR("Failed to add exits sandbox task %s for sandbox: %s", ++ task_id, m_id.c_str()); ++ return false; ++ } ++ m_tasks[taskId] = std::make_shared<SandboxTask>(task); ++ return true; ++} ++ ++auto Sandbox::ReadSandboxTasksJson() -> sandbox_tasks * ++{ ++ const std::string path = GetTasksJsonPath(); ++ __isula_auto_free parser_error err = nullptr; ++ sandbox_tasks *tasksArray = nullptr; ++ ++ ReadGuard<RWMutex> lock(m_tasksMutex); ++ tasksArray = sandbox_tasks_parse_file(path.c_str(), nullptr, &err); ++ if (tasksArray == nullptr) { ++ WARN("Failed to read %s tasks json: %s", path.c_str(), err); ++ } ++ return tasksArray; ++} ++ ++auto Sandbox::WriteSandboxTasksJson(std::string &tasks_json) -> bool ++{ ++ int nret = 0; ++ const std::string path = GetTasksJsonPath(); ++ ++ WriteGuard<RWMutex> lock(m_tasksMutex); ++ nret = util_atomic_write_file(path.c_str(), tasks_json.c_str(), tasks_json.size(), CONFIG_FILE_MODE, false); ++ if (nret != 0) { ++ SYSERROR("Failed to write file %s", path.c_str()); ++ } ++ return nret == 0; ++} ++ ++auto Sandbox::DeleteSandboxTasksJson() -> bool ++{ ++ int get_err = 0; ++ const std::string path = GetTasksJsonPath(); ++ ++ WriteGuard<RWMutex> lock(m_tasksMutex); ++ if (util_fileself_exists(path.c_str()) && ++ !util_force_remove_file(path.c_str(), &get_err)) { ++ errno = get_err; ++ SYSERROR("Failed to remove file %s", path.c_str()); ++ return false; ++ } ++ ++ return true; ++} ++ ++void Sandbox::AddSandboxTasksByArray(sandbox_tasks *tasksArray) ++{ ++ size_t i; ++ ++ WriteGuard<RWMutex> lock(m_tasksMutex); ++ for (i = 0; i < tasksArray->tasks_len; i++) { ++ if (!AddTaskById(tasksArray->tasks[i]->task_id, tasksArray->tasks[i])) { ++ return; ++ } ++ tasksArray->tasks[i] = nullptr; ++ } ++ tasksArray->tasks_len = 0; ++} ++ ++void Sandbox::LoadSandboxTasks() ++{ ++ sandbox_tasks *tasksArray = nullptr; ++ ++ tasksArray = ReadSandboxTasksJson(); ++ if (tasksArray == nullptr) { ++ return; ++ } ++ ++ AddSandboxTasksByArray(tasksArray); ++ ++ free_sandbox_tasks(tasksArray); ++} ++ ++auto Sandbox::SaveSandboxTasks() -> bool ++{ ++ std::string tasks_json; ++ ++ if (m_tasks.empty()) { ++ return DeleteSandboxTasksJson(); ++ } ++ ++ tasks_json = GetAnySandboxTasks(); ++ if (tasks_json.empty()) { ++ ERROR("Failed to get sandbox tasks json for sandbox: '%s'", m_id.c_str()); ++ return false; ++ } ++ ++ return WriteSandboxTasksJson(tasks_json); ++} ++ ++auto Sandbox::AddSandboxTasks(sandbox_task *task) -> bool ++{ ++ if (task == nullptr) { ++ return true; ++ } ++ if (task->task_id == nullptr) { ++ return false; ++ } ++ ++ WriteGuard<RWMutex> lock(m_tasksMutex); ++ ++ return AddTaskById(task->task_id, task); ++} ++ ++auto Sandbox::GetAnySandboxTasks() -> std::string ++{ ++ __isula_auto_free parser_error err = nullptr; ++ sandbox_tasks tasksArray = { 0 }; ++ size_t i = 0; ++ __isula_auto_free char *tasks_json = nullptr; ++ ++ tasksArray.tasks = (sandbox_task **)util_smart_calloc_s(sizeof(sandbox_task *), m_tasks.size()); ++ if (tasksArray.tasks == nullptr) { ++ SYSERROR("Out of memory."); ++ return std::string(""); ++ } ++ ++ ReadGuard<RWMutex> lock(m_tasksMutex); ++ for (auto const& [_, val] : m_tasks) { ++ /* ++ * We ignore that the processes are modified ++ * when we generate tasks json string. ++ * Because no matter whether a process is deleted or added, ++ * the Update of sandbox api will be called eventually. ++ * ++ * And we ignore that the task is freed after we do GetTask(). ++ * Because the only way to free task is DeleteSandboxTasks() ++ * which needs write lock of m_tasksMutex. ++ */ ++ tasksArray.tasks[i] = val->GetTask(); ++ i++; ++ } ++ tasksArray.tasks_len = m_tasks.size(); ++ ++ tasks_json = sandbox_tasks_generate_json(&tasksArray, nullptr, &(err)); ++ if (tasks_json == nullptr || strlen(tasks_json) == 0) { ++ ERROR("Failed to get sandbox tasks json for sandbox: '%s'", m_id.c_str()); ++ free(tasksArray.tasks); ++ return std::string(""); ++ } ++ ++ free(tasksArray.tasks); ++ return std::string(tasks_json); ++} ++ ++void Sandbox::DeleteSandboxTasks(const char *containerId) ++{ ++ if (containerId == nullptr) { ++ return; ++ } ++ ++ std::string taskId = std::string(containerId); ++ ++ WriteGuard<RWMutex> lock(m_tasksMutex); ++ auto iter = m_tasks.find(taskId); ++ if (iter == m_tasks.end()) { ++ return; ++ } ++ m_tasks.erase(iter); ++} ++ ++auto Sandbox::AddSandboxTasksProcess(const char *containerId, sandbox_process *processes) -> bool ++{ ++ if (containerId == nullptr || processes == nullptr) { ++ ERROR("Empty args."); ++ return false; ++ } ++ ++ std::string taskId = std::string(containerId); ++ ++ ReadGuard<RWMutex> lock(m_tasksMutex); ++ auto iter = m_tasks.find(taskId); ++ if (iter == m_tasks.end()) { ++ SYSERROR("Failed to find container %s", containerId); ++ return false; ++ } ++ ++ return iter->second->AddSandboxTasksProcess(processes); ++} ++ ++void Sandbox::DeleteSandboxTasksProcess(const char *containerId, const char *execId) ++{ ++ if (containerId == nullptr || execId == nullptr) { ++ return; ++ } ++ ++ std::string taskId = std::string(containerId); ++ ++ ReadGuard<RWMutex> lock(m_tasksMutex); ++ auto iter = m_tasks.find(taskId); ++ if (iter == m_tasks.end()) { ++ return; ++ } ++ iter->second->DeleteSandboxTasksProcess(execId); ++} ++ + } +\ No newline at end of file +diff --git a/src/daemon/sandbox/sandbox.h b/src/daemon/sandbox/sandbox.h +index 42fbee2a..437b6113 100644 +--- a/src/daemon/sandbox/sandbox.h ++++ b/src/daemon/sandbox/sandbox.h +@@ -30,12 +30,14 @@ + #include "controller_manager.h" + #include "cstruct_wrapper.h" + #include "read_write_lock.h" ++#include "sandbox_task.h" + + namespace sandbox { + + const std::string SANDBOX_METADATA_JSON = "sandbox_metadata.json"; + const std::string SANDBOX_STATE_JSON = "sandbox_state.json"; + const std::string NETWORK_SETTINGS_JSON = "network_settings.json"; ++const std::string SANDBOX_TASKS_JSON = "sandbox_tasks.json"; + + // Keep consistent with the default values set in containerd and cri-o. + const uint32_t DEFAULT_STOP_TIMEOUT = 10; +@@ -138,6 +140,15 @@ public: + auto Remove(Errors &error) -> bool; + void Status(runtime::v1::PodSandboxStatus &status); + ++ // for sandbox api update ++ void LoadSandboxTasks(); ++ auto SaveSandboxTasks() -> bool; ++ auto AddSandboxTasks(sandbox_task *task) -> bool; ++ auto GetAnySandboxTasks() -> std::string; ++ void DeleteSandboxTasks(const char *containerId); ++ auto AddSandboxTasksProcess(const char *containerId, sandbox_process *processes) -> bool; ++ void DeleteSandboxTasksProcess(const char *containerId, const char *execId); ++ + private: + auto SaveState(Errors &error) -> bool; + auto SaveMetadata(Errors &error) -> bool; +@@ -161,6 +172,7 @@ private: + auto GetMetadataJsonPath() -> std::string; + auto GetStatePath() -> std::string; + auto GetNetworkSettingsPath() -> std::string; ++ auto GetTasksJsonPath() -> std::string; + + void FillSandboxState(sandbox_state *state); + void FillSandboxMetadata(sandbox_metadata* metadata, Errors &error); +@@ -177,6 +189,12 @@ private: + + void updateSelinuxLabels(std::string &selinuxLabels); + ++ auto AddTaskById(const char *task_id, sandbox_task *task) -> bool; ++ auto ReadSandboxTasksJson() -> sandbox_tasks *; ++ auto WriteSandboxTasksJson(std::string &tasks_json) -> bool; ++ auto DeleteSandboxTasksJson() -> bool; ++ void AddSandboxTasksByArray(sandbox_tasks *tasksArray); ++ + private: + // Since the cri module will operate concurrently on the sandbox instance, + // use m_mutex to ensure the correctness of the sandbox instance +@@ -191,6 +209,7 @@ private: + std::string m_rootdir; + std::string m_statedir; + std::string m_taskAddress; ++ uint32_t m_version; + StatsInfo m_statsInfo; + // Store network information in the sandbox, which is convenient for the cri module to obtain + // and update the network settings of the pause container in the shim-controller. +@@ -211,6 +230,11 @@ private: + // vsock ports + std::mutex m_vsockPortsMutex; + std::set<uint32_t> m_vsockPorts; ++ ++ // use m_tasksMutex to ensure the correctness of the tasks ++ RWMutex m_tasksMutex; ++ // for sandbox api update, containerId --> tasks ++ std::map<std::string, std::shared_ptr<SandboxTask>> m_tasks; + }; + + } // namespace sandbox +diff --git a/src/daemon/sandbox/sandbox_ops.cc b/src/daemon/sandbox/sandbox_ops.cc +index 22cfea95..96e541a4 100644 +--- a/src/daemon/sandbox/sandbox_ops.cc ++++ b/src/daemon/sandbox/sandbox_ops.cc +@@ -16,11 +16,18 @@ + + #include <isula_libutils/auto_cleanup.h> + #include <isula_libutils/log.h> ++#include <google/protobuf/util/time_util.h> + + #include "controller_manager.h" + #include "sandbox_manager.h" ++#include "sandbox.h" + #include "namespace.h" + #include "utils.h" ++#include "utils_timestamp.h" ++ ++const std::string SANDBOX_EXTENSIONS_TASKS = "extensions.tasks"; ++const std::string SANDBOX_TASKS_KEY = "tasks"; ++const std::string SANDBOX_TASKS_TYPEURL = "github.com/containerd/containerd/Tasks"; + + static inline bool validate_sandbox_info(const container_sandbox_info *sandbox) + { +@@ -28,106 +35,166 @@ static inline bool validate_sandbox_info(const container_sandbox_info *sandbox) + sandbox->id != NULL); + } + +-static int generate_ctrl_rootfs(sandbox::ControllerPrepareParams ¶ms, ++static int generate_ctrl_rootfs(sandbox_task *task, + const container_config_v2_common_config *config) + { ++ size_t len = 1; + if (nullptr == config->base_fs) { + ERROR("Container %s has no base fs", config->id); + return -1; + } + + // TODO: rootfs's options left to be configured +- std::unique_ptr<sandbox::ControllerMountInfo> mount_info(new sandbox::ControllerMountInfo()); +- mount_info->type = MOUNT_TYPE_BIND; +- mount_info->source = config->base_fs; +- params.rootfs.push_back(std::move(mount_info)); ++ task->rootfs = (sandbox_mount **)util_smart_calloc_s(sizeof(sandbox_mount *), len); ++ if (task->rootfs == nullptr) { ++ ERROR("Out of memory."); ++ return -1; ++ } ++ task->rootfs[0] = (sandbox_mount *)util_common_calloc_s(sizeof(sandbox_mount)); ++ if (task->rootfs[0] == nullptr) { ++ ERROR("Out of memory."); ++ return -1; ++ } ++ task->rootfs_len = len; ++ task->rootfs[0]->type = util_strdup_s(MOUNT_TYPE_BIND); ++ task->rootfs[0]->source = util_strdup_s(config->base_fs); + + return 0; + } + +-static int do_sandbox_prepare(const container_config_v2_common_config *config, +- const char *exec_id, const char *oci_spec, +- const char * console_fifos[], bool tty) ++static int do_sandbox_prepare(std::shared_ptr<sandbox::Sandbox> &sandbox, containerd::types::Sandbox &apiSandbox) + { + Errors err; +- sandbox::ControllerPrepareParams params; +- std::unique_ptr<sandbox::ControllerStreamInfo> stream_info(new sandbox::ControllerStreamInfo()); +- const container_sandbox_info *sandbox_info = nullptr; +- +- if (nullptr == config || nullptr == config->id) { +- ERROR("Invalid parameter: config"); +- return -1; +- } ++ std::vector<std::string> fields; ++ ++ fields.push_back(SANDBOX_EXTENSIONS_TASKS); + +- sandbox_info = config->sandbox_info; +- if (false == validate_sandbox_info(sandbox_info)) { +- ERROR("Invalid parameter: sandbox"); ++ auto controller = sandbox::ControllerManager::GetInstance()->GetController(sandbox->GetSandboxer()); ++ if (nullptr == controller) { ++ ERROR("Invalid sandboxer name: %s", sandbox->GetSandboxer().c_str()); + return -1; + } + +- if (nullptr == console_fifos) { +- ERROR("Invlaid parameter: console_fifos"); ++ if (!controller->Prepare(apiSandbox, fields, err)) { ++ ERROR("Failed to prepare in container controller prepare: %s", err.GetCMessage()); + return -1; + } + +- params.containerId = config->id; +- params.execId = (nullptr == exec_id) ? "" : exec_id; +- params.spec = std::unique_ptr<std::string>(new std::string(oci_spec)); +- +- if (generate_ctrl_rootfs(params, config) != 0) { +- ERROR("Invalid rootfs"); +- return -1; +- } ++ return 0; ++} + +- stream_info->stdin = (nullptr == console_fifos[0]) ? "" : console_fifos[0]; +- stream_info->stdout = (nullptr == console_fifos[1]) ? "" : console_fifos[1]; +- stream_info->stderr = (nullptr == console_fifos[2]) ? "" : console_fifos[2]; +- stream_info->terminal = tty; +- params.streamInfo = std::move(stream_info); ++static int do_sandbox_purge(std::shared_ptr<sandbox::Sandbox> &sandbox, containerd::types::Sandbox &apiSandbox) ++{ ++ Errors err; ++ std::vector<std::string> fields; ++ ++ fields.push_back(SANDBOX_EXTENSIONS_TASKS); + +- auto controller = sandbox::ControllerManager::GetInstance()->GetController(sandbox_info->sandboxer); ++ auto controller = sandbox::ControllerManager::GetInstance()->GetController(sandbox->GetSandboxer()); + if (nullptr == controller) { +- ERROR("Invalid sandboxer name: %s", sandbox_info->sandboxer); ++ ERROR("Invalid sandboxer name: %s", sandbox->GetSandboxer().c_str()); + return -1; + } + +- std::string bundle = controller->Prepare(sandbox_info->id, params, err); +- if (err.NotEmpty()) { +- ERROR("Failed to prepare in container controller prepare: %s", err.GetCMessage()); ++ if (!controller->Purge(apiSandbox, fields, err)) { ++ ERROR("Failed to purge: %s", err.GetCMessage()); + return -1; + } + + return 0; + } + +-static int do_sandbox_purge(const container_config_v2_common_config *config, +- const char *exec_id) ++static oci_runtime_spec *clone_oci_runtime_spec(const oci_runtime_spec *oci_spec) + { +- Errors err; +- const container_sandbox_info *sandbox_info = nullptr; ++ __isula_auto_free char *json_str = nullptr; ++ __isula_auto_free parser_error err = nullptr; ++ oci_runtime_spec *ret = nullptr; ++ ++ json_str = oci_runtime_spec_generate_json(oci_spec, nullptr, &err); ++ if (json_str == nullptr) { ++ ERROR("Failed to generate spec json: %s", err); ++ return nullptr; ++ } ++ ret = oci_runtime_spec_parse_data(json_str, nullptr, &err); ++ if (ret == nullptr) { ++ ERROR("Failed to generate spec: %s", err); ++ } ++ return ret; ++} ++ ++static defs_process *clone_defs_process(defs_process *process_spec) ++{ ++ __isula_auto_free char *json_str = nullptr; ++ __isula_auto_free parser_error err = nullptr; ++ defs_process *ret = nullptr; ++ ++ json_str = defs_process_generate_json(process_spec, nullptr, &err); ++ if (json_str == nullptr) { ++ ERROR("Failed to generate process spec json: %s", err); ++ return nullptr; ++ } ++ ret = defs_process_parse_data(json_str, nullptr, &err); ++ if (ret == nullptr) { ++ ERROR("Failed to generate process spec: %s", err); ++ } ++ return ret; ++} + ++static std::shared_ptr<sandbox::Sandbox> get_prepare_sandbox(const container_config_v2_common_config *config) ++{ + if (nullptr == config || nullptr == config->id) { + ERROR("Invalid parameter: config"); +- return -1; ++ return nullptr; + } + +- sandbox_info = config->sandbox_info; ++ auto sandbox_info = config->sandbox_info; + if (false == validate_sandbox_info(sandbox_info)) { + ERROR("Invalid parameter: sandbox"); +- return -1; ++ return nullptr; + } + +- auto controller = sandbox::ControllerManager::GetInstance()->GetController(sandbox_info->sandboxer); +- if (nullptr == controller) { +- ERROR("Invalid sandboxer name: %s", sandbox_info->sandboxer); +- return -1; ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(sandbox_info->id); ++ if (nullptr == sandbox) { ++ ERROR("Sandbox not found"); ++ return nullptr; + } ++ return sandbox; ++} + +- if (!controller->Purge(sandbox_info->id, config->id, +- (nullptr == exec_id) ? "" : exec_id, err)) { +- ERROR("Failed to purge: %s", err.GetCMessage()); ++static int init_prepare_api_sandbox(std::shared_ptr<sandbox::Sandbox> sandbox, const char *containerId, ++ containerd::types::Sandbox &apiSandbox) ++{ ++ google::protobuf::Map<std::string, std::string> *labels = apiSandbox.mutable_labels(); ++ google::protobuf::Map<std::string, google::protobuf::Any> *extensions = apiSandbox.mutable_extensions(); ++ google::protobuf::Any any; ++ auto created_at = new (std::nothrow) google::protobuf::Timestamp; ++ auto updated_at = new (std::nothrow) google::protobuf::Timestamp; ++ ++ apiSandbox.set_sandbox_id(sandbox->GetId()); ++ apiSandbox.mutable_runtime()->set_name(sandbox->GetRuntime()); ++ // TODO how get options ++ // apiSandbox.mutable_runtime()->set_options(sandbox->GetRuntime()); ++ // Just ignore spec ++ (*labels)[std::string("name")] = sandbox->GetName(); ++ ++ *created_at = google::protobuf::util::TimeUtil::NanosecondsToTimestamp( ++ sandbox->GetCreatedAt()); ++ apiSandbox.set_allocated_created_at(created_at); ++ *updated_at = google::protobuf::util::TimeUtil::NanosecondsToTimestamp(util_get_now_time_nanos()); ++ apiSandbox.set_allocated_updated_at(updated_at); ++ ++ auto any_type_url = any.mutable_type_url(); ++ *any_type_url = SANDBOX_TASKS_TYPEURL; ++ auto any_value = any.mutable_value(); ++ *any_value = sandbox->GetAnySandboxTasks(); ++ if ((*any_value).empty()) { ++ ERROR("Failed to get any sandbox tasks"); + return -1; + } ++ DEBUG("Get any sandbox tasks %s", (*any_value).c_str()); ++ (*extensions)[SANDBOX_TASKS_KEY] = any; ++ ++ apiSandbox.set_sandboxer(sandbox->GetSandboxer()); + + return 0; + } +@@ -136,45 +203,177 @@ int sandbox_prepare_container(const container_config_v2_common_config *config, + const oci_runtime_spec *oci_spec, + const char * console_fifos[], bool tty) + { +- __isula_auto_free char *json_oci_spec = nullptr; +- __isula_auto_free parser_error err = nullptr; ++ sandbox_task *task = nullptr; ++ containerd::types::Sandbox apiSandbox; ++ int ret = -1; + + INFO("Prepare container for sandbox"); + +- json_oci_spec = oci_runtime_spec_generate_json(oci_spec, nullptr, &err); +- if (nullptr == json_oci_spec) { +- ERROR("Failed to generate container spec json: %s", err); ++ if (nullptr == console_fifos) { ++ ERROR("Invlaid parameter: console_fifos"); ++ return -1; ++ } ++ ++ auto sandbox = get_prepare_sandbox(config); ++ if (sandbox == nullptr) { ++ ERROR("Sandbox not found"); ++ return -1; ++ } ++ ++ task = (sandbox_task *)util_common_calloc_s(sizeof(sandbox_task)); ++ if (task == nullptr) { ++ ERROR("Out of memory."); + return -1; + } +- return do_sandbox_prepare(config, nullptr, json_oci_spec, console_fifos, tty); ++ task->task_id = util_strdup_s(config->id); ++ task->spec = clone_oci_runtime_spec(oci_spec); ++ if (task->spec == nullptr) { ++ ERROR("Out of memory."); ++ goto free_out; ++ } ++ if (generate_ctrl_rootfs(task, config) != 0) { ++ ERROR("Invalid rootfs"); ++ goto free_out; ++ } ++ task->stdin = util_strdup_s((nullptr == console_fifos[0]) ? "" : console_fifos[0]); ++ task->stdout = util_strdup_s((nullptr == console_fifos[1]) ? "" : console_fifos[1]); ++ task->stderr = util_strdup_s((nullptr == console_fifos[2]) ? "" : console_fifos[2]); ++ ++ if (!sandbox->AddSandboxTasks(task)) { ++ ERROR("Failed to add sandbox %s task.", config->id); ++ goto free_out; ++ } ++ task = nullptr; ++ ret = init_prepare_api_sandbox(sandbox, config->id, apiSandbox); ++ if (ret != 0) { ++ ERROR("Failed to init %s api sandbox.", config->id); ++ goto del_out; ++ } ++ ret = do_sandbox_prepare(sandbox, apiSandbox); ++ ++del_out: ++ if (ret != 0) { ++ sandbox->DeleteSandboxTasks(config->id); ++ } ++ if (!sandbox->SaveSandboxTasks()) { ++ ERROR("Failed to Save %s sandbox tasks.", config->id); ++ ret = -1; ++ } ++free_out: ++ free_sandbox_task(task); ++ return ret; + } + + int sandbox_prepare_exec(const container_config_v2_common_config *config, + const char *exec_id, defs_process *process_spec, + const char * console_fifos[], bool tty) + { +- __isula_auto_free char *json_process = nullptr; +- __isula_auto_free parser_error err = nullptr; ++ sandbox_process *process = nullptr; ++ containerd::types::Sandbox apiSandbox; ++ int ret = -1; + + INFO("Prepare exec for container in sandbox"); + +- json_process = defs_process_generate_json(process_spec, nullptr, &err); +- if (nullptr == json_process) { +- ERROR("Failed to generate process spec json: %s", err); ++ if (nullptr == console_fifos) { ++ ERROR("Invlaid parameter: console_fifos"); + return -1; + } + +- return do_sandbox_prepare(config, exec_id, json_process, console_fifos, tty); ++ auto sandbox = get_prepare_sandbox(config); ++ if (sandbox == nullptr) { ++ ERROR("Sandbox not found"); ++ return -1; ++ } ++ ++ process = (sandbox_process *)util_common_calloc_s(sizeof(sandbox_process)); ++ if (process == nullptr) { ++ ERROR("Out of memory."); ++ return -1; ++ } ++ process->exec_id = util_strdup_s(exec_id); ++ process->spec = clone_defs_process(process_spec); ++ if (process->spec == nullptr) { ++ ERROR("Out of memory."); ++ goto free_out; ++ } ++ process->stdin = util_strdup_s((nullptr == console_fifos[0]) ? "" : console_fifos[0]); ++ process->stdout = util_strdup_s((nullptr == console_fifos[1]) ? "" : console_fifos[1]); ++ process->stderr = util_strdup_s((nullptr == console_fifos[2]) ? "" : console_fifos[2]); ++ ++ if (!sandbox->AddSandboxTasksProcess(config->id, process)) { ++ ERROR("Failed to add sandbox %s process.", config->id); ++ goto free_out; ++ } ++ process = nullptr; ++ ret = init_prepare_api_sandbox(sandbox, config->id, apiSandbox); ++ if (ret != 0) { ++ ERROR("Failed to init %s api sandbox.", config->id); ++ goto del_out; ++ } ++ ret = do_sandbox_prepare(sandbox, apiSandbox); ++ ++del_out: ++ if (ret != 0) { ++ sandbox->DeleteSandboxTasksProcess(config->id, exec_id); ++ } ++ if (!sandbox->SaveSandboxTasks()) { ++ ERROR("Failed to Save %s sandbox tasks.", config->id); ++ ret = -1; ++ } ++free_out: ++ free_sandbox_process(process); ++ return ret; + } + + int sandbox_purge_container(const container_config_v2_common_config *config) + { +- return do_sandbox_purge(config, nullptr); ++ containerd::types::Sandbox apiSandbox; ++ ++ INFO("Purge container for sandbox"); ++ ++ auto sandbox = get_prepare_sandbox(config); ++ if (sandbox == nullptr) { ++ ERROR("Sandbox not found"); ++ return -1; ++ } ++ ++ sandbox->DeleteSandboxTasks(config->id); ++ if (!sandbox->SaveSandboxTasks()) { ++ ERROR("Failed to Save %s sandbox tasks.", config->id); ++ return -1; ++ } ++ ++ if (init_prepare_api_sandbox(sandbox, config->id, apiSandbox) != 0) { ++ ERROR("Failed to init %s api sandbox.", config->id); ++ return -1; ++ } ++ return do_sandbox_purge(sandbox, apiSandbox); + } + + int sandbox_purge_exec(const container_config_v2_common_config *config, const char *exec_id) + { +- return do_sandbox_purge(config, exec_id); ++ containerd::types::Sandbox apiSandbox; ++ ++ INFO("Purge exec for container in sandbox"); ++ ++ auto sandbox = get_prepare_sandbox(config); ++ if (sandbox == nullptr) { ++ ERROR("Sandbox not found"); ++ return -1; ++ } ++ ++ sandbox->DeleteSandboxTasksProcess(config->id, exec_id); ++ if (!sandbox->SaveSandboxTasks()) { ++ ERROR("Failed to Save %s sandbox tasks.", config->id); ++ return -1; ++ } ++ ++ if (init_prepare_api_sandbox(sandbox, config->id, apiSandbox) != 0) { ++ ERROR("Failed to init %s api sandbox.", exec_id); ++ return -1; ++ } ++ ++ return do_sandbox_purge(sandbox, apiSandbox); + } + + int sandbox_on_sandbox_exit(const char *sandbox_id, int exit_code) +diff --git a/src/daemon/sandbox/sandbox_task.cc b/src/daemon/sandbox/sandbox_task.cc +new file mode 100644 +index 00000000..b1efc340 +--- /dev/null ++++ b/src/daemon/sandbox/sandbox_task.cc +@@ -0,0 +1,99 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-10-22 ++ * Description: provide sandbox class definition ++ *********************************************************************************/ ++#include "sandbox_task.h" ++ ++#include <mutex> ++ ++#include <isula_libutils/log.h> ++ ++#include "utils.h" ++#include "errors.h" ++ ++namespace sandbox { ++ ++SandboxTask::SandboxTask(sandbox_task *task): m_task(task) ++{ ++} ++ ++SandboxTask::~SandboxTask() ++{ ++ free_sandbox_task(m_task); ++ m_task = nullptr; ++} ++ ++auto SandboxTask::GetTask() -> sandbox_task * ++{ ++ ReadGuard<RWMutex> lock(m_taskMutex); ++ return m_task; ++} ++ ++auto SandboxTask::AddSandboxTasksProcess(sandbox_process *processes) -> bool ++{ ++ if (processes == nullptr) { ++ ERROR("Empty args."); ++ return false; ++ } ++ ++ WriteGuard<RWMutex> lock(m_taskMutex); ++ if (util_mem_realloc((void **)(&m_task->processes), ++ (m_task->processes_len + 1) * sizeof(sandbox_process *), ++ (void *)m_task->processes, ++ m_task->processes_len * sizeof(sandbox_process *)) != 0) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ m_task->processes[m_task->processes_len] = processes; ++ m_task->processes_len++; ++ ++ return true; ++} ++ ++auto SandboxTask::FindProcessByID(const char *execId) -> int ++{ ++ int i; ++ int processes_len = m_task->processes_len; ++ ++ if (m_task->processes == nullptr) { ++ return -1; ++ } ++ ++ for (i = 0; i < processes_len; i++) { ++ if (strcmp(m_task->processes[i]->exec_id, execId) == 0) { ++ return i; ++ } ++ } ++ return -1; ++} ++ ++void SandboxTask::DeleteSandboxTasksProcess(const char *execId) ++{ ++ if (execId == nullptr) { ++ return; ++ } ++ ++ int idx; ++ ++ WriteGuard<RWMutex> lock(m_taskMutex); ++ idx = FindProcessByID(execId); ++ if (idx < 0) { ++ return; ++ } ++ free_sandbox_process(m_task->processes[idx]); ++ (void)memcpy((void **)&m_task->processes[idx], (void **)&m_task->processes[idx + 1], ++ (m_task->processes_len - idx - 1) * sizeof(void *)); ++ m_task->processes_len--; ++} ++ ++} +\ No newline at end of file +diff --git a/src/daemon/sandbox/sandbox_task.h b/src/daemon/sandbox/sandbox_task.h +new file mode 100644 +index 00000000..1bd0ce58 +--- /dev/null ++++ b/src/daemon/sandbox/sandbox_task.h +@@ -0,0 +1,48 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-10-22 ++ * Description: provide sandbox class definition ++ *********************************************************************************/ ++ ++#ifndef DAEMON_SANDBOX_SANDBOX_TASK_H ++#define DAEMON_SANDBOX_SANDBOX_TASK_H ++ ++#include <string> ++#include <mutex> ++ ++#include <isula_libutils/sandbox_tasks.h> ++ ++#include "api_v1.grpc.pb.h" ++#include "errors.h" ++#include "read_write_lock.h" ++ ++namespace sandbox { ++ ++class SandboxTask : public std::enable_shared_from_this<SandboxTask> { ++public: ++ SandboxTask(sandbox_task *task); ++ ~SandboxTask(); ++ ++ auto GetTask() -> sandbox_task *; ++ auto AddSandboxTasksProcess(sandbox_process *processes) -> bool; ++ void DeleteSandboxTasksProcess(const char *execId); ++ ++private: ++ auto FindProcessByID(const char *execId) -> int; ++private: ++ // Do not modify m_task concurrently. ++ RWMutex m_taskMutex; ++ sandbox_task *m_task; ++}; ++} // namespace sandbox ++ ++#endif // DAEMON_SANDBOX_SANDBOX_TASK_H +\ No newline at end of file +-- +2.34.1 + diff --git a/0152-add-omitted-macro-definition.patch b/0152-add-omitted-macro-definition.patch new file mode 100644 index 0000000..630282f --- /dev/null +++ b/0152-add-omitted-macro-definition.patch @@ -0,0 +1,48 @@ +From 9aadae1a71243bab41f5bf43c60f8f1b72806d54 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Fri, 22 Nov 2024 02:40:13 +1400 +Subject: [PATCH 152/156] add omitted macro definition + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/modules/api/runtime_api.h | 2 ++ + src/daemon/modules/service/service_container.c | 2 ++ + 2 files changed, 4 insertions(+) + +diff --git a/src/daemon/modules/api/runtime_api.h b/src/daemon/modules/api/runtime_api.h +index 930710ca..1f23efe3 100644 +--- a/src/daemon/modules/api/runtime_api.h ++++ b/src/daemon/modules/api/runtime_api.h +@@ -84,7 +84,9 @@ typedef struct _rt_create_params_t { + bool tty; + bool open_stdin; + const char *task_addr; ++#ifdef ENABLE_NO_PIVOT_ROOT + bool no_pivot_root; ++#endif + } rt_create_params_t; + + typedef struct _rt_start_params_t { +diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c +index 754c28ac..8e80e936 100644 +--- a/src/daemon/modules/service/service_container.c ++++ b/src/daemon/modules/service/service_container.c +@@ -743,6 +743,7 @@ static int do_oci_spec_update(const char *id, oci_runtime_spec *oci_spec, contai + return 0; + } + ++#ifdef ENABLE_NO_PIVOT_ROOT + static bool pack_no_pivot_root(const container_t *cont) + { + size_t i = 0; +@@ -759,6 +760,7 @@ static bool pack_no_pivot_root(const container_t *cont) + } + return ret; + } ++#endif + + static int do_start_container(container_t *cont, const char *console_fifos[], bool reset_rm, pid_ppid_info_t *pid_info) + { +-- +2.34.1 + diff --git a/0153-sandbox-sandbox-api-adapt-rust-interface.patch b/0153-sandbox-sandbox-api-adapt-rust-interface.patch new file mode 100644 index 0000000..bacaa54 --- /dev/null +++ b/0153-sandbox-sandbox-api-adapt-rust-interface.patch @@ -0,0 +1,4272 @@ +From 32208449b2cf4de7b9c6b05b1bf58538f7f8640a Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Thu, 21 Nov 2024 11:58:36 +0800 +Subject: [PATCH 153/156] sandbox: sandbox api adapt rust interface + +Signed-off-by: liuxu <liuxu156@huawei.com> +--- + cmake/options.cmake | 10 +- + cmake/protoc.cmake | 10 - + src/CMakeLists.txt | 4 + + .../sandbox/google/protobuf/any.proto | 155 ------ + .../sandbox/google/protobuf/empty.proto | 52 -- + .../sandbox/google/protobuf/timestamp.proto | 138 ------ + src/api/services/sandbox/sandbox.proto | 204 -------- + .../sandbox/sandbox/types/metrics.proto | 30 -- + .../sandbox/sandbox/types/mount.proto | 43 -- + .../sandbox/sandbox/types/platform.proto | 30 -- + .../sandbox/sandbox/types/sandbox.proto | 54 -- + src/daemon/common/cri/v1/v1_cri_helpers.cc | 3 +- + src/daemon/sandbox/CMakeLists.txt | 27 +- + .../sandbox/{controller => }/controller.h | 34 +- + src/daemon/sandbox/controller/CMakeLists.txt | 32 -- + .../sandboxer/client/CMakeLists.txt | 20 - + .../sandboxer/client/grpc_async_wait_call.cc | 142 ------ + .../sandboxer/client/grpc_async_wait_call.h | 94 ---- + .../sandboxer/client/grpc_client_utils.cc | 36 -- + .../sandboxer/client/grpc_client_utils.h | 23 - + .../sandboxer/client/grpc_sandboxer_client.cc | 337 ------------- + .../client/grpc_sandboxer_monitor.cc | 314 ------------ + .../sandboxer/client/grpc_sandboxer_monitor.h | 85 ---- + .../{controller => }/controller_manager.cc | 3 - + .../{controller => }/controller_manager.h | 0 + src/daemon/sandbox/sandbox.cc | 215 -------- + src/daemon/sandbox/sandbox.h | 26 +- + src/daemon/sandbox/sandbox_manager.cc | 44 +- + src/daemon/sandbox/sandbox_ops.cc | 250 +++++++--- + src/daemon/sandbox/sandbox_task.cc | 7 +- + src/daemon/sandbox/sandboxer/CMakeLists.txt | 15 + + .../controller}/CMakeLists.txt | 0 + .../controller/client/CMakeLists.txt | 12 + + .../client/grpc_sandboxer_client.cc | 463 ++++++++++++++++++ + .../client/grpc_sandboxer_client.h | 46 +- + .../controller}/sandboxer_controller.cc | 27 +- + .../controller}/sandboxer_controller.h | 9 +- + .../sandbox/sandboxer/sandboxer_sandbox.cc | 254 ++++++++++ + .../sandbox/sandboxer/sandboxer_sandbox.h | 63 +++ + src/daemon/sandbox/shim/CMakeLists.txt | 15 + + .../shim => shim/controller}/CMakeLists.txt | 0 + .../controller}/shim_controller.cc | 19 +- + .../controller}/shim_controller.h | 12 +- + src/daemon/sandbox/shim/shim_sandbox.cc | 65 +++ + src/daemon/sandbox/shim/shim_sandbox.h | 49 ++ + 45 files changed, 1225 insertions(+), 2246 deletions(-) + delete mode 100644 src/api/services/sandbox/google/protobuf/any.proto + delete mode 100644 src/api/services/sandbox/google/protobuf/empty.proto + delete mode 100644 src/api/services/sandbox/google/protobuf/timestamp.proto + delete mode 100644 src/api/services/sandbox/sandbox.proto + delete mode 100644 src/api/services/sandbox/sandbox/types/metrics.proto + delete mode 100644 src/api/services/sandbox/sandbox/types/mount.proto + delete mode 100644 src/api/services/sandbox/sandbox/types/platform.proto + delete mode 100644 src/api/services/sandbox/sandbox/types/sandbox.proto + rename src/daemon/sandbox/{controller => }/controller.h (79%) + delete mode 100644 src/daemon/sandbox/controller/CMakeLists.txt + delete mode 100755 src/daemon/sandbox/controller/sandboxer/client/CMakeLists.txt + delete mode 100644 src/daemon/sandbox/controller/sandboxer/client/grpc_async_wait_call.cc + delete mode 100644 src/daemon/sandbox/controller/sandboxer/client/grpc_async_wait_call.h + delete mode 100644 src/daemon/sandbox/controller/sandboxer/client/grpc_client_utils.cc + delete mode 100644 src/daemon/sandbox/controller/sandboxer/client/grpc_client_utils.h + delete mode 100644 src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_client.cc + delete mode 100644 src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_monitor.cc + delete mode 100644 src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_monitor.h + rename src/daemon/sandbox/{controller => }/controller_manager.cc (98%) + rename src/daemon/sandbox/{controller => }/controller_manager.h (100%) + create mode 100755 src/daemon/sandbox/sandboxer/CMakeLists.txt + rename src/daemon/sandbox/{controller/sandboxer => sandboxer/controller}/CMakeLists.txt (100%) + create mode 100755 src/daemon/sandbox/sandboxer/controller/client/CMakeLists.txt + create mode 100644 src/daemon/sandbox/sandboxer/controller/client/grpc_sandboxer_client.cc + rename src/daemon/sandbox/{controller/sandboxer => sandboxer/controller}/client/grpc_sandboxer_client.h (54%) + rename src/daemon/sandbox/{controller/sandboxer => sandboxer/controller}/sandboxer_controller.cc (78%) + rename src/daemon/sandbox/{controller/sandboxer => sandboxer/controller}/sandboxer_controller.h (82%) + create mode 100644 src/daemon/sandbox/sandboxer/sandboxer_sandbox.cc + create mode 100644 src/daemon/sandbox/sandboxer/sandboxer_sandbox.h + create mode 100755 src/daemon/sandbox/shim/CMakeLists.txt + rename src/daemon/sandbox/{controller/shim => shim/controller}/CMakeLists.txt (100%) + rename src/daemon/sandbox/{controller/shim => shim/controller}/shim_controller.cc (97%) + rename src/daemon/sandbox/{controller/shim => shim/controller}/shim_controller.h (89%) + create mode 100644 src/daemon/sandbox/shim/shim_sandbox.cc + create mode 100644 src/daemon/sandbox/shim/shim_sandbox.h + +diff --git a/cmake/options.cmake b/cmake/options.cmake +index 41177fe0..cee83ef2 100644 +--- a/cmake/options.cmake ++++ b/cmake/options.cmake +@@ -53,9 +53,13 @@ endif() + + option(ENABLE_SANDBOXER "Enable sandbox API" OFF) + if (ENABLE_SANDBOXER STREQUAL "ON") +- add_definitions(-DENABLE_SANDBOXER) +- set(ENABLE_SANDBOXER 1) +- message("${Green}-- Enable sandbox API${ColourReset}") ++ if (ENABLE_CRI_API_V1) ++ add_definitions(-DENABLE_SANDBOXER) ++ set(ENABLE_SANDBOXER 1) ++ message("${Green}-- Enable sandbox API${ColourReset}") ++ else() ++ message("${Yellow}-- Can not enable sandboxer, sandboxer need enable CRI API V1 first ${ColourReset}") ++ endif() + endif() + + option(ENABLE_OOM_MONITOR "Enable oom monitor" ON) +diff --git a/cmake/protoc.cmake b/cmake/protoc.cmake +index 6343fe3e..de337f25 100644 +--- a/cmake/protoc.cmake ++++ b/cmake/protoc.cmake +@@ -66,13 +66,3 @@ if (GRPC_CONNECTOR) + PROTOC_GRPC_GEN(network ${NETWORK_PROTOS_OUT_PATH} ${PROTOS_PATH}/network/network.proto) + endif() + endif() +- +-if (ENABLE_CRI_API_V1 AND ENABLE_SANDBOXER) +- execute_process(COMMAND mkdir -p ${SANDBOX_PROTOS_OUT_PATH}) +- PROTOC_CPP_GEN(sandbox ${SANDBOX_PROTOS_OUT_PATH} ${PROTOS_PATH}/sandbox/sandbox/types/sandbox.proto) +- PROTOC_CPP_GEN(sandbox ${SANDBOX_PROTOS_OUT_PATH} ${PROTOS_PATH}/sandbox/sandbox/types/mount.proto) +- PROTOC_CPP_GEN(sandbox ${SANDBOX_PROTOS_OUT_PATH} ${PROTOS_PATH}/sandbox/sandbox/types/platform.proto) +- PROTOC_CPP_GEN(sandbox ${SANDBOX_PROTOS_OUT_PATH} ${PROTOS_PATH}/sandbox/sandbox/types/metrics.proto) +- PROTOC_CPP_GEN(sandbox ${SANDBOX_PROTOS_OUT_PATH} ${PROTOS_PATH}/sandbox/sandbox.proto) +- PROTOC_GRPC_GEN(sandbox ${SANDBOX_PROTOS_OUT_PATH} ${PROTOS_PATH}/sandbox/sandbox.proto) +-endif() +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index 70a8ac91..6aab3f69 100644 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -200,6 +200,10 @@ else() + set_target_properties(isulad PROPERTIES LINKER_LANGUAGE "C") + endif() + ++if (ENABLE_SANDBOXER) ++ target_link_libraries(isulad isula_sandbox) ++endif() ++ + target_link_libraries(isulad libisulad_tools libhttpclient -ldl) + if (ANDROID OR MUSL) + target_link_libraries(isulad ${LIBSSL_LIBRARY} ${LIBYAJL_LIBRARY}) +diff --git a/src/api/services/sandbox/google/protobuf/any.proto b/src/api/services/sandbox/google/protobuf/any.proto +deleted file mode 100644 +index c9be8541..00000000 +--- a/src/api/services/sandbox/google/protobuf/any.proto ++++ /dev/null +@@ -1,155 +0,0 @@ +-// Protocol Buffers - Google's data interchange format +-// Copyright 2008 Google Inc. All rights reserved. +-// https://developers.google.com/protocol-buffers/ +-// +-// Redistribution and use in source and binary forms, with or without +-// modification, are permitted provided that the following conditions are +-// met: +-// +-// * Redistributions of source code must retain the above copyright +-// notice, this list of conditions and the following disclaimer. +-// * Redistributions in binary form must reproduce the above +-// copyright notice, this list of conditions and the following disclaimer +-// in the documentation and/or other materials provided with the +-// distribution. +-// * Neither the name of Google Inc. nor the names of its +-// contributors may be used to endorse or promote products derived from +-// this software without specific prior written permission. +-// +-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +- +-syntax = "proto3"; +- +-package google.protobuf; +- +-option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +-option go_package = "github.com/golang/protobuf/ptypes/any"; +-option java_package = "com.google.protobuf"; +-option java_outer_classname = "AnyProto"; +-option java_multiple_files = true; +-option objc_class_prefix = "GPB"; +- +-// `Any` contains an arbitrary serialized protocol buffer message along with a +-// URL that describes the type of the serialized message. +-// +-// Protobuf library provides support to pack/unpack Any values in the form +-// of utility functions or additional generated methods of the Any type. +-// +-// Example 1: Pack and unpack a message in C++. +-// +-// Foo foo = ...; +-// Any any; +-// any.PackFrom(foo); +-// ... +-// if (any.UnpackTo(&foo)) { +-// ... +-// } +-// +-// Example 2: Pack and unpack a message in Java. +-// +-// Foo foo = ...; +-// Any any = Any.pack(foo); +-// ... +-// if (any.is(Foo.class)) { +-// foo = any.unpack(Foo.class); +-// } +-// +-// Example 3: Pack and unpack a message in Python. +-// +-// foo = Foo(...) +-// any = Any() +-// any.Pack(foo) +-// ... +-// if any.Is(Foo.DESCRIPTOR): +-// any.Unpack(foo) +-// ... +-// +-// Example 4: Pack and unpack a message in Go +-// +-// foo := &pb.Foo{...} +-// any, err := ptypes.MarshalAny(foo) +-// ... +-// foo := &pb.Foo{} +-// if err := ptypes.UnmarshalAny(any, foo); err != nil { +-// ... +-// } +-// +-// The pack methods provided by protobuf library will by default use +-// 'type.googleapis.com/full.type.name' as the type URL and the unpack +-// methods only use the fully qualified type name after the last '/' +-// in the type URL, for example "foo.bar.com/x/y.z" will yield type +-// name "y.z". +-// +-// +-// JSON +-// ==== +-// The JSON representation of an `Any` value uses the regular +-// representation of the deserialized, embedded message, with an +-// additional field `@type` which contains the type URL. Example: +-// +-// package google.profile; +-// message Person { +-// string first_name = 1; +-// string last_name = 2; +-// } +-// +-// { +-// "@type": "type.googleapis.com/google.profile.Person", +-// "firstName": <string>, +-// "lastName": <string> +-// } +-// +-// If the embedded message type is well-known and has a custom JSON +-// representation, that representation will be embedded adding a field +-// `value` which holds the custom JSON in addition to the `@type` +-// field. Example (for message [google.protobuf.Duration][]): +-// +-// { +-// "@type": "type.googleapis.com/google.protobuf.Duration", +-// "value": "1.212s" +-// } +-// +-message Any { +- // A URL/resource name that uniquely identifies the type of the serialized +- // protocol buffer message. This string must contain at least +- // one "/" character. The last segment of the URL's path must represent +- // the fully qualified name of the type (as in +- // `path/google.protobuf.Duration`). The name should be in a canonical form +- // (e.g., leading "." is not accepted). +- // +- // In practice, teams usually precompile into the binary all types that they +- // expect it to use in the context of Any. However, for URLs which use the +- // scheme `http`, `https`, or no scheme, one can optionally set up a type +- // server that maps type URLs to message definitions as follows: +- // +- // * If no scheme is provided, `https` is assumed. +- // * An HTTP GET on the URL must yield a [google.protobuf.Type][] +- // value in binary format, or produce an error. +- // * Applications are allowed to cache lookup results based on the +- // URL, or have them precompiled into a binary to avoid any +- // lookup. Therefore, binary compatibility needs to be preserved +- // on changes to types. (Use versioned type names to manage +- // breaking changes.) +- // +- // Note: this functionality is not currently available in the official +- // protobuf release, and it is not used for type URLs beginning with +- // type.googleapis.com. +- // +- // Schemes other than `http`, `https` (or the empty scheme) might be +- // used with implementation specific semantics. +- // +- string type_url = 1; +- +- // Must be a valid serialized protocol buffer of the above specified type. +- bytes value = 2; +-} +diff --git a/src/api/services/sandbox/google/protobuf/empty.proto b/src/api/services/sandbox/google/protobuf/empty.proto +deleted file mode 100644 +index 03cacd23..00000000 +--- a/src/api/services/sandbox/google/protobuf/empty.proto ++++ /dev/null +@@ -1,52 +0,0 @@ +-// Protocol Buffers - Google's data interchange format +-// Copyright 2008 Google Inc. All rights reserved. +-// https://developers.google.com/protocol-buffers/ +-// +-// Redistribution and use in source and binary forms, with or without +-// modification, are permitted provided that the following conditions are +-// met: +-// +-// * Redistributions of source code must retain the above copyright +-// notice, this list of conditions and the following disclaimer. +-// * Redistributions in binary form must reproduce the above +-// copyright notice, this list of conditions and the following disclaimer +-// in the documentation and/or other materials provided with the +-// distribution. +-// * Neither the name of Google Inc. nor the names of its +-// contributors may be used to endorse or promote products derived from +-// this software without specific prior written permission. +-// +-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +- +-syntax = "proto3"; +- +-package google.protobuf; +- +-option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +-option go_package = "github.com/golang/protobuf/ptypes/empty"; +-option java_package = "com.google.protobuf"; +-option java_outer_classname = "EmptyProto"; +-option java_multiple_files = true; +-option objc_class_prefix = "GPB"; +-option cc_enable_arenas = true; +- +-// A generic empty message that you can re-use to avoid defining duplicated +-// empty messages in your APIs. A typical example is to use it as the request +-// or the response type of an API method. For instance: +-// +-// service Foo { +-// rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty); +-// } +-// +-// The JSON representation for `Empty` is empty JSON object `{}`. +-message Empty {} +diff --git a/src/api/services/sandbox/google/protobuf/timestamp.proto b/src/api/services/sandbox/google/protobuf/timestamp.proto +deleted file mode 100644 +index cd357864..00000000 +--- a/src/api/services/sandbox/google/protobuf/timestamp.proto ++++ /dev/null +@@ -1,138 +0,0 @@ +-// Protocol Buffers - Google's data interchange format +-// Copyright 2008 Google Inc. All rights reserved. +-// https://developers.google.com/protocol-buffers/ +-// +-// Redistribution and use in source and binary forms, with or without +-// modification, are permitted provided that the following conditions are +-// met: +-// +-// * Redistributions of source code must retain the above copyright +-// notice, this list of conditions and the following disclaimer. +-// * Redistributions in binary form must reproduce the above +-// copyright notice, this list of conditions and the following disclaimer +-// in the documentation and/or other materials provided with the +-// distribution. +-// * Neither the name of Google Inc. nor the names of its +-// contributors may be used to endorse or promote products derived from +-// this software without specific prior written permission. +-// +-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +- +-syntax = "proto3"; +- +-package google.protobuf; +- +-option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +-option cc_enable_arenas = true; +-option go_package = "github.com/golang/protobuf/ptypes/timestamp"; +-option java_package = "com.google.protobuf"; +-option java_outer_classname = "TimestampProto"; +-option java_multiple_files = true; +-option objc_class_prefix = "GPB"; +- +-// A Timestamp represents a point in time independent of any time zone or local +-// calendar, encoded as a count of seconds and fractions of seconds at +-// nanosecond resolution. The count is relative to an epoch at UTC midnight on +-// January 1, 1970, in the proleptic Gregorian calendar which extends the +-// Gregorian calendar backwards to year one. +-// +-// All minutes are 60 seconds long. Leap seconds are "smeared" so that no leap +-// second table is needed for interpretation, using a [24-hour linear +-// smear](https://developers.google.com/time/smear). +-// +-// The range is from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. By +-// restricting to that range, we ensure that we can convert to and from [RFC +-// 3339](https://www.ietf.org/rfc/rfc3339.txt) date strings. +-// +-// # Examples +-// +-// Example 1: Compute Timestamp from POSIX `time()`. +-// +-// Timestamp timestamp; +-// timestamp.set_seconds(time(NULL)); +-// timestamp.set_nanos(0); +-// +-// Example 2: Compute Timestamp from POSIX `gettimeofday()`. +-// +-// struct timeval tv; +-// gettimeofday(&tv, NULL); +-// +-// Timestamp timestamp; +-// timestamp.set_seconds(tv.tv_sec); +-// timestamp.set_nanos(tv.tv_usec * 1000); +-// +-// Example 3: Compute Timestamp from Win32 `GetSystemTimeAsFileTime()`. +-// +-// FILETIME ft; +-// GetSystemTimeAsFileTime(&ft); +-// UINT64 ticks = (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; +-// +-// // A Windows tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z +-// // is 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z. +-// Timestamp timestamp; +-// timestamp.set_seconds((INT64) ((ticks / 10000000) - 11644473600LL)); +-// timestamp.set_nanos((INT32) ((ticks % 10000000) * 100)); +-// +-// Example 4: Compute Timestamp from Java `System.currentTimeMillis()`. +-// +-// long millis = System.currentTimeMillis(); +-// +-// Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000) +-// .setNanos((int) ((millis % 1000) * 1000000)).build(); +-// +-// +-// Example 5: Compute Timestamp from current time in Python. +-// +-// timestamp = Timestamp() +-// timestamp.GetCurrentTime() +-// +-// # JSON Mapping +-// +-// In JSON format, the Timestamp type is encoded as a string in the +-// [RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format. That is, the +-// format is "{year}-{month}-{day}T{hour}:{min}:{sec}[.{frac_sec}]Z" +-// where {year} is always expressed using four digits while {month}, {day}, +-// {hour}, {min}, and {sec} are zero-padded to two digits each. The fractional +-// seconds, which can go up to 9 digits (i.e. up to 1 nanosecond resolution), +-// are optional. The "Z" suffix indicates the timezone ("UTC"); the timezone +-// is required. A proto3 JSON serializer should always use UTC (as indicated by +-// "Z") when printing the Timestamp type and a proto3 JSON parser should be +-// able to accept both UTC and other timezones (as indicated by an offset). +-// +-// For example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past +-// 01:30 UTC on January 15, 2017. +-// +-// In JavaScript, one can convert a Date object to this format using the +-// standard +-// [toISOString()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString) +-// method. In Python, a standard `datetime.datetime` object can be converted +-// to this format using +-// [`strftime`](https://docs.python.org/2/library/time.html#time.strftime) with +-// the time format spec '%Y-%m-%dT%H:%M:%S.%fZ'. Likewise, in Java, one can use +-// the Joda Time's [`ISODateTimeFormat.dateTime()`]( +-// http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime%2D%2D +-// ) to obtain a formatter capable of generating timestamps in this format. +-// +-// +-message Timestamp { +- // Represents seconds of UTC time since Unix epoch +- // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to +- // 9999-12-31T23:59:59Z inclusive. +- int64 seconds = 1; +- +- // Non-negative fractions of a second at nanosecond resolution. Negative +- // second values with fractions must still have non-negative nanos values +- // that count forward in time. Must be from 0 to 999,999,999 +- // inclusive. +- int32 nanos = 2; +-} +diff --git a/src/api/services/sandbox/sandbox.proto b/src/api/services/sandbox/sandbox.proto +deleted file mode 100644 +index dcc78444..00000000 +--- a/src/api/services/sandbox/sandbox.proto ++++ /dev/null +@@ -1,204 +0,0 @@ +-/* +- Copyright The containerd Authors. +- +- Licensed under the Apache License, Version 2.0 (the "License"); +- you may not use this file except in compliance with the License. +- You may obtain a copy of the License at +- +- http://www.apache.org/licenses/LICENSE-2.0 +- +- Unless required by applicable law or agreed to in writing, software +- distributed under the License is distributed on an "AS IS" BASIS, +- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +- See the License for the specific language governing permissions and +- limitations under the License. +-*/ +- +-syntax = "proto3"; +- +-// Sandbox is a v2 runtime extension that allows more complex execution environments for containers. +-// This adds a notion of groups of containers that share same lifecycle and/or resources. +-// A few good fits for sandbox can be: +-// - A "pause" container in k8s, that acts as a parent process for child containers to hold network namespace. +-// - (micro)VMs that launch a VM process and executes containers inside guest OS. +-// containerd in this case remains implementation agnostic and delegates sandbox handling to runtimes. +-// See proposal and discussion here: https://github.com/containerd/containerd/issues/4131 +-package containerd.services.sandbox.v1; +- +-import "google/protobuf/any.proto"; +-import "google/protobuf/timestamp.proto"; +- +-import "sandbox/types/sandbox.proto"; +-import "sandbox/types/mount.proto"; +-import "sandbox/types/platform.proto"; +-import "sandbox/types/metrics.proto"; +- +-option go_package = "github.com/containerd/containerd/api/services/sandbox/v1;sandbox"; +- +-// Store provides a metadata storage interface for sandboxes. Similarly to `Containers`, +-// sandbox object includes info required to start a new instance, but no runtime state. +-// When running a new sandbox instance, store objects are used as base type to create from. +-service Store { +- rpc Create(StoreCreateRequest) returns (StoreCreateResponse); +- rpc Update(StoreUpdateRequest) returns (StoreUpdateResponse); +- rpc Delete(StoreDeleteRequest) returns (StoreDeleteResponse); +- rpc List(StoreListRequest) returns (StoreListResponse); +- rpc Get(StoreGetRequest) returns (StoreGetResponse); +-} +- +-message StoreCreateRequest { +- containerd.types.Sandbox sandbox = 1; +-} +- +-message StoreCreateResponse { +- containerd.types.Sandbox sandbox = 1; +-} +- +-message StoreUpdateRequest { +- containerd.types.Sandbox sandbox = 1; +- repeated string fields = 2; +-} +- +-message StoreUpdateResponse { +- containerd.types.Sandbox sandbox = 1; +-} +- +-message StoreDeleteRequest { +- string sandbox_id = 1; +-} +- +-message StoreDeleteResponse {} +- +-message StoreListRequest { +- repeated string filters = 1; +-} +- +-message StoreListResponse { +- repeated containerd.types.Sandbox list = 1; +-} +- +-message StoreGetRequest { +- string sandbox_id = 1; +-} +- +-message StoreGetResponse { +- containerd.types.Sandbox sandbox = 1; +-} +- +-// Controller is an interface to manage runtime sandbox instances. +-service Controller { +- rpc Create(ControllerCreateRequest) returns (ControllerCreateResponse); +- rpc Start(ControllerStartRequest) returns (ControllerStartResponse); +- rpc Platform(ControllerPlatformRequest) returns (ControllerPlatformResponse); +- rpc Stop(ControllerStopRequest) returns (ControllerStopResponse); +- rpc Wait(ControllerWaitRequest) returns (ControllerWaitResponse); +- rpc Status(ControllerStatusRequest) returns (ControllerStatusResponse); +- rpc Shutdown(ControllerShutdownRequest) returns (ControllerShutdownResponse); +- rpc Metrics(ControllerMetricsRequest) returns (ControllerMetricsResponse); +- rpc Update(ControllerUpdateRequest) returns (ControllerUpdateResponse); +-} +- +-message ControllerCreateRequest { +- string sandbox_id = 1; +- repeated containerd.types.Mount rootfs = 2; +- google.protobuf.Any options = 3; +- string netns_path = 4; +- map<string, string> annotations = 5; +- containerd.types.Sandbox sandbox = 6; +- string sandboxer = 10; +-} +- +-message ControllerCreateResponse { +- string sandbox_id = 1; +-} +- +-message ControllerStartRequest { +- string sandbox_id = 1; +- string sandboxer = 10; +-} +- +-message ControllerStartResponse { +- string sandbox_id = 1; +- uint32 pid = 2; +- google.protobuf.Timestamp created_at = 3; +- map<string, string> labels = 4; +- // Address of the sandbox for containerd to connect, +- // for calling Task or other APIs serving in the sandbox. +- // it is in the form of ttrpc+unix://path/to/uds or grpc+vsock://<vsock cid>:<port>. +- string address = 5; +- uint32 version = 6; +-} +- +-message ControllerPlatformRequest { +- string sandbox_id = 1; +- string sandboxer = 10; +-} +- +-message ControllerPlatformResponse { +- containerd.types.Platform platform = 1; +-} +- +-message ControllerStopRequest { +- string sandbox_id = 1; +- uint32 timeout_secs = 2; +- string sandboxer = 10; +-} +- +-message ControllerStopResponse {} +- +-message ControllerWaitRequest { +- string sandbox_id = 1; +- string sandboxer = 10; +-} +- +-message ControllerWaitResponse { +- uint32 exit_status = 1; +- google.protobuf.Timestamp exited_at = 2; +-} +- +-message ControllerStatusRequest { +- string sandbox_id = 1; +- bool verbose = 2; +- string sandboxer = 10; +-} +- +-message ControllerStatusResponse { +- string sandbox_id = 1; +- uint32 pid = 2; +- string state = 3; +- map<string, string> info = 4; +- google.protobuf.Timestamp created_at = 5; +- google.protobuf.Timestamp exited_at = 6; +- google.protobuf.Any extra = 7; +- // Address of the sandbox for containerd to connect, +- // for calling Task or other APIs serving in the sandbox. +- // it is in the form of ttrpc+unix://path/to/uds or grpc+vsock://<vsock cid>:<port>. +- string address = 8; +- uint32 version = 9; +-} +- +-message ControllerShutdownRequest { +- string sandbox_id = 1; +- string sandboxer = 10; +-} +- +-message ControllerShutdownResponse {} +- +-message ControllerMetricsRequest { +- string sandbox_id = 1; +- string sandboxer = 10; +-} +- +-message ControllerMetricsResponse { +- types.Metric metrics = 1; +-} +- +-message ControllerUpdateRequest { +- string sandbox_id = 1; +- string sandboxer = 2; +- containerd.types.Sandbox sandbox = 3; +- repeated string fields = 4; +-} +- +-message ControllerUpdateResponse { +-} +\ No newline at end of file +diff --git a/src/api/services/sandbox/sandbox/types/metrics.proto b/src/api/services/sandbox/sandbox/types/metrics.proto +deleted file mode 100644 +index 61185939..00000000 +--- a/src/api/services/sandbox/sandbox/types/metrics.proto ++++ /dev/null +@@ -1,30 +0,0 @@ +-/* +- Copyright The containerd Authors. +- +- Licensed under the Apache License, Version 2.0 (the "License"); +- you may not use this file except in compliance with the License. +- You may obtain a copy of the License at +- +- http://www.apache.org/licenses/LICENSE-2.0 +- +- Unless required by applicable law or agreed to in writing, software +- distributed under the License is distributed on an "AS IS" BASIS, +- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +- See the License for the specific language governing permissions and +- limitations under the License. +-*/ +- +-syntax = "proto3"; +- +-package containerd.types; +- +-import "google/protobuf/any.proto"; +-import "google/protobuf/timestamp.proto"; +- +-option go_package = "github.com/containerd/containerd/api/types;types"; +- +-message Metric { +- google.protobuf.Timestamp timestamp = 1; +- string id = 2; +- google.protobuf.Any data = 3; +-} +\ No newline at end of file +diff --git a/src/api/services/sandbox/sandbox/types/mount.proto b/src/api/services/sandbox/sandbox/types/mount.proto +deleted file mode 100644 +index 54e0a0cd..00000000 +--- a/src/api/services/sandbox/sandbox/types/mount.proto ++++ /dev/null +@@ -1,43 +0,0 @@ +-/* +- Copyright The containerd Authors. +- +- Licensed under the Apache License, Version 2.0 (the "License"); +- you may not use this file except in compliance with the License. +- You may obtain a copy of the License at +- +- http://www.apache.org/licenses/LICENSE-2.0 +- +- Unless required by applicable law or agreed to in writing, software +- distributed under the License is distributed on an "AS IS" BASIS, +- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +- See the License for the specific language governing permissions and +- limitations under the License. +-*/ +- +-syntax = "proto3"; +- +-package containerd.types; +- +-option go_package = "github.com/containerd/containerd/api/types;types"; +- +-// Mount describes mounts for a container. +-// +-// This type is the lingua franca of ContainerD. All services provide mounts +-// to be used with the container at creation time. +-// +-// The Mount type follows the structure of the mount syscall, including a type, +-// source, target and options. +-message Mount { +- // Type defines the nature of the mount. +- string type = 1; +- +- // Source specifies the name of the mount. Depending on mount type, this +- // may be a volume name or a host path, or even ignored. +- string source = 2; +- +- // Target path in container +- string target = 3; +- +- // Options specifies zero or more fstab style mount options. +- repeated string options = 4; +-} +diff --git a/src/api/services/sandbox/sandbox/types/platform.proto b/src/api/services/sandbox/sandbox/types/platform.proto +deleted file mode 100644 +index 102e6e2b..00000000 +--- a/src/api/services/sandbox/sandbox/types/platform.proto ++++ /dev/null +@@ -1,30 +0,0 @@ +-/* +- Copyright The containerd Authors. +- +- Licensed under the Apache License, Version 2.0 (the "License"); +- you may not use this file except in compliance with the License. +- You may obtain a copy of the License at +- +- http://www.apache.org/licenses/LICENSE-2.0 +- +- Unless required by applicable law or agreed to in writing, software +- distributed under the License is distributed on an "AS IS" BASIS, +- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +- See the License for the specific language governing permissions and +- limitations under the License. +-*/ +- +-syntax = "proto3"; +- +-package containerd.types; +- +-option go_package = "github.com/containerd/containerd/api/types;types"; +- +-// Platform follows the structure of the OCI platform specification, from +-// descriptors. +-message Platform { +- string os = 1; +- string architecture = 2; +- string variant = 3; +- string os_version = 4; +-} +\ No newline at end of file +diff --git a/src/api/services/sandbox/sandbox/types/sandbox.proto b/src/api/services/sandbox/sandbox/types/sandbox.proto +deleted file mode 100644 +index 6fe08d40..00000000 +--- a/src/api/services/sandbox/sandbox/types/sandbox.proto ++++ /dev/null +@@ -1,54 +0,0 @@ +-/* +- Copyright The containerd Authors. +- +- Licensed under the Apache License, Version 2.0 (the "License"); +- you may not use this file except in compliance with the License. +- You may obtain a copy of the License at +- +- http://www.apache.org/licenses/LICENSE-2.0 +- +- Unless required by applicable law or agreed to in writing, software +- distributed under the License is distributed on an "AS IS" BASIS, +- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +- See the License for the specific language governing permissions and +- limitations under the License. +-*/ +- +-syntax = "proto3"; +- +-package containerd.types; +- +-import "google/protobuf/any.proto"; +-import "google/protobuf/timestamp.proto"; +- +-option go_package = "github.com/containerd/containerd/api/types;types"; +- +-// Sandbox represents a sandbox metadata object that keeps all info required by controller to +-// work with a particular instance. +-message Sandbox { +- // SandboxID is a unique instance identifier within namespace +- string sandbox_id = 1; +- message Runtime { +- // Name is the name of the runtime. +- string name = 1; +- // Options specify additional runtime initialization options for the shim (this data will be available in StartShim). +- // Typically this data expected to be runtime shim implementation specific. +- google.protobuf.Any options = 2; +- } +- // Runtime specifies which runtime to use for executing this container. +- Runtime runtime = 2; +- // Spec is sandbox configuration (kin of OCI runtime spec), spec's data will be written to a config.json file in the +- // bundle directory (similary to OCI spec). +- google.protobuf.Any spec = 3; +- // Labels provides an area to include arbitrary data on containers. +- map<string, string> labels = 4; +- // CreatedAt is the time the container was first created. +- google.protobuf.Timestamp created_at = 5; +- // UpdatedAt is the last time the container was mutated. +- google.protobuf.Timestamp updated_at = 6; +- // Extensions allow clients to provide optional blobs that can be handled by runtime. +- map<string, google.protobuf.Any> extensions = 7; +- // Sandboxer is the name of the sandbox controller who manages the sandbox. +- string sandboxer = 10; +- +-} +\ No newline at end of file +diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.cc b/src/daemon/common/cri/v1/v1_cri_helpers.cc +index 31b6b137..dfe14ade 100644 +--- a/src/daemon/common/cri/v1/v1_cri_helpers.cc ++++ b/src/daemon/common/cri/v1/v1_cri_helpers.cc +@@ -397,7 +397,7 @@ std::string CRISandboxerConvert(const std::string &runtime) + std::string sandboxer; + defs_map_string_object_sandboxer *criSandboxerList = nullptr; + +- if (runtime.empty() || runtime == DEFAULT_SANDBOXER_NAME) { ++ if (runtime.empty()) { + return DEFAULT_SANDBOXER_NAME; + } + +@@ -412,6 +412,7 @@ std::string CRISandboxerConvert(const std::string &runtime) + goto out; + } + ++ sandboxer = DEFAULT_SANDBOXER_NAME; + criSandboxerList = args->json_confs->cri_sandboxers; + for (size_t i = 0; i < criSandboxerList->len; i++) { + if (criSandboxerList->keys[i] == nullptr || criSandboxerList->values[i] == nullptr || +diff --git a/src/daemon/sandbox/CMakeLists.txt b/src/daemon/sandbox/CMakeLists.txt +index 7679ea68..2c31f243 100644 +--- a/src/daemon/sandbox/CMakeLists.txt ++++ b/src/daemon/sandbox/CMakeLists.txt +@@ -1,15 +1,32 @@ + # get current directory sources files + + aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} sandbox_top_srcs) +-add_subdirectory(controller) + +-set(SANDBOX_SRCS ++set(local_sandbox_top_srcs + ${sandbox_top_srcs} +- ${SANDBOX_CONTROLLER_SRCS} ++ ) ++set(local_sandbox_top_incs ++ ${CMAKE_CURRENT_SOURCE_DIR} ++ ) ++ ++if (ENABLE_CRI_API_V1 AND ENABLE_SANDBOXER) ++ add_subdirectory(sandboxer) ++ list (APPEND local_sandbox_top_srcs ++ ${SANDBOXER_SANDBOX_SRCS} ++ ) ++ list (APPEND local_sandbox_top_incs ++ ${SANDBOXER_SANDBOX_INCS} ++ ) ++endif() ++ ++add_subdirectory(shim) ++set(SANDBOX_SRCS ++ ${local_sandbox_top_srcs} ++ ${SHIM_SANDBOX_SRCS} + PARENT_SCOPE + ) + set(SANDBOX_INCS +- ${CMAKE_CURRENT_SOURCE_DIR} +- ${SANDBOX_CONTROLLER_INCS} ++ ${local_sandbox_top_incs} ++ ${SHIM_SANDBOX_INCS} + PARENT_SCOPE + ) +\ No newline at end of file +diff --git a/src/daemon/sandbox/controller/controller.h b/src/daemon/sandbox/controller.h +similarity index 79% +rename from src/daemon/sandbox/controller/controller.h +rename to src/daemon/sandbox/controller.h +index 60d2dee5..0975015a 100644 +--- a/src/daemon/sandbox/controller/controller.h ++++ b/src/daemon/sandbox/controller.h +@@ -22,15 +22,19 @@ + #include <grpc++/grpc++.h> + #include <stdint.h> + ++#include <isula_libutils/sandbox_sandbox.h> ++ + #include "errors.h" + #include "api_v1.pb.h" +-#include "sandbox.pb.h" ++#include "utils_array.h" + + namespace sandbox { + + #define SANDBOX_READY_STATE_STR "SANDBOX_READY" + #define SANDBOX_NOTREADY_STATE_STR "SANDBOX_NOTREADY" + ++const std::string SHIM_CONTROLLER_NAME = "shim"; ++ + struct ControllerMountInfo { + std::string source; + std::string destination; +@@ -66,7 +70,7 @@ struct ControllerSandboxInfo { + uint32_t pid; + uint64_t createdAt; + std::string taskAddress; +- std::string version; ++ uint32_t version; + google::protobuf::Map<std::string, std::string> labels; + }; + +@@ -80,7 +84,7 @@ struct ControllerSandboxStatus { + uint32_t pid; + std::string state; + std::string taskAddress; +- std::string version; ++ uint32_t version; + google::protobuf::Map<std::string, std::string> info; + uint64_t createdAt; + uint64_t exitedAt; +@@ -95,20 +99,6 @@ struct ControllerStreamInfo { + bool terminal; + }; + +-struct ControllerPrepareParams { +- std::string containerId; +- std::string execId; +- std::unique_ptr<std::string> spec; +- std::vector<std::unique_ptr<ControllerMountInfo>> rootfs; +- std::unique_ptr<ControllerStreamInfo> streamInfo; +-}; +- +-struct ControllerUpdateResourcesParams { +- std::string containerId; +- std::unique_ptr<std::string> resources; +- google::protobuf::Map<std::string, std::string> &annotations; +-}; +- + class SandboxStatusCallback { + public: + virtual void OnSandboxReady() = 0; +@@ -120,19 +110,13 @@ class Controller { + public: + virtual ~Controller() {}; + virtual bool Init(Errors &error) = 0; +- virtual void Destroy() = 0; + virtual bool Create(const std::string &sandboxId, + const ControllerCreateParams ¶ms, + Errors &error) = 0; + virtual std::unique_ptr<ControllerSandboxInfo> Start(const std::string &sandboxId, Errors &error) = 0 ; + virtual std::unique_ptr<ControllerPlatformInfo> Platform(const std::string &sandboxId, Errors &error) = 0; +- virtual bool Prepare(containerd::types::Sandbox &apiSandbox, +- std::vector<std::string> &fields, Errors &error) = 0; +- virtual bool Purge(containerd::types::Sandbox &apiSandbox, +- std::vector<std::string> &fields, Errors &error) = 0; +- virtual bool UpdateResources(const std::string &sandboxId, +- const ControllerUpdateResourcesParams ¶ms, +- Errors &error) = 0; ++ virtual bool Update(sandbox_sandbox *apiSandbox, ++ string_array *fields, Errors &error) = 0; + virtual bool Stop(const std::string &sandboxId, uint32_t timeoutSecs, Errors &error) = 0; + virtual bool Wait(std::shared_ptr<SandboxStatusCallback> cb, const std::string &sandboxId, Errors &error) = 0; + virtual std::unique_ptr<ControllerSandboxStatus> Status(const std::string &sandboxId, bool verbose, Errors &error) = 0; +diff --git a/src/daemon/sandbox/controller/CMakeLists.txt b/src/daemon/sandbox/controller/CMakeLists.txt +deleted file mode 100644 +index 8764c05b..00000000 +--- a/src/daemon/sandbox/controller/CMakeLists.txt ++++ /dev/null +@@ -1,32 +0,0 @@ +-# get current directory sources files +- +-aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} sandbox_controller_top_srcs) +- +-set(local_sandbox_controller_top_srcs +- ${sandbox_controller_top_srcs} +- ) +-set(local_sandbox_controller_top_incs +- ${CMAKE_CURRENT_SOURCE_DIR} +- ) +- +-if (ENABLE_CRI_API_V1 AND ENABLE_SANDBOXER) +- add_subdirectory(sandboxer) +- list (APPEND local_sandbox_controller_top_srcs +- ${CONTROLLER_SANDBOXER_SRCS} +- ) +- list (APPEND local_sandbox_controller_top_incs +- ${CONTROLLER_SANDBOXER_INCS} +- ) +-endif() +- +-add_subdirectory(shim) +-set(SANDBOX_CONTROLLER_SRCS +- ${local_sandbox_controller_top_srcs} +- ${CONTROLLER_SHIM_SRCS} +- PARENT_SCOPE +- ) +-set(SANDBOX_CONTROLLER_INCS +- ${local_sandbox_controller_top_incs} +- ${CONTROLLER_SHIM_INCS} +- PARENT_SCOPE +- ) +\ No newline at end of file +diff --git a/src/daemon/sandbox/controller/sandboxer/client/CMakeLists.txt b/src/daemon/sandbox/controller/sandboxer/client/CMakeLists.txt +deleted file mode 100755 +index 958ebaa5..00000000 +--- a/src/daemon/sandbox/controller/sandboxer/client/CMakeLists.txt ++++ /dev/null +@@ -1,20 +0,0 @@ +-# get current directory sources files +-aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} sandbox_controller_sandboxer_client_srcs) +-aux_source_directory(${CMAKE_BINARY_DIR}/grpc/src/api/services/sandbox grpc_sandbox_api_srcs) +-aux_source_directory(${CMAKE_BINARY_DIR}/grpc/src/api/services/sandbox/google/protobuf grpc_sandbox_google_protobuf_api_srcs) +-aux_source_directory(${CMAKE_BINARY_DIR}/grpc/src/api/services/sandbox/sandbox/types sandbox_type_srcs) +- +- +-set(CONTROLLER_SANDBOXER_CLIENT_SRCS +- ${sandbox_controller_sandboxer_client_srcs} +- ${grpc_sandbox_api_srcs} +- ${grpc_sandbox_google_protobuf_api_srcs} +- ${sandbox_type_srcs} +- PARENT_SCOPE +- ) +- +-set(CONTROLLER_SANDBOXER_CLIENT_INCS +- ${CMAKE_CURRENT_SOURCE_DIR} +- ${CMAKE_BINARY_DIR}/grpc/src/api/services/sandbox +- PARENT_SCOPE +- ) +diff --git a/src/daemon/sandbox/controller/sandboxer/client/grpc_async_wait_call.cc b/src/daemon/sandbox/controller/sandboxer/client/grpc_async_wait_call.cc +deleted file mode 100644 +index c6f97da1..00000000 +--- a/src/daemon/sandbox/controller/sandboxer/client/grpc_async_wait_call.cc ++++ /dev/null +@@ -1,142 +0,0 @@ +-/****************************************************************************** +- * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +- * iSulad licensed under the Mulan PSL v2. +- * You can use this software according to the terms and conditions of the Mulan PSL v2. +- * You may obtain a copy of Mulan PSL v2 at: +- * http://license.coscl.org.cn/MulanPSL2 +- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +- * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +- * PURPOSE. +- * See the Mulan PSL v2 for more details. +- * Author: xuxuepeng +- * Create: 2023-07-28 +- * Description: gRPC Async wait call object +- ******************************************************************************/ +- +-#include "grpc_async_wait_call.h" +- +-#include <grpc++/grpc++.h> +-#include <isula_libutils/log.h> +- +-#include "grpc_client_utils.h" +- +-namespace sandbox { +- +-// Max retry counter. In monitor thread, we will check deferred calls every 200ms +-// so if a call has retry counter for 1500, it means the call will be retried after 300 seconds +-const int64_t MAX_RETRY_COUNTER = 1500; // 1500 * 200ms = 300s +- +-SandboxerAsyncWaitCall::SandboxerAsyncWaitCall(std::shared_ptr<SandboxStatusCallback> cb, +- const std::string &sandboxId, const std::string &sandboxer) +- : m_cb(cb), m_sandboxId(sandboxId), m_sandboxer(sandboxer) +-{ +- m_status = grpc::Status::OK; +- m_retryTimes = 0; +- m_retryCounter = 0; +- m_remove = false; +-} +- +-auto SandboxerAsyncWaitCall::Call(containerd::services::sandbox::v1::Controller::StubInterface &stub, +- grpc::CompletionQueue &cq) -> bool +-{ +- containerd::services::sandbox::v1::ControllerWaitRequest request; +- m_context = std::unique_ptr<grpc::ClientContext>(new grpc::ClientContext()); +- request.set_sandboxer(m_sandboxer); +- request.set_sandbox_id(m_sandboxId); +- m_responseReader = stub.PrepareAsyncWait(m_context.get(), request, &cq); +- if (m_responseReader == nullptr) { +- // Most likely the completion queue is shutdown +- ERROR("Failed to prepare async wait request for sandboxer wait request, sandbox id: %s", m_sandboxId.c_str()); +- SandboxExitCallback(false, ControllerExitInfo()); +- return false; +- } +- m_responseReader->StartCall(); +- m_responseReader->Finish(&m_response, &m_status, (void *)this); +- return true; +-} +- +-auto SandboxerAsyncWaitCall::SandboxExitCallback(bool statusOK, const ControllerExitInfo &exitInfo) -> void +-{ +- // If statusOK is false, it means something unexpected happened during async wait, +- // the exitInfo is not valid, but we assume that the sandbox has exited. +- if (!statusOK) { +- ControllerExitInfo info; +- auto currentTime = std::chrono::high_resolution_clock::now(); +- auto duration = currentTime.time_since_epoch(); +- info.exitedAt = std::chrono::duration_cast<std::chrono::nanoseconds>(duration).count(); +- info.exitStatus = -1; +- m_cb->OnSandboxExit(info); +- return; +- } +- m_cb->OnSandboxExit(exitInfo); +-} +- +-auto SandboxerAsyncWaitCall::SandboxPendingCallback() -> void +-{ +- m_cb->OnSandboxPending(); +-} +- +-auto SandboxerAsyncWaitCall::SandboxReadyCallback() -> void +-{ +- m_cb->OnSandboxReady(); +-} +- +-auto SandboxerAsyncWaitCall::Timeout() -> bool +-{ +- m_retryCounter--; +- return m_retryCounter == 0; +-} +- +-auto SandboxerAsyncWaitCall::GetSandboxId() -> const std::string & +-{ +- return m_sandboxId; +-} +- +-SandboxerAsyncWaitStatus SandboxerAsyncWaitCall::HandleResponse() +-{ +- ControllerExitInfo exitInfo; +- SandboxerAsyncWaitStatus waitStatus = SANDBOXER_ASYNC_WAIT_STATUS_ERROR; +- +- switch (m_status.error_code()) { +- case grpc::StatusCode::UNAVAILABLE: +- // If the status is unavailable, connection failed, we should retry +- WARN("Sandboxer controller wait rpc server unavailable, error_code: %d: %s", m_status.error_code(), +- m_status.error_message().c_str()); +- waitStatus = SANDBOXER_ASYNC_WAIT_STATUS_RETRY; +- m_retryTimes++; +- // If retried times is more than 10, we should retry every 300 seconds +- if (m_retryTimes > 10) { +- m_retryCounter = MAX_RETRY_COUNTER; +- } else { +- // Retry interval is 2 ^ retry times +- m_retryCounter = 1 << m_retryTimes; +- } +- m_cb->OnSandboxPending(); +- break; +- case grpc::StatusCode::OK: +- exitInfo.exitedAt = TimestampToNanos(m_response.exited_at()); +- exitInfo.exitStatus = m_response.exit_status(); +- DEBUG("Sandboxer controller wait request success, sandbox id: %s, exit status: %d, exited at: %lu", +- m_sandboxId.c_str(), exitInfo.exitStatus, exitInfo.exitedAt); +- SandboxExitCallback(true, exitInfo); +- waitStatus = SANDBOXER_ASYNC_WAIT_STATUS_OK; +- break; +- case grpc::StatusCode::NOT_FOUND: +- // If the sandbox is not found, it has been deleted, we should return not found +- WARN("The sandbox wait doesn't exist, sandbox id: %s, error_code: %d: %s", +- m_sandboxId.c_str(), m_status.error_code(), m_status.error_message().c_str()); +- waitStatus = SANDBOXER_ASYNC_WAIT_STATUS_NOT_FOUND; +- SandboxExitCallback(false, exitInfo); +- break; +- default: +- // TODO: More error code should be handled +- ERROR("Sandboxer controller wait request failed, error_code: %d: %s", m_status.error_code(), +- m_status.error_message().c_str()); +- SandboxExitCallback(false, exitInfo); +- break; +- } +- +- return waitStatus; +-} +- +-}; // namespace sandbox +\ No newline at end of file +diff --git a/src/daemon/sandbox/controller/sandboxer/client/grpc_async_wait_call.h b/src/daemon/sandbox/controller/sandboxer/client/grpc_async_wait_call.h +deleted file mode 100644 +index 6e5a6756..00000000 +--- a/src/daemon/sandbox/controller/sandboxer/client/grpc_async_wait_call.h ++++ /dev/null +@@ -1,94 +0,0 @@ +-/****************************************************************************** +- * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +- * iSulad licensed under the Mulan PSL v2. +- * You can use this software according to the terms and conditions of the Mulan PSL v2. +- * You may obtain a copy of Mulan PSL v2 at: +- * http://license.coscl.org.cn/MulanPSL2 +- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +- * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +- * PURPOSE. +- * See the Mulan PSL v2 for more details. +- * Author: xuxuepeng +- * Create: 2023-07-28 +- * Description: gRPC Async wait call object +- ******************************************************************************/ +- +-#ifndef DAEMON_SANDBOX_CONTROLLER_SANDBOXER_CLIENT_GRPC_ASYNC_WAIT_CALL_H +-#define DAEMON_SANDBOX_CONTROLLER_SANDBOXER_CLIENT_GRPC_ASYNC_WAIT_CALL_H +- +-#include <string> +- +-#include "sandbox.pb.h" +-#include "sandbox.grpc.pb.h" +- +-#include "controller.h" +- +-namespace sandbox { +- +-enum SandboxerAsyncWaitStatus { +- SANDBOXER_ASYNC_WAIT_STATUS_OK = 0, +- SANDBOXER_ASYNC_WAIT_STATUS_RETRY, +- SANDBOXER_ASYNC_WAIT_STATUS_NOT_FOUND, +- SANDBOXER_ASYNC_WAIT_STATUS_ERROR, +-}; +- +-/** +- * SandboxerAsyncWaitCall is used to call the async wait method. +- * It will be passed to completion queue to wait for the response. +- * When the response is received, HandleResponse will be called. +- */ +-class SandboxerAsyncWaitCall { +-public: +- SandboxerAsyncWaitCall(std::shared_ptr<SandboxStatusCallback> cb, +- const std::string &sandboxId, const std::string &sandboxer); +- virtual ~SandboxerAsyncWaitCall() = default; +- auto Call(containerd::services::sandbox::v1::Controller::StubInterface &stub, grpc::CompletionQueue &cq) -> bool; +- auto HandleResponse() -> SandboxerAsyncWaitStatus; +- auto Timeout() -> bool; +- void SandboxExitCallback(bool statusOK, const ControllerExitInfo &exitInfo); +- void SandboxPendingCallback(); +- void SandboxReadyCallback(); +- auto GetSandboxId() -> const std::string &; +- auto MarkRemove() -> void +- { +- m_remove = true; +- } +- auto ToRemove() -> bool +- { +- return m_remove; +- } +- auto ResetRetryTimes() -> void +- { +- m_retryTimes = 0; +- } +- +-protected: +- std::shared_ptr<containerd::services::sandbox::v1::Controller::StubInterface> m_stub; +- std::shared_ptr<SandboxStatusCallback> m_cb; +- std::string m_sandboxId; +- std::string m_sandboxer; +- std::unique_ptr<grpc::ClientAsyncResponseReaderInterface<containerd::services::sandbox::v1::ControllerWaitResponse>> +- m_responseReader; +- std::unique_ptr<grpc::ClientContext> m_context; +- containerd::services::sandbox::v1::ControllerWaitResponse m_response; +- grpc::Status m_status; +- uint32_t m_retryTimes; +- uint64_t m_retryCounter; +- // The Call object will be deleted when it is unable to be enqueued to completion queue. +- // However, since gRPC call happens in async function in Future object, there is potential race condition +- // when deleting the call object: +- // 1. Monitor thread cleanup future object first, which holding the reference to call object, and +- // then AysncCompletionRpcThread received response for the call later, and delete the call object. +- // It is OK in this case, since future release the call ownership, and the call object deleted later. +- // 2. AsyncCompletionRpcThread received response for the call first, and delete the call object, and +- // then Monitor thread cleanup future object later, which could cause use-after-free, since monitor +- // thread will use the call to call the callback when cleaning the future object. +- // So we need to mark the call object as removed, cleanup them together in Monitor thread after checking +- // the future objects. +- bool m_remove; +-}; +- +- +-}; // namespace sandbox +- +-#endif // DAEMON_SANDBOX_CONTROLLER_SANDBOXER_CLIENT_GRPC_ASYNC_WAIT_CALL_H +\ No newline at end of file +diff --git a/src/daemon/sandbox/controller/sandboxer/client/grpc_client_utils.cc b/src/daemon/sandbox/controller/sandboxer/client/grpc_client_utils.cc +deleted file mode 100644 +index 2cc5a11e..00000000 +--- a/src/daemon/sandbox/controller/sandboxer/client/grpc_client_utils.cc ++++ /dev/null +@@ -1,36 +0,0 @@ +-/****************************************************************************** +- * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +- * iSulad licensed under the Mulan PSL v2. +- * You can use this software according to the terms and conditions of the Mulan PSL v2. +- * You may obtain a copy of Mulan PSL v2 at: +- * http://license.coscl.org.cn/MulanPSL2 +- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +- * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +- * PURPOSE. +- * See the Mulan PSL v2 for more details. +- * Author: xuxuepeng +- * Create: 2023-07-28 +- * Description: grpc client utils functions +- ******************************************************************************/ +- +-#include "grpc_client_utils.h" +- +-const uint64_t SECOND_TO_NANOS = 1000000000; +-const int64_t MAX_SECONDS_FOR_TIMESTAMP = 253402300799; // 9999-12-31T23:59:59Z +-const int32_t MAX_NANOS_FOR_TIMESTAMP = 999999999; +- +-auto TimestampToNanos(const google::protobuf::Timestamp ×tamp) -> uint64_t +-{ +- int64_t seconds = 0; +- int32_t nanos = 0; +- +- seconds = timestamp.seconds(); +- seconds = seconds < 0 ? 0 : seconds; +- seconds = seconds > MAX_SECONDS_FOR_TIMESTAMP ? MAX_SECONDS_FOR_TIMESTAMP : seconds; +- +- nanos = timestamp.nanos(); +- nanos = nanos < 0 ? 0 : nanos; +- nanos = nanos > MAX_NANOS_FOR_TIMESTAMP ? MAX_NANOS_FOR_TIMESTAMP : nanos; +- +- return seconds * SECOND_TO_NANOS + nanos; +-} +diff --git a/src/daemon/sandbox/controller/sandboxer/client/grpc_client_utils.h b/src/daemon/sandbox/controller/sandboxer/client/grpc_client_utils.h +deleted file mode 100644 +index 105d33fc..00000000 +--- a/src/daemon/sandbox/controller/sandboxer/client/grpc_client_utils.h ++++ /dev/null +@@ -1,23 +0,0 @@ +-/****************************************************************************** +- * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +- * iSulad licensed under the Mulan PSL v2. +- * You can use this software according to the terms and conditions of the Mulan PSL v2. +- * You may obtain a copy of Mulan PSL v2 at: +- * http://license.coscl.org.cn/MulanPSL2 +- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +- * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +- * PURPOSE. +- * See the Mulan PSL v2 for more details. +- * Author: xuxuepeng +- * Create: 2023-07-28 +- * Description: grpc client utils functions +- ******************************************************************************/ +- +-#ifndef DAEMON_SANDBOX_CONTROLLER_SANDBOXER_CLIENT_GRPC_CLIENT_UTILS_H +-#define DAEMON_SANDBOX_CONTROLLER_SANDBOXER_CLIENT_GRPC_CLIENT_UTILS_H +- +-#include <google/protobuf/timestamp.pb.h> +- +-uint64_t TimestampToNanos(const google::protobuf::Timestamp ×tamp); +- +-#endif // DAEMON_SANDBOX_CONTROLLER_SANDBOXER_CLIENT_GRPC_CLIENT_UTILS_H +\ No newline at end of file +diff --git a/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_client.cc b/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_client.cc +deleted file mode 100644 +index e042ad45..00000000 +--- a/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_client.cc ++++ /dev/null +@@ -1,337 +0,0 @@ +-/****************************************************************************** +- * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +- * iSulad licensed under the Mulan PSL v2. +- * You can use this software according to the terms and conditions of the Mulan PSL v2. +- * You may obtain a copy of Mulan PSL v2 at: +- * http://license.coscl.org.cn/MulanPSL2 +- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +- * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +- * PURPOSE. +- * See the Mulan PSL v2 for more details. +- * Author: xuxuepeng +- * Create: 2023-07-10 +- * Description: Sandboxer grpc client +- ******************************************************************************/ +- +-#include "grpc_sandboxer_client.h" +- +-#include <grpc++/grpc++.h> +-#include <iostream> +-#include <memory> +-#include <string> +-#include <random> +- +-#include "sandbox/types/platform.pb.h" +-#include "sandbox.pb.h" +-#include "sandbox.grpc.pb.h" +-#include "utils.h" +-#include "isula_libutils/log.h" +- +-#include "grpc_client_utils.h" +- +-namespace sandbox { +- +-SandboxerClient::SandboxerClient(const std::string &sandboxer, const std::string &address): +- m_sandboxer(sandboxer), m_address(address) +-{ +- std::string unixPrefix(UNIX_SOCKET_PREFIX); +- +- // Only support unix domain socket +- if (m_address.compare(0, unixPrefix.length(), unixPrefix) != 0) { +- m_address = unixPrefix + m_address; +- } +- m_channel = grpc::CreateChannel(m_address, grpc::InsecureChannelCredentials()); +- m_stub = containerd::services::sandbox::v1::Controller::NewStub(m_channel); +- // Monitor shares the same channel with client and has its own stub +- m_monitor = std::unique_ptr<SandboxerClientMonitor>(new SandboxerClientMonitor(m_channel, m_sandboxer)); +-} +- +-void SandboxerClient::InitMountInfo(Mount &mount, const ControllerMountInfo &mountInfo) +-{ +- mount.set_type(mountInfo.type); +- mount.set_source(mountInfo.source); +- mount.set_target(mountInfo.destination); +- for (const auto &option : mountInfo.options) { +- mount.add_options(option); +- } +-} +- +-auto SandboxerClient::InitCreateRequest(containerd::services::sandbox::v1::ControllerCreateRequest &request, +- const std::string &sandboxId, +- const ControllerCreateParams ¶ms) -> bool +-{ +- if (params.config == nullptr) { +- ERROR("Sandboxer controller create request failed, config is null"); +- return false; +- } +- request.mutable_options()->PackFrom(*params.config); +- request.set_sandboxer(m_sandboxer); +- request.set_sandbox_id(sandboxId); +- for (const auto &entry : params.mounts) { +- if (entry != nullptr) { +- Mount* mount = request.add_rootfs(); +- InitMountInfo(*mount, *entry); +- } +- } +- request.set_netns_path(params.netNSPath); +- // The arg sandbox is useless for now +- return true; +-} +- +-void SandboxerClient::Init(Errors &error) +-{ +- m_monitor->Start(); +-} +- +-void SandboxerClient::Destroy() +-{ +- m_monitor->Stop(); +-} +- +-auto SandboxerClient::Create(const std::string &sandboxId, const ControllerCreateParams ¶ms, Errors &error) -> bool +-{ +- grpc::ClientContext context; +- containerd::services::sandbox::v1::ControllerCreateRequest request; +- containerd::services::sandbox::v1::ControllerCreateResponse response; +- grpc::Status status; +- +- if (!InitCreateRequest(request, sandboxId, params)) { +- error.SetError("Failed to init create request for sandboxer create request, sandbox id: " + sandboxId); +- return false; +- } +- +- status = m_stub->Create(&context, request, &response); +- if (!status.ok()) { +- error.SetError(status.error_message()); +- ERROR("Sandboxer controller create request failed, error_code: %d: %s", status.error_code(), +- status.error_message().c_str()); +- return false; +- } +- +- return true; +-} +- +-void SandboxerClient::StartResponseToSandboxInfo(const containerd::services::sandbox::v1::ControllerStartResponse +- &response, +- ControllerSandboxInfo &sandboxInfo) +-{ +- sandboxInfo.id = response.sandbox_id(); +- sandboxInfo.pid = response.pid(); +- sandboxInfo.createdAt = TimestampToNanos(response.created_at()); +- sandboxInfo.taskAddress = response.address(); +- sandboxInfo.version = response.version(); +- sandboxInfo.labels = response.labels(); +-} +- +-auto SandboxerClient::Start(const std::string &sandboxId, ControllerSandboxInfo &sandboxInfo, Errors &error) -> bool +-{ +- grpc::ClientContext context; +- containerd::services::sandbox::v1::ControllerStartRequest request; +- containerd::services::sandbox::v1::ControllerStartResponse response; +- grpc::Status status; +- +- request.set_sandboxer(m_sandboxer); +- request.set_sandbox_id(sandboxId); +- +- status = m_stub->Start(&context, request, &response); +- if (!status.ok()) { +- error.SetError(status.error_message()); +- ERROR("Sandboxer controller start request failed, error_code: %d: %s", status.error_code(), +- status.error_message().c_str()); +- return false; +- } +- +- StartResponseToSandboxInfo(response, sandboxInfo); +- +- return true; +-} +- +-void SandboxerClient::InitUpdateRequest(containerd::services::sandbox::v1::ControllerUpdateRequest &request, +- containerd::types::Sandbox &apiSandbox, +- std::vector<std::string> &fields) +-{ +- request.set_sandbox_id(apiSandbox.sandbox_id()); +- request.set_sandboxer(apiSandbox.sandboxer()); +- *(request.mutable_sandbox()) = apiSandbox; +- *(request.mutable_fields()) = {fields.begin(), fields.end()}; +-} +- +-auto SandboxerClient::Prepare(containerd::types::Sandbox &apiSandbox, +- std::vector<std::string> &fields, Errors &error) -> bool +-{ +- grpc::ClientContext context; +- containerd::services::sandbox::v1::ControllerUpdateRequest request; +- containerd::services::sandbox::v1::ControllerUpdateResponse response; +- grpc::Status status; +- +- InitUpdateRequest(request, apiSandbox, fields); +- +- status = m_stub->Update(&context, request, &response); +- if (!status.ok()) { +- error.SetError(status.error_message()); +- ERROR("Sandboxer controller prepare request failed, error_code: %d: %s", status.error_code(), +- status.error_message().c_str()); +- return false; +- } +- +- return true; +-} +- +-auto SandboxerClient::Purge(containerd::types::Sandbox &apiSandbox, +- std::vector<std::string> &fields, Errors &error) -> bool +-{ +- grpc::ClientContext context; +- containerd::services::sandbox::v1::ControllerUpdateRequest request; +- containerd::services::sandbox::v1::ControllerUpdateResponse response; +- grpc::Status status; +- +- InitUpdateRequest(request, apiSandbox, fields); +- +- status = m_stub->Update(&context, request, &response); +- if (!status.ok()) { +- error.SetError(status.error_message()); +- ERROR("Sandboxer controller purge request failed, error_code: %d: %s", status.error_code(), +- status.error_message().c_str()); +- return false; +- } +- +- return true; +-} +- +-auto SandboxerClient::UpdateResources(const std::string &sandboxId, const ControllerUpdateResourcesParams ¶ms, +- Errors &error) -> bool +-{ +- return true; +-} +- +-void SandboxerClient::PlatformResponseToPlatformInfo(const containerd::services::sandbox::v1::ControllerPlatformResponse +- &response, +- ControllerPlatformInfo &platformInfo) +-{ +- auto &platform = response.platform(); +- platformInfo.os = platform.os(); +- platformInfo.arch = platform.architecture(); +- platformInfo.variant = platform.variant(); +-} +- +-auto SandboxerClient::Platform(const std::string &sandboxId, ControllerPlatformInfo &platformInfo, +- Errors &error) -> bool +-{ +- grpc::ClientContext context; +- containerd::services::sandbox::v1::ControllerPlatformRequest request; +- containerd::services::sandbox::v1::ControllerPlatformResponse response; +- grpc::Status status; +- +- request.set_sandboxer(m_sandboxer); +- request.set_sandbox_id(sandboxId); +- +- status = m_stub->Platform(&context, request, &response); +- if (!status.ok()) { +- error.SetError(status.error_message()); +- ERROR("Sandboxer controller platform request failed, error_code: %d: %s", status.error_code(), +- status.error_message().c_str()); +- return false; +- } +- +- PlatformResponseToPlatformInfo(response, platformInfo); +- +- return true; +-} +- +-auto SandboxerClient::Stop(const std::string &sandboxId, uint32_t timeoutSecs, Errors &error) -> bool +-{ +- grpc::ClientContext context; +- containerd::services::sandbox::v1::ControllerStopRequest request; +- containerd::services::sandbox::v1::ControllerStopResponse response; +- grpc::Status status; +- +- request.set_sandboxer(m_sandboxer); +- request.set_sandbox_id(sandboxId); +- request.set_timeout_secs(timeoutSecs); +- +- status = m_stub->Stop(&context, request, &response); +- if (!status.ok()) { +- error.SetError(status.error_message()); +- ERROR("Sandboxer controller stop request failed, error_code: %d: %s", status.error_code(), +- status.error_message().c_str()); +- return false; +- } +- +- return true; +-} +- +-void SandboxerClient::StatusResponseToSandboxStatus(const containerd::services::sandbox::v1::ControllerStatusResponse +- &response, +- ControllerSandboxStatus &sandboxStatus) +-{ +- sandboxStatus.id = response.sandbox_id(); +- sandboxStatus.pid = response.pid(); +- sandboxStatus.state = response.state(); +- sandboxStatus.taskAddress = response.address(); +- sandboxStatus.version = response.version(); +- sandboxStatus.info.insert(response.info().begin(), response.info().end()); +- sandboxStatus.createdAt = TimestampToNanos(response.created_at()); +- sandboxStatus.exitedAt = TimestampToNanos(response.exited_at()); +- sandboxStatus.extra = response.extra().value(); +-} +- +-auto SandboxerClient::Status(const std::string &sandboxId, bool verbose, ControllerSandboxStatus &sandboxStatus, +- Errors &error) -> bool +-{ +- grpc::ClientContext context; +- containerd::services::sandbox::v1::ControllerStatusRequest request; +- containerd::services::sandbox::v1::ControllerStatusResponse response; +- grpc::Status status; +- +- request.set_sandboxer(m_sandboxer); +- request.set_sandbox_id(sandboxId); +- request.set_verbose(verbose); +- +- status = m_stub->Status(&context, request, &response); +- if (!status.ok()) { +- error.SetError(status.error_message()); +- ERROR("Sandboxer controller status request failed, error_code: %d: %s", status.error_code(), +- status.error_message().c_str()); +- return false; +- } +- +- StatusResponseToSandboxStatus(response, sandboxStatus); +- +- return true; +-} +- +-auto SandboxerClient::Shutdown(const std::string &sandboxId, Errors &error) -> bool +-{ +- grpc::ClientContext context; +- containerd::services::sandbox::v1::ControllerShutdownRequest request; +- containerd::services::sandbox::v1::ControllerShutdownResponse response; +- grpc::Status status; +- +- request.set_sandboxer(m_sandboxer); +- request.set_sandbox_id(sandboxId); +- +- status = m_stub->Shutdown(&context, request, &response); +- if (!status.ok()) { +- error.SetError(status.error_message()); +- ERROR("Sandboxer controller shutdown request failed, error_code: %d: %s", status.error_code(), +- status.error_message().c_str()); +- return false; +- } +- +- return true; +-} +- +-auto SandboxerClient::Wait(std::shared_ptr<SandboxStatusCallback> cb, const std::string &sandboxId, +- Errors &error) -> bool +-{ +- if (m_monitor == nullptr) { +- error.SetError("Cannot wait for sandbox, sandboxer client monitor is not initialized, " +- "sandboxer: " + m_sandboxer); +- return false; +- } +- SandboxerAsyncWaitCall *call = new SandboxerAsyncWaitCall(cb, sandboxId, m_sandboxer); +- // Transfer ownership of call to monitor +- return m_monitor->Monitor(call); +-} +- +-} // namespace +diff --git a/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_monitor.cc b/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_monitor.cc +deleted file mode 100644 +index 485a0b23..00000000 +--- a/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_monitor.cc ++++ /dev/null +@@ -1,314 +0,0 @@ +-/****************************************************************************** +- * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +- * iSulad licensed under the Mulan PSL v2. +- * You can use this software according to the terms and conditions of the Mulan PSL v2. +- * You may obtain a copy of Mulan PSL v2 at: +- * http://license.coscl.org.cn/MulanPSL2 +- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +- * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +- * PURPOSE. +- * See the Mulan PSL v2 for more details. +- * Author: xuxuepeng +- * Create: 2023-07-28 +- * Description: Sandboxer grpc client monitor +- ******************************************************************************/ +- +-#include "grpc_sandboxer_monitor.h" +- +-#include <random> +- +-#include "isula_libutils/log.h" +- +-namespace sandbox { +- +-const int64_t DEFERRED_QUEUE_CHECK_INTERVAL = 200; // milliseconds +- +-SandboxerClientMonitor::SandboxerClientMonitor(std::shared_ptr<grpc::Channel> channel, const std::string &sandboxer): +- m_channel(channel), m_sandboxer(sandboxer), m_teardown(false) +-{ +- m_stub = containerd::services::sandbox::v1::Controller::NewStub(m_channel); +-} +- +-// Monitor will gain the ownership of the call object, and is responsible +-// for deleting it +-auto SandboxerClientMonitor::Monitor(SandboxerAsyncWaitCall *call) -> bool +-{ +- if (call == nullptr) { +- ERROR("Async wait call is null, failed to monitor."); +- return false; +- } +- +- if (m_teardown) { +- ERROR("Monitor is already shutdown, failed to monitor sandbox, id: %s", +- call->GetSandboxId().c_str()); +- delete call; +- return false; +- } +- +- // Try to monitor the call, if failed, we should delete it right way +- if (!call->Call(*m_stub, m_cq)) { +- // The failure is most likely due to the fact that the completion queue is shutdown +- delete call; +- return false; +- } +- // The call will be enqueued into completion queue, and the callback +- // will be invoked when the response is ready +- std::unique_lock<std::mutex> lock(m_callsMutex); +- m_calls.push_back(call); +- INFO("Start to monitor sandboxer wait call, sandbox id: %s", +- call->GetSandboxId().c_str()); +- return true; +-} +- +-void SandboxerClientMonitor::Start() +-{ +- m_cqThread = std::thread(&SandboxerClientMonitor::AsyncCompleteRpcThread, this); +- m_deferredThread = std::thread(&SandboxerClientMonitor::MonitorThread, this); +-} +- +-void SandboxerClientMonitor::Stop() +-{ +- // At the time of shutdown, the calls objects are in three states: +- // 1. In the completion queue, waiting for the response +- // 2. In the deferred queue, waiting for the dispatch +- // 3. In the future queue, are about to get into completion queue +- // We should cleanup all those objects +- m_cq.Shutdown(); +- m_teardown = true; +- // In case the deferred queue is empty, we should notify the condition variable +- m_deferredCallsCond.notify_one(); +- INFO("Stopping sandboxer wait call monitor, sandboxer: %s", m_sandboxer.c_str()); +- // Wait until all the threads exit +- m_cqThread.join(); +- m_deferredThread.join(); +-} +- +-auto SandboxerClientMonitor::IsAlive() -> bool +-{ +- // GetState will retry the channel connection if channel is not active when the parameter is true. +- // Monitor will keep retrying with wait call, so we dont have to let channel reconnect here, use false instead. +- return m_channel->GetState(false) == GRPC_CHANNEL_READY; +-} +- +-// When the call needs to be retried, we should put it into the deferred queue +-void SandboxerClientMonitor::AddDeferredCall(SandboxerAsyncWaitCall *call) +-{ +- std::unique_lock<std::mutex> lock(m_deferredCallsMutex); +- m_deferredCalls.push_back(call); +- m_deferredCallsCond.notify_one(); +-} +- +-// When the deferred queue and future queue are empty, we should wait for the condition signal +-void SandboxerClientMonitor::WaitForDeferredCall() +-{ +- std::unique_lock<std::mutex> lock(m_deferredCallsMutex); +- // Check m_futures queue here as well just in case m_futures queue is +- if (m_deferredCalls.empty() && m_futures.empty() && !m_teardown) { +- m_deferredCallsCond.wait(lock); +- } +-} +- +-// Retry the call after a random sleep time +-// There are two return values for the future: +-// 1. bool: Indicate the call has been successfully retried and enqueued into completion queue +-// 2. SandboxerAsyncWaitCall *: The call handled by the future +-void SandboxerClientMonitor::InvokeDeferredCall(SandboxerAsyncWaitCall *call) +-{ +- m_futures.push_back(std::async([this, call]() { +- // Random sleep for 50 ~ 200 milliseconds to avoid thundering herd +- std::random_device rd; +- std::mt19937 gen(rd()); +- std::uniform_int_distribution<> dis(DEFERRED_QUEUE_CHECK_INTERVAL / 4, DEFERRED_QUEUE_CHECK_INTERVAL); +- int sleepTime = dis(gen); +- std::this_thread::sleep_for(std::chrono::milliseconds(sleepTime)); +- +- if (this->m_teardown) { +- // If the monitor is already shutdown, the call will be cleaned up +- // by the cleanup function +- return std::pair<bool, SandboxerAsyncWaitCall*>(false, nullptr); +- } +- if (!call->Call(*m_stub, m_cq)) { +- return std::pair<bool, SandboxerAsyncWaitCall*>(false, call); +- } +- return std::pair<bool, SandboxerAsyncWaitCall*>(true, call); +- })); +-} +- +-// Iterate the calls that are timeout and retry them +-void SandboxerClientMonitor::DispatchDeferredCalls() +-{ +- std::unique_lock<std::mutex> lock(m_deferredCallsMutex); +- if (m_teardown) { +- // Stop dispatching deferred calls +- return; +- } +- for (auto it = m_deferredCalls.begin(); it != m_deferredCalls.end();) { +- auto &call = (*it); +- if (call->Timeout()) { +- // If the call is timeout, we should retry +- InvokeDeferredCall(call); +- it = m_deferredCalls.erase(it); +- } else { +- it++; +- } +- } +-} +- +-// We use future to invoke the deferred call, when the future is ready, we should check the result +-void SandboxerClientMonitor::CheckCompletedFutures() +-{ +- for (auto it = m_futures.begin(); it != m_futures.end();) { +- auto &future = (*it); +- if (future.wait_for(std::chrono::seconds(0)) == std::future_status::ready) { +- auto result = future.get(); +- auto enqueued = result.first; +- auto call = result.second; +- if (!enqueued) { +- if (call != nullptr) { +- // Sandbox has been notified due to call failure in call->Call, +- // so no need to notify again +- call->MarkRemove(); +- } +- } else { +- // When the async wait is enqueued into completion queue, +- // and wait does not return error, we have no feadback +- // from the sandboxer, since async wait only return when +- // sandbox exited or some errors happened. +- // Therefore, one way to assume that the wait call suceeded +- // is that check the client status. If the client goes alive, +- // we assume the wait call succeeded. +- if (IsAlive()) { +- // The future is responsible to enqueue the call +- // into completion queue. So there is still a chance +- // that before we check future status here, +- // the call in completion queue has already finished, +- // and the callback has been invoked with OnSandboxPending +- // or OnSandboxExit in the HandleResponse function. +- // In this case, the OnSandboxReady will overwrite the +- // status, but it is ok, because: +- // 1. If OnSandboxPending has been invoked, +- // retry will happen pretty soon, and the +- // callback will be invoked again. +- // 2. If OnSandboxExit has been invoked, the caller +- // should check the sandbox status in the OnSandboxReady +- // callback, and it will find out the sandbox has exited, +- // and not set sandbox to ready status. +- call->SandboxReadyCallback(); +- call->ResetRetryTimes(); +- } +- } +- it = m_futures.erase(it); +- } else { +- it++; +- } +- } +-} +- +-void SandboxerClientMonitor::ClearDeferredCalls() +-{ +- std::unique_lock<std::mutex> lock(m_deferredCallsMutex); +- m_deferredCalls.clear(); +-} +- +-void SandboxerClientMonitor::ClearAllCalls() +-{ +- std::unique_lock<std::mutex> lock(m_callsMutex); +- for (auto &call : m_calls) { +- delete call; +- } +- m_calls.clear(); +-} +- +-void SandboxerClientMonitor::Cleanup() +-{ +- for (auto &future : m_futures) { +- future.wait(); +- } +- m_futures.clear(); +- ClearDeferredCalls(); +- ClearAllCalls(); +-} +- +-/** +- * Thread for handling completion queue. +- */ +-void SandboxerClientMonitor::AsyncCompleteRpcThread() +-{ +- void *got_tag; +- bool ok = false; +- SandboxerAsyncWaitStatus waitStatus; +- INFO("Start thread to monitor wait call completion queue for sandboxer, sandboxer: %s", +- m_sandboxer.c_str()); +- +- pthread_setname_np(pthread_self(), "SandboxerAsyncWaitThread"); +- +- // Next only return false when the completion queue is shutdown and +- // the queue is fully drained +- while (m_cq.Next(&got_tag, &ok)) { +- if (got_tag == nullptr || m_teardown) { +- // The completion queue is shutdown +- // The calls will be cleaned up by the cleanup function +- break; +- } +- auto call = static_cast<SandboxerAsyncWaitCall *>(got_tag); +- waitStatus = call->HandleResponse(); +- if (waitStatus == SANDBOXER_ASYNC_WAIT_STATUS_RETRY) { +- AddDeferredCall(call); +- continue; +- } +- // The reason for not deleting the call here is that +- // while the call is being handled in this thread, +- // the call object could still possibly be referenced in the future +- // that has not been checked yet by the monitor thread. This could happen +- // especially when the completion queue is shutdown. +- call->MarkRemove(); +- // Should notify deferred thread to clean up the call +- m_deferredCallsCond.notify_one(); +- } +- INFO("Completion queue is shutdown, wait call monitor thread exit for sandboxer: %s", +- m_sandboxer.c_str()); +-} +- +-/** +- * Delete the calls that are marked to be removed. +- */ +-void SandboxerClientMonitor::DeleteRemovedCalls() +-{ +- std::unique_lock<std::mutex> lock(m_callsMutex); +- for (auto it = m_calls.begin(); it != m_calls.end();) { +- auto &call = (*it); +- if (call->ToRemove()) { +- delete call; +- it = m_calls.erase(it); +- } else { +- it++; +- } +- } +-} +- +-/** +- * Thread for handling deferred retry calls and cleanup. +- */ +-void SandboxerClientMonitor::MonitorThread() +-{ +- INFO("Start wait call monitoring thread, sandboxer: %s", +- m_sandboxer.c_str()); +- pthread_setname_np(pthread_self(), "SandboxerMonitorThread"); +- while (!m_teardown) { +- // 1. Clean up futures that are ready +- CheckCompletedFutures(); +- // 2. Delete the calls that are marked to be removed +- DeleteRemovedCalls(); +- // 3. If deferred queue is empty, wait for the condition signal +- WaitForDeferredCall(); +- // 4. We have deferred calls in the queue, deferred for 200 ms +- std::this_thread::sleep_for(std::chrono::milliseconds(DEFERRED_QUEUE_CHECK_INTERVAL)); +- // 5. Dispatch the deferred calls +- DispatchDeferredCalls(); +- } +- // Cleanup the queue +- Cleanup(); +- INFO("Start wait call monitoring thread, sandboxer: %s", m_sandboxer.c_str()); +-} +- +-} // namespace sandbox +\ No newline at end of file +diff --git a/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_monitor.h b/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_monitor.h +deleted file mode 100644 +index b5740b44..00000000 +--- a/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_monitor.h ++++ /dev/null +@@ -1,85 +0,0 @@ +-/****************************************************************************** +- * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +- * iSulad licensed under the Mulan PSL v2. +- * You can use this software according to the terms and conditions of the Mulan PSL v2. +- * You may obtain a copy of Mulan PSL v2 at: +- * http://license.coscl.org.cn/MulanPSL2 +- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +- * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +- * PURPOSE. +- * See the Mulan PSL v2 for more details. +- * Author: xuxuepeng +- * Create: 2023-07-28 +- * Description: Sandboxer grpc client monitor +- ******************************************************************************/ +- +-#ifndef DAEMON_SANDBOX_CONTROLLER_SANDBOXER_CLIENT_GRPC_SANDBOXER_MONITOR_H +-#define DAEMON_SANDBOX_CONTROLLER_SANDBOXER_CLIENT_GRPC_SANDBOXER_MONITOR_H +- +-#include <memory> +-#include <string> +-#include <thread> +-#include <future> +- +-#include "grpc_async_wait_call.h" +- +-namespace sandbox { +- +-/** +- * SandboxerClientMonitor is used to monitor the async wait call. +- * It will start two threads, one for monitoring the completion queue, +- * the other for retrying the async wait call. +- * The lifecycle of SandboxerAsyncWaitCall object is managed in the following way: +- * 1. When the async call first called, it will be directly passed into completion queue. +- * 2. If the call status received from completion queue is SANDBOXER_ASYNC_WAIT_STATUS_RETRY, +- * the call will be enqueued to m_deferredCalls, otherwise it will be deleted. +- * 3. When the deferred duration is reached for the call, it will be retried, and passed to +- * completion queue again. +- */ +-class SandboxerClientMonitor { +-public: +- SandboxerClientMonitor(std::shared_ptr<grpc::Channel> channel, const std::string &sandboxer); +- ~SandboxerClientMonitor() = default; +- +- auto Monitor(SandboxerAsyncWaitCall *call) -> bool; +- void Start(); +- void Stop(); +-private: +- auto IsAlive() -> bool; +- void AddDeferredCall(SandboxerAsyncWaitCall *call); +- void WaitForDeferredCall(); +- void InvokeDeferredCall(SandboxerAsyncWaitCall *call); +- void DispatchDeferredCalls(); +- void AsyncCompleteRpcThread(); +- void MonitorThread(); +- void CheckCompletedFutures(); +- void DeleteRemovedCalls(); +- void ClearDeferredCalls(); +- void ClearAllCalls(); +- void Cleanup(); +- +- std::thread m_cqThread; +- std::thread m_deferredThread; +- std::unique_ptr<containerd::services::sandbox::v1::Controller::StubInterface> m_stub; +- std::shared_ptr<grpc::Channel> m_channel; +- std::string m_sandboxer; +- // Completion queue is thread safe, no mutex needed +- grpc::CompletionQueue m_cq; +- // Vector for holding all the calls for monitoring +- std::vector<SandboxerAsyncWaitCall *> m_calls; +- std::mutex m_callsMutex; +- // Use to indicate whether +- bool m_teardown; +- // Vector for holding all the retry calls +- std::vector<SandboxerAsyncWaitCall *> m_deferredCalls; +- std::mutex m_deferredCallsMutex; +- std::condition_variable m_deferredCallsCond; +- +- // Vector for holding all the futures in flight +- // No need to lock, only used in m_deferredThread thread +- std::vector<std::future<std::pair<bool, SandboxerAsyncWaitCall*>>> m_futures; +-}; +- +-} // namespace sandbox +- +-#endif // DAEMON_SANDBOX_CONTROLLER_SANDBOXER_CLIENT_GRPC_SANDBOXER_MONITOR_H +\ No newline at end of file +diff --git a/src/daemon/sandbox/controller/controller_manager.cc b/src/daemon/sandbox/controller_manager.cc +similarity index 98% +rename from src/daemon/sandbox/controller/controller_manager.cc +rename to src/daemon/sandbox/controller_manager.cc +index 91c98d26..947f2d19 100644 +--- a/src/daemon/sandbox/controller/controller_manager.cc ++++ b/src/daemon/sandbox/controller_manager.cc +@@ -57,9 +57,6 @@ bool ControllerManager::Init(Errors &error) + + bool ControllerManager::Cleanup(Errors &error) + { +- for (auto &it : m_controllers) { +- it.second->Destroy(); +- } + return true; + } + +diff --git a/src/daemon/sandbox/controller/controller_manager.h b/src/daemon/sandbox/controller_manager.h +similarity index 100% +rename from src/daemon/sandbox/controller/controller_manager.h +rename to src/daemon/sandbox/controller_manager.h +diff --git a/src/daemon/sandbox/sandbox.cc b/src/daemon/sandbox/sandbox.cc +index 97b77f22..3715e5e0 100644 +--- a/src/daemon/sandbox/sandbox.cc ++++ b/src/daemon/sandbox/sandbox.cc +@@ -1098,11 +1098,6 @@ auto Sandbox::GetNetworkSettingsPath() -> std::string + return m_rootdir + std::string("/") + NETWORK_SETTINGS_JSON; + } + +-auto Sandbox::GetTasksJsonPath() -> std::string +-{ +- return m_rootdir + std::string("/") + SANDBOX_TASKS_JSON; +-} +- + void Sandbox::FillSandboxMetadata(sandbox_metadata* metadata, Errors &error) + { + std::string jsonStr; +@@ -1123,214 +1118,4 @@ void Sandbox::FillSandboxMetadata(sandbox_metadata* metadata, Errors &error) + + metadata->sandbox_config_json = util_strdup_s(jsonStr.c_str()); + } +- +-auto Sandbox::AddTaskById(const char *task_id, sandbox_task *task) -> bool +-{ +- +- std::string taskId = std::string(task_id); +- auto iter = m_tasks.find(taskId); +- +- if (iter != m_tasks.end()) { +- ERROR("Failed to add exits sandbox task %s for sandbox: %s", +- task_id, m_id.c_str()); +- return false; +- } +- m_tasks[taskId] = std::make_shared<SandboxTask>(task); +- return true; +-} +- +-auto Sandbox::ReadSandboxTasksJson() -> sandbox_tasks * +-{ +- const std::string path = GetTasksJsonPath(); +- __isula_auto_free parser_error err = nullptr; +- sandbox_tasks *tasksArray = nullptr; +- +- ReadGuard<RWMutex> lock(m_tasksMutex); +- tasksArray = sandbox_tasks_parse_file(path.c_str(), nullptr, &err); +- if (tasksArray == nullptr) { +- WARN("Failed to read %s tasks json: %s", path.c_str(), err); +- } +- return tasksArray; +-} +- +-auto Sandbox::WriteSandboxTasksJson(std::string &tasks_json) -> bool +-{ +- int nret = 0; +- const std::string path = GetTasksJsonPath(); +- +- WriteGuard<RWMutex> lock(m_tasksMutex); +- nret = util_atomic_write_file(path.c_str(), tasks_json.c_str(), tasks_json.size(), CONFIG_FILE_MODE, false); +- if (nret != 0) { +- SYSERROR("Failed to write file %s", path.c_str()); +- } +- return nret == 0; +-} +- +-auto Sandbox::DeleteSandboxTasksJson() -> bool +-{ +- int get_err = 0; +- const std::string path = GetTasksJsonPath(); +- +- WriteGuard<RWMutex> lock(m_tasksMutex); +- if (util_fileself_exists(path.c_str()) && +- !util_force_remove_file(path.c_str(), &get_err)) { +- errno = get_err; +- SYSERROR("Failed to remove file %s", path.c_str()); +- return false; +- } +- +- return true; +-} +- +-void Sandbox::AddSandboxTasksByArray(sandbox_tasks *tasksArray) +-{ +- size_t i; +- +- WriteGuard<RWMutex> lock(m_tasksMutex); +- for (i = 0; i < tasksArray->tasks_len; i++) { +- if (!AddTaskById(tasksArray->tasks[i]->task_id, tasksArray->tasks[i])) { +- return; +- } +- tasksArray->tasks[i] = nullptr; +- } +- tasksArray->tasks_len = 0; +-} +- +-void Sandbox::LoadSandboxTasks() +-{ +- sandbox_tasks *tasksArray = nullptr; +- +- tasksArray = ReadSandboxTasksJson(); +- if (tasksArray == nullptr) { +- return; +- } +- +- AddSandboxTasksByArray(tasksArray); +- +- free_sandbox_tasks(tasksArray); +-} +- +-auto Sandbox::SaveSandboxTasks() -> bool +-{ +- std::string tasks_json; +- +- if (m_tasks.empty()) { +- return DeleteSandboxTasksJson(); +- } +- +- tasks_json = GetAnySandboxTasks(); +- if (tasks_json.empty()) { +- ERROR("Failed to get sandbox tasks json for sandbox: '%s'", m_id.c_str()); +- return false; +- } +- +- return WriteSandboxTasksJson(tasks_json); +-} +- +-auto Sandbox::AddSandboxTasks(sandbox_task *task) -> bool +-{ +- if (task == nullptr) { +- return true; +- } +- if (task->task_id == nullptr) { +- return false; +- } +- +- WriteGuard<RWMutex> lock(m_tasksMutex); +- +- return AddTaskById(task->task_id, task); +-} +- +-auto Sandbox::GetAnySandboxTasks() -> std::string +-{ +- __isula_auto_free parser_error err = nullptr; +- sandbox_tasks tasksArray = { 0 }; +- size_t i = 0; +- __isula_auto_free char *tasks_json = nullptr; +- +- tasksArray.tasks = (sandbox_task **)util_smart_calloc_s(sizeof(sandbox_task *), m_tasks.size()); +- if (tasksArray.tasks == nullptr) { +- SYSERROR("Out of memory."); +- return std::string(""); +- } +- +- ReadGuard<RWMutex> lock(m_tasksMutex); +- for (auto const& [_, val] : m_tasks) { +- /* +- * We ignore that the processes are modified +- * when we generate tasks json string. +- * Because no matter whether a process is deleted or added, +- * the Update of sandbox api will be called eventually. +- * +- * And we ignore that the task is freed after we do GetTask(). +- * Because the only way to free task is DeleteSandboxTasks() +- * which needs write lock of m_tasksMutex. +- */ +- tasksArray.tasks[i] = val->GetTask(); +- i++; +- } +- tasksArray.tasks_len = m_tasks.size(); +- +- tasks_json = sandbox_tasks_generate_json(&tasksArray, nullptr, &(err)); +- if (tasks_json == nullptr || strlen(tasks_json) == 0) { +- ERROR("Failed to get sandbox tasks json for sandbox: '%s'", m_id.c_str()); +- free(tasksArray.tasks); +- return std::string(""); +- } +- +- free(tasksArray.tasks); +- return std::string(tasks_json); +-} +- +-void Sandbox::DeleteSandboxTasks(const char *containerId) +-{ +- if (containerId == nullptr) { +- return; +- } +- +- std::string taskId = std::string(containerId); +- +- WriteGuard<RWMutex> lock(m_tasksMutex); +- auto iter = m_tasks.find(taskId); +- if (iter == m_tasks.end()) { +- return; +- } +- m_tasks.erase(iter); +-} +- +-auto Sandbox::AddSandboxTasksProcess(const char *containerId, sandbox_process *processes) -> bool +-{ +- if (containerId == nullptr || processes == nullptr) { +- ERROR("Empty args."); +- return false; +- } +- +- std::string taskId = std::string(containerId); +- +- ReadGuard<RWMutex> lock(m_tasksMutex); +- auto iter = m_tasks.find(taskId); +- if (iter == m_tasks.end()) { +- SYSERROR("Failed to find container %s", containerId); +- return false; +- } +- +- return iter->second->AddSandboxTasksProcess(processes); +-} +- +-void Sandbox::DeleteSandboxTasksProcess(const char *containerId, const char *execId) +-{ +- if (containerId == nullptr || execId == nullptr) { +- return; +- } +- +- std::string taskId = std::string(containerId); +- +- ReadGuard<RWMutex> lock(m_tasksMutex); +- auto iter = m_tasks.find(taskId); +- if (iter == m_tasks.end()) { +- return; +- } +- iter->second->DeleteSandboxTasksProcess(execId); +-} +- + } +\ No newline at end of file +diff --git a/src/daemon/sandbox/sandbox.h b/src/daemon/sandbox/sandbox.h +index 437b6113..415406ff 100644 +--- a/src/daemon/sandbox/sandbox.h ++++ b/src/daemon/sandbox/sandbox.h +@@ -141,13 +141,13 @@ public: + void Status(runtime::v1::PodSandboxStatus &status); + + // for sandbox api update +- void LoadSandboxTasks(); +- auto SaveSandboxTasks() -> bool; +- auto AddSandboxTasks(sandbox_task *task) -> bool; +- auto GetAnySandboxTasks() -> std::string; +- void DeleteSandboxTasks(const char *containerId); +- auto AddSandboxTasksProcess(const char *containerId, sandbox_process *processes) -> bool; +- void DeleteSandboxTasksProcess(const char *containerId, const char *execId); ++ virtual void LoadSandboxTasks() = 0; ++ virtual auto SaveSandboxTasks() -> bool = 0; ++ virtual auto AddSandboxTasks(sandbox_task *task) -> bool = 0; ++ virtual auto GetAnySandboxTasks() -> std::string = 0; ++ virtual void DeleteSandboxTasks(const char *containerId) = 0; ++ virtual auto AddSandboxTasksProcess(const char *containerId, sandbox_process *processes) -> bool = 0; ++ virtual void DeleteSandboxTasksProcess(const char *containerId, const char *execId) = 0; + + private: + auto SaveState(Errors &error) -> bool; +@@ -172,7 +172,6 @@ private: + auto GetMetadataJsonPath() -> std::string; + auto GetStatePath() -> std::string; + auto GetNetworkSettingsPath() -> std::string; +- auto GetTasksJsonPath() -> std::string; + + void FillSandboxState(sandbox_state *state); + void FillSandboxMetadata(sandbox_metadata* metadata, Errors &error); +@@ -189,12 +188,6 @@ private: + + void updateSelinuxLabels(std::string &selinuxLabels); + +- auto AddTaskById(const char *task_id, sandbox_task *task) -> bool; +- auto ReadSandboxTasksJson() -> sandbox_tasks *; +- auto WriteSandboxTasksJson(std::string &tasks_json) -> bool; +- auto DeleteSandboxTasksJson() -> bool; +- void AddSandboxTasksByArray(sandbox_tasks *tasksArray); +- + private: + // Since the cri module will operate concurrently on the sandbox instance, + // use m_mutex to ensure the correctness of the sandbox instance +@@ -230,11 +223,6 @@ private: + // vsock ports + std::mutex m_vsockPortsMutex; + std::set<uint32_t> m_vsockPorts; +- +- // use m_tasksMutex to ensure the correctness of the tasks +- RWMutex m_tasksMutex; +- // for sandbox api update, containerId --> tasks +- std::map<std::string, std::shared_ptr<SandboxTask>> m_tasks; + }; + + } // namespace sandbox +diff --git a/src/daemon/sandbox/sandbox_manager.cc b/src/daemon/sandbox/sandbox_manager.cc +index 4159993f..ba003d56 100644 +--- a/src/daemon/sandbox/sandbox_manager.cc ++++ b/src/daemon/sandbox/sandbox_manager.cc +@@ -21,8 +21,13 @@ + + #include <isula_libutils/auto_cleanup.h> + #include <isula_libutils/log.h> ++#include <isula_libutils/sandbox_metadata.h> + + #include "sandbox.h" ++#ifdef ENABLE_SANDBOXER ++#include "sandboxer_sandbox.h" ++#endif ++#include "shim_sandbox.h" + #include "isulad_config.h" + #include "utils_verify.h" + #include "utils_file.h" +@@ -109,7 +114,15 @@ auto SandboxManager::CreateSandbox(const std::string &name, RuntimeInfo &info, s + return nullptr; + } + +- sandbox = std::make_shared<Sandbox>(id, m_rootdir, m_statedir, name, info, netMode, netNsPath, sandboxConfig, image); ++#ifdef ENABLE_SANDBOXER ++ if (info.sandboxer == SHIM_CONTROLLER_NAME) { ++ sandbox = std::make_shared<ShimSandbox>(id, m_rootdir, m_statedir, name, info, netMode, netNsPath, sandboxConfig, image); ++ } else { ++ sandbox = std::make_shared<SandboxerSandbox>(id, m_rootdir, m_statedir, name, info, netMode, netNsPath, sandboxConfig, image); ++ } ++#else ++ sandbox = std::make_shared<ShimSandbox>(id, m_rootdir, m_statedir, name, info, netMode, netNsPath, sandboxConfig, image); ++#endif + if (sandbox == nullptr) { + ERROR("Failed to malloc for sandbox: %s", name.c_str()); + error.Errorf("Failed to malloc for sandbox: %s", name.c_str()); +@@ -442,6 +455,25 @@ bool SandboxManager::ListAllSandboxdir(std::vector<std::string> &allSubdir) + return true; + } + ++#ifdef ENABLE_SANDBOXER ++static auto IsShimSandbox(const std::string &id, const std::string &rootdir) -> bool ++{ ++ __isula_auto_free parser_error err = NULL; ++ const std::string path = rootdir + "/" + id + "/" + SANDBOX_METADATA_JSON; ++ sandbox_metadata *metadata = NULL; ++ ++ metadata = sandbox_metadata_parse_file(path.c_str(), NULL, &err); ++ if (metadata == NULL) { ++ return false; ++ } ++ ++ bool ret = (strcmp(metadata->runtime_info->sandboxer, SHIM_CONTROLLER_NAME.c_str()) == 0); ++ ++ free_sandbox_metadata(metadata); ++ return ret; ++} ++#endif ++ + auto SandboxManager::LoadSandbox(std::string &id) -> std::shared_ptr<Sandbox> + { + std::shared_ptr<Sandbox> sandbox = nullptr; +@@ -451,7 +483,15 @@ auto SandboxManager::LoadSandbox(std::string &id) -> std::shared_ptr<Sandbox> + return nullptr; + } + +- sandbox = std::make_shared<Sandbox>(id, m_rootdir, m_statedir); ++#ifdef ENABLE_SANDBOXER ++ if (IsShimSandbox(id, m_rootdir)) { ++ sandbox = std::make_shared<ShimSandbox>(id, m_rootdir, m_statedir); ++ } else { ++ sandbox = std::make_shared<SandboxerSandbox>(id, m_rootdir, m_statedir); ++ } ++#else ++ sandbox = std::make_shared<ShimSandbox>(id, m_rootdir, m_statedir); ++#endif + if (sandbox == nullptr) { + ERROR("Failed to malloc for sandboxes: %s", id.c_str()); + return nullptr; +diff --git a/src/daemon/sandbox/sandbox_ops.cc b/src/daemon/sandbox/sandbox_ops.cc +index 96e541a4..f50a1033 100644 +--- a/src/daemon/sandbox/sandbox_ops.cc ++++ b/src/daemon/sandbox/sandbox_ops.cc +@@ -16,6 +16,7 @@ + + #include <isula_libutils/auto_cleanup.h> + #include <isula_libutils/log.h> ++#include <isula_libutils/sandbox_sandbox.h> + #include <google/protobuf/util/time_util.h> + + #include "controller_manager.h" +@@ -24,6 +25,7 @@ + #include "namespace.h" + #include "utils.h" + #include "utils_timestamp.h" ++#include "utils_array.h" + + const std::string SANDBOX_EXTENSIONS_TASKS = "extensions.tasks"; + const std::string SANDBOX_TASKS_KEY = "tasks"; +@@ -62,42 +64,30 @@ static int generate_ctrl_rootfs(sandbox_task *task, + return 0; + } + +-static int do_sandbox_prepare(std::shared_ptr<sandbox::Sandbox> &sandbox, containerd::types::Sandbox &apiSandbox) ++static int do_sandbox_update(std::shared_ptr<sandbox::Sandbox> &sandbox, sandbox_sandbox *apiSandbox) + { + Errors err; +- std::vector<std::string> fields; +- +- fields.push_back(SANDBOX_EXTENSIONS_TASKS); ++ size_t fields_len = 1; ++ __isula_auto_string_array_t string_array *fields = nullptr; + +- auto controller = sandbox::ControllerManager::GetInstance()->GetController(sandbox->GetSandboxer()); +- if (nullptr == controller) { +- ERROR("Invalid sandboxer name: %s", sandbox->GetSandboxer().c_str()); ++ fields = util_string_array_new(fields_len); ++ if (fields == nullptr) { ++ ERROR("Out of memory."); + return -1; + } +- +- if (!controller->Prepare(apiSandbox, fields, err)) { +- ERROR("Failed to prepare in container controller prepare: %s", err.GetCMessage()); ++ if (util_append_string_array(fields, SANDBOX_EXTENSIONS_TASKS.c_str())) { ++ ERROR("Out of memory."); + return -1; + } + +- return 0; +-} +- +-static int do_sandbox_purge(std::shared_ptr<sandbox::Sandbox> &sandbox, containerd::types::Sandbox &apiSandbox) +-{ +- Errors err; +- std::vector<std::string> fields; +- +- fields.push_back(SANDBOX_EXTENSIONS_TASKS); +- + auto controller = sandbox::ControllerManager::GetInstance()->GetController(sandbox->GetSandboxer()); + if (nullptr == controller) { + ERROR("Invalid sandboxer name: %s", sandbox->GetSandboxer().c_str()); + return -1; + } + +- if (!controller->Purge(apiSandbox, fields, err)) { +- ERROR("Failed to purge: %s", err.GetCMessage()); ++ if (!controller->Update(apiSandbox, fields, err)) { ++ ERROR("Failed to update in container controller update: %s", err.GetCMessage()); + return -1; + } + +@@ -161,40 +151,112 @@ static std::shared_ptr<sandbox::Sandbox> get_prepare_sandbox(const container_con + return sandbox; + } + +-static int init_prepare_api_sandbox(std::shared_ptr<sandbox::Sandbox> sandbox, const char *containerId, +- containerd::types::Sandbox &apiSandbox) ++static sandbox_sandbox_runtime *init_sandbox_runtime(std::shared_ptr<sandbox::Sandbox> sandbox) ++{ ++ sandbox_sandbox_runtime *runtime = nullptr; ++ ++ auto runtime_wrapper = makeUniquePtrCStructWrapper<sandbox_sandbox_runtime>(free_sandbox_sandbox_runtime); ++ if (runtime_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return nullptr; ++ } ++ runtime = runtime_wrapper->get(); ++ runtime->name = util_strdup_s(sandbox->GetRuntime().c_str()); ++ // Just ignore options for now ++ ++ return runtime_wrapper->move(); ++} ++ ++static json_map_string_string *init_sandbox_labels(std::shared_ptr<sandbox::Sandbox> sandbox) + { +- google::protobuf::Map<std::string, std::string> *labels = apiSandbox.mutable_labels(); +- google::protobuf::Map<std::string, google::protobuf::Any> *extensions = apiSandbox.mutable_extensions(); +- google::protobuf::Any any; +- auto created_at = new (std::nothrow) google::protobuf::Timestamp; +- auto updated_at = new (std::nothrow) google::protobuf::Timestamp; ++ json_map_string_string *labels = nullptr; ++ ++ auto labels_wrapper = makeUniquePtrCStructWrapper<json_map_string_string>(free_json_map_string_string); ++ if (labels_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return nullptr; ++ } ++ labels = labels_wrapper->get(); ++ if (append_json_map_string_string(labels, "name", sandbox->GetName().c_str()) != 0) { ++ ERROR("Out of memory"); ++ return nullptr; ++ } + +- apiSandbox.set_sandbox_id(sandbox->GetId()); +- apiSandbox.mutable_runtime()->set_name(sandbox->GetRuntime()); +- // TODO how get options +- // apiSandbox.mutable_runtime()->set_options(sandbox->GetRuntime()); +- // Just ignore spec +- (*labels)[std::string("name")] = sandbox->GetName(); +- +- *created_at = google::protobuf::util::TimeUtil::NanosecondsToTimestamp( +- sandbox->GetCreatedAt()); +- apiSandbox.set_allocated_created_at(created_at); +- *updated_at = google::protobuf::util::TimeUtil::NanosecondsToTimestamp(util_get_now_time_nanos()); +- apiSandbox.set_allocated_updated_at(updated_at); +- +- auto any_type_url = any.mutable_type_url(); +- *any_type_url = SANDBOX_TASKS_TYPEURL; +- auto any_value = any.mutable_value(); +- *any_value = sandbox->GetAnySandboxTasks(); +- if ((*any_value).empty()) { ++ return labels_wrapper->move(); ++} ++ ++static defs_map_string_object_any *init_sandbox_extensions(std::shared_ptr<sandbox::Sandbox> sandbox) ++{ ++ defs_map_string_object_any *extensions = nullptr; ++ size_t len = 1; ++ std::string task_json; ++ ++ auto extensions_wrapper = makeUniquePtrCStructWrapper<defs_map_string_object_any>(free_defs_map_string_object_any); ++ if (extensions_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return nullptr; ++ } ++ extensions = extensions_wrapper->get(); ++ extensions->keys = (char **)util_smart_calloc_s(sizeof(char *), len); ++ if (extensions->keys == nullptr) { ++ ERROR("Out of memory."); ++ return nullptr; ++ } ++ extensions->len = len; ++ extensions->values = (defs_map_string_object_any_element **) ++ util_smart_calloc_s(sizeof(defs_map_string_object_any_element *), len); ++ if (extensions->values == nullptr) { ++ ERROR("Out of memory."); ++ return nullptr; ++ } ++ extensions->values[0] = (defs_map_string_object_any_element *) ++ util_common_calloc_s(sizeof(defs_map_string_object_any_element)); ++ if (extensions->values[0] == nullptr) { ++ ERROR("Out of memory."); ++ return nullptr; ++ } ++ extensions->values[0]->element = (defs_any *)util_common_calloc_s(sizeof(defs_any)); ++ if (extensions->values[0]->element == nullptr) { ++ ERROR("Out of memory."); ++ return nullptr; ++ } ++ ++ extensions->keys[0] = util_strdup_s(SANDBOX_TASKS_KEY.c_str()); ++ task_json = sandbox->GetAnySandboxTasks(); ++ if (task_json.empty()) { + ERROR("Failed to get any sandbox tasks"); +- return -1; ++ return nullptr; + } +- DEBUG("Get any sandbox tasks %s", (*any_value).c_str()); +- (*extensions)[SANDBOX_TASKS_KEY] = any; ++ DEBUG("Get any sandbox tasks %s", task_json.c_str()); ++ extensions->values[0]->element->type_url = util_strdup_s(SANDBOX_TASKS_TYPEURL.c_str()); ++ extensions->values[0]->element->value = reinterpret_cast<uint8_t *>(util_strdup_s(task_json.c_str())); ++ extensions->values[0]->element->value_len = strlen(task_json.c_str()); ++ ++ return extensions_wrapper->move(); ++} + +- apiSandbox.set_sandboxer(sandbox->GetSandboxer()); ++static int init_api_sandbox(std::shared_ptr<sandbox::Sandbox> sandbox, sandbox_sandbox *apiSandbox) ++{ ++ apiSandbox->sandbox_id = util_strdup_s(sandbox->GetId().c_str()); ++ apiSandbox->runtime = init_sandbox_runtime(sandbox); ++ if (apiSandbox->runtime == nullptr) { ++ ERROR("Failed to init sandbox runtime"); ++ return -1; ++ } ++ // Just ignore spec ++ apiSandbox->labels = init_sandbox_labels(sandbox); ++ if (apiSandbox->labels == nullptr) { ++ ERROR("Failed to init sandbox runtime"); ++ return -1; ++ } ++ apiSandbox->created_at = sandbox->GetCreatedAt(); ++ apiSandbox->updated_at = util_get_now_time_nanos(); ++ apiSandbox->extensions = init_sandbox_extensions(sandbox); ++ if (apiSandbox->extensions == nullptr) { ++ ERROR("Failed to init sandbox runtime"); ++ return -1; ++ } ++ apiSandbox->sandboxer = util_strdup_s(sandbox->GetSandboxer().c_str()); + + return 0; + } +@@ -204,7 +266,7 @@ int sandbox_prepare_container(const container_config_v2_common_config *config, + const char * console_fifos[], bool tty) + { + sandbox_task *task = nullptr; +- containerd::types::Sandbox apiSandbox; ++ sandbox_sandbox *apiSandbox = nullptr; + int ret = -1; + + INFO("Prepare container for sandbox"); +@@ -220,20 +282,28 @@ int sandbox_prepare_container(const container_config_v2_common_config *config, + return -1; + } + +- task = (sandbox_task *)util_common_calloc_s(sizeof(sandbox_task)); +- if (task == nullptr) { +- ERROR("Out of memory."); ++ auto apiSandbox_wrapper = makeUniquePtrCStructWrapper<sandbox_sandbox>(free_sandbox_sandbox); ++ if (apiSandbox_wrapper == nullptr) { ++ ERROR("Out of memory"); + return -1; + } ++ apiSandbox = apiSandbox_wrapper->get(); ++ auto task_wrapper = makeUniquePtrCStructWrapper<sandbox_task>(free_sandbox_task); ++ if (task_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ task = task_wrapper->get(); ++ + task->task_id = util_strdup_s(config->id); + task->spec = clone_oci_runtime_spec(oci_spec); + if (task->spec == nullptr) { + ERROR("Out of memory."); +- goto free_out; ++ return -1; + } + if (generate_ctrl_rootfs(task, config) != 0) { + ERROR("Invalid rootfs"); +- goto free_out; ++ return -1; + } + task->stdin = util_strdup_s((nullptr == console_fifos[0]) ? "" : console_fifos[0]); + task->stdout = util_strdup_s((nullptr == console_fifos[1]) ? "" : console_fifos[1]); +@@ -241,15 +311,15 @@ int sandbox_prepare_container(const container_config_v2_common_config *config, + + if (!sandbox->AddSandboxTasks(task)) { + ERROR("Failed to add sandbox %s task.", config->id); +- goto free_out; ++ return -1; + } +- task = nullptr; +- ret = init_prepare_api_sandbox(sandbox, config->id, apiSandbox); ++ task = task_wrapper->move(); ++ ret = init_api_sandbox(sandbox, apiSandbox); + if (ret != 0) { + ERROR("Failed to init %s api sandbox.", config->id); + goto del_out; + } +- ret = do_sandbox_prepare(sandbox, apiSandbox); ++ ret = do_sandbox_update(sandbox, apiSandbox); + + del_out: + if (ret != 0) { +@@ -259,8 +329,7 @@ del_out: + ERROR("Failed to Save %s sandbox tasks.", config->id); + ret = -1; + } +-free_out: +- free_sandbox_task(task); ++ + return ret; + } + +@@ -269,7 +338,7 @@ int sandbox_prepare_exec(const container_config_v2_common_config *config, + const char * console_fifos[], bool tty) + { + sandbox_process *process = nullptr; +- containerd::types::Sandbox apiSandbox; ++ sandbox_sandbox *apiSandbox = nullptr; + int ret = -1; + + INFO("Prepare exec for container in sandbox"); +@@ -285,16 +354,24 @@ int sandbox_prepare_exec(const container_config_v2_common_config *config, + return -1; + } + +- process = (sandbox_process *)util_common_calloc_s(sizeof(sandbox_process)); +- if (process == nullptr) { +- ERROR("Out of memory."); ++ auto apiSandbox_wrapper = makeUniquePtrCStructWrapper<sandbox_sandbox>(free_sandbox_sandbox); ++ if (apiSandbox_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ apiSandbox = apiSandbox_wrapper->get(); ++ auto process_wrapper = makeUniquePtrCStructWrapper<sandbox_process>(free_sandbox_process); ++ if (process_wrapper == nullptr) { ++ ERROR("Out of memory"); + return -1; + } ++ process = process_wrapper->get(); ++ + process->exec_id = util_strdup_s(exec_id); + process->spec = clone_defs_process(process_spec); + if (process->spec == nullptr) { + ERROR("Out of memory."); +- goto free_out; ++ return -1; + } + process->stdin = util_strdup_s((nullptr == console_fifos[0]) ? "" : console_fifos[0]); + process->stdout = util_strdup_s((nullptr == console_fifos[1]) ? "" : console_fifos[1]); +@@ -302,15 +379,15 @@ int sandbox_prepare_exec(const container_config_v2_common_config *config, + + if (!sandbox->AddSandboxTasksProcess(config->id, process)) { + ERROR("Failed to add sandbox %s process.", config->id); +- goto free_out; ++ return -1; + } +- process = nullptr; +- ret = init_prepare_api_sandbox(sandbox, config->id, apiSandbox); ++ process = process_wrapper->move(); ++ ret = init_api_sandbox(sandbox, apiSandbox); + if (ret != 0) { + ERROR("Failed to init %s api sandbox.", config->id); + goto del_out; + } +- ret = do_sandbox_prepare(sandbox, apiSandbox); ++ ret = do_sandbox_update(sandbox, apiSandbox); + + del_out: + if (ret != 0) { +@@ -320,14 +397,13 @@ del_out: + ERROR("Failed to Save %s sandbox tasks.", config->id); + ret = -1; + } +-free_out: +- free_sandbox_process(process); ++ + return ret; + } + + int sandbox_purge_container(const container_config_v2_common_config *config) + { +- containerd::types::Sandbox apiSandbox; ++ sandbox_sandbox *apiSandbox = nullptr; + + INFO("Purge container for sandbox"); + +@@ -337,22 +413,29 @@ int sandbox_purge_container(const container_config_v2_common_config *config) + return -1; + } + ++ auto apiSandbox_wrapper = makeUniquePtrCStructWrapper<sandbox_sandbox>(free_sandbox_sandbox); ++ if (apiSandbox_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ apiSandbox = apiSandbox_wrapper->get(); ++ + sandbox->DeleteSandboxTasks(config->id); + if (!sandbox->SaveSandboxTasks()) { + ERROR("Failed to Save %s sandbox tasks.", config->id); + return -1; + } + +- if (init_prepare_api_sandbox(sandbox, config->id, apiSandbox) != 0) { ++ if (init_api_sandbox(sandbox, apiSandbox) != 0) { + ERROR("Failed to init %s api sandbox.", config->id); + return -1; + } +- return do_sandbox_purge(sandbox, apiSandbox); ++ return do_sandbox_update(sandbox, apiSandbox); + } + + int sandbox_purge_exec(const container_config_v2_common_config *config, const char *exec_id) + { +- containerd::types::Sandbox apiSandbox; ++ sandbox_sandbox *apiSandbox = nullptr; + + INFO("Purge exec for container in sandbox"); + +@@ -362,18 +445,25 @@ int sandbox_purge_exec(const container_config_v2_common_config *config, const ch + return -1; + } + ++ auto apiSandbox_wrapper = makeUniquePtrCStructWrapper<sandbox_sandbox>(free_sandbox_sandbox); ++ if (apiSandbox_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ apiSandbox = apiSandbox_wrapper->get(); ++ + sandbox->DeleteSandboxTasksProcess(config->id, exec_id); + if (!sandbox->SaveSandboxTasks()) { + ERROR("Failed to Save %s sandbox tasks.", config->id); + return -1; + } + +- if (init_prepare_api_sandbox(sandbox, config->id, apiSandbox) != 0) { ++ if (init_api_sandbox(sandbox, apiSandbox) != 0) { + ERROR("Failed to init %s api sandbox.", exec_id); + return -1; + } + +- return do_sandbox_purge(sandbox, apiSandbox); ++ return do_sandbox_update(sandbox, apiSandbox); + } + + int sandbox_on_sandbox_exit(const char *sandbox_id, int exit_code) +diff --git a/src/daemon/sandbox/sandbox_task.cc b/src/daemon/sandbox/sandbox_task.cc +index b1efc340..4533f381 100644 +--- a/src/daemon/sandbox/sandbox_task.cc ++++ b/src/daemon/sandbox/sandbox_task.cc +@@ -91,8 +91,11 @@ void SandboxTask::DeleteSandboxTasksProcess(const char *execId) + return; + } + free_sandbox_process(m_task->processes[idx]); +- (void)memcpy((void **)&m_task->processes[idx], (void **)&m_task->processes[idx + 1], +- (m_task->processes_len - idx - 1) * sizeof(void *)); ++ m_task->processes[idx] = nullptr; ++ if (idx != (int)m_task->processes_len - 1) { ++ (void)memcpy((void **)&m_task->processes[idx], (void **)&m_task->processes[idx + 1], ++ (m_task->processes_len - idx - 1) * sizeof(void *)); ++ } + m_task->processes_len--; + } + +diff --git a/src/daemon/sandbox/sandboxer/CMakeLists.txt b/src/daemon/sandbox/sandboxer/CMakeLists.txt +new file mode 100755 +index 00000000..2b0607e0 +--- /dev/null ++++ b/src/daemon/sandbox/sandboxer/CMakeLists.txt +@@ -0,0 +1,15 @@ ++# get current directory sources files ++aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} sandboxer_sandbox_srcs) ++ ++add_subdirectory(controller) ++ ++set(SANDBOXER_SANDBOX_SRCS ++ ${sandboxer_sandbox_srcs} ++ ${CONTROLLER_SANDBOXER_SRCS} ++ PARENT_SCOPE ++ ) ++set(SANDBOXER_SANDBOX_INCS ++ ${CMAKE_CURRENT_SOURCE_DIR} ++ ${CONTROLLER_SANDBOXER_INCS} ++ PARENT_SCOPE ++ ) +diff --git a/src/daemon/sandbox/controller/sandboxer/CMakeLists.txt b/src/daemon/sandbox/sandboxer/controller/CMakeLists.txt +similarity index 100% +rename from src/daemon/sandbox/controller/sandboxer/CMakeLists.txt +rename to src/daemon/sandbox/sandboxer/controller/CMakeLists.txt +diff --git a/src/daemon/sandbox/sandboxer/controller/client/CMakeLists.txt b/src/daemon/sandbox/sandboxer/controller/client/CMakeLists.txt +new file mode 100755 +index 00000000..6ecd350b +--- /dev/null ++++ b/src/daemon/sandbox/sandboxer/controller/client/CMakeLists.txt +@@ -0,0 +1,12 @@ ++# get current directory sources files ++aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} sandbox_controller_sandboxer_client_srcs) ++ ++set(CONTROLLER_SANDBOXER_CLIENT_SRCS ++ ${sandbox_controller_sandboxer_client_srcs} ++ PARENT_SCOPE ++ ) ++ ++set(CONTROLLER_SANDBOXER_CLIENT_INCS ++ ${CMAKE_CURRENT_SOURCE_DIR} ++ PARENT_SCOPE ++ ) +diff --git a/src/daemon/sandbox/sandboxer/controller/client/grpc_sandboxer_client.cc b/src/daemon/sandbox/sandboxer/controller/client/grpc_sandboxer_client.cc +new file mode 100644 +index 00000000..d7ff76d7 +--- /dev/null ++++ b/src/daemon/sandbox/sandboxer/controller/client/grpc_sandboxer_client.cc +@@ -0,0 +1,463 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: xuxuepeng ++ * Create: 2023-07-10 ++ * Description: Sandboxer grpc client ++ ******************************************************************************/ ++ ++#include "grpc_sandboxer_client.h" ++ ++#include <grpc++/grpc++.h> ++#include <iostream> ++#include <memory> ++#include <string> ++#include <random> ++ ++#include <isula_libutils/log.h> ++ ++#include "utils.h" ++#include "cstruct_wrapper.h" ++#include "transform.h" ++#include "sandbox_manager.h" ++ ++namespace sandbox { ++ ++SandboxerClient::SandboxerClient(const std::string &sandboxer, const std::string &address): ++ m_sandboxer(sandboxer), m_address(address) ++{ ++ std::string unixPrefix(UNIX_SOCKET_PREFIX); ++ ++ // Only support unix domain socket ++ if (m_address.compare(0, unixPrefix.length(), unixPrefix) == 0) { ++ m_address = m_address.substr(unixPrefix.length()); ++ } ++ m_controller_handle = sandbox_api_build_controller(m_sandboxer.c_str(), m_address.c_str()); ++ if (m_controller_handle == nullptr) { ++ ERROR("Failed to create sandboxer client, sandboxer: %s, address: %s", m_sandboxer.c_str(), m_address.c_str()); ++ } ++} ++ ++auto SandboxerClient::InitMountInfo(sandbox_mount &m, const ControllerMountInfo &mountInfo) -> int ++{ ++ m.type = util_strdup_s(mountInfo.type.c_str()); ++ m.source = util_strdup_s(mountInfo.source.c_str()); ++ m.target = util_strdup_s(mountInfo.destination.c_str()); ++ ++ size_t mount_options_len = mountInfo.options.size(); ++ int ret = 0; ++ for (size_t j = 0; j < mount_options_len; j++) { ++ ret = util_array_append(&(m.options), mountInfo.options[j].c_str()); ++ if (ret != 0) { ++ ERROR("append mount options to array failed"); ++ return -1; ++ } ++ m.options_len++; ++ } ++ return 0; ++} ++ ++auto SandboxerClient::InitCreateRequest(sandbox_create_request &request, ++ const std::string &sandboxId, ++ const ControllerCreateParams ¶ms) -> bool ++{ ++ if (params.config == nullptr) { ++ ERROR("Sandboxer controller create request failed, config is null"); ++ return false; ++ } ++ request.sandboxer = util_strdup_s(m_sandboxer.c_str()); ++ request.sandbox_id = util_strdup_s(sandboxId.c_str()); ++ std::string encoded; ++ if (!params.config->SerializeToString(&encoded)) { ++ ERROR("Failed to serialize config"); ++ return false; ++ } ++ request.options = (defs_any *)util_common_calloc_s(sizeof(defs_any)); ++ if (request.options == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ request.options->value = (uint8_t *)util_common_calloc_s(encoded.size()); ++ if (request.options == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ (void)memcpy(request.options->value, encoded.c_str(), encoded.size()); ++ request.options->value_len = encoded.size(); ++ request.netns_path = util_strdup_s(params.netNSPath.c_str()); ++ sandbox_mount **mounts = NULL; ++ size_t mounts_len = params.mounts.size(); ++ if (mounts_len > 0) { ++ request.rootfs = (sandbox_mount**)util_common_calloc_s(mounts_len * sizeof(sandbox_mount *)); ++ if (request.rootfs == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ for (const auto &entry : params.mounts) { ++ if (entry != nullptr) { ++ sandbox_mount *m = (sandbox_mount *)util_common_calloc_s(sizeof(sandbox_mount)); ++ if (m == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ if (InitMountInfo(*m, *entry) != 0) { ++ ERROR("Failed to init mount info"); ++ free(m); ++ return false; ++ } ++ mounts[request.rootfs_len++] = m; ++ m = NULL; ++ } ++ } ++ } ++ return true; ++} ++ ++auto SandboxerClient::Create(const std::string &sandboxId, const ControllerCreateParams ¶ms, Errors &error) -> bool ++{ ++ sandbox_create_request *request { nullptr }; ++ sandbox_create_response *response { nullptr }; ++ ++ auto create_request_wrapper = makeUniquePtrCStructWrapper<sandbox_create_request>(free_sandbox_create_request); ++ if (create_request_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ error.SetError("Out of memory"); ++ return false; ++ } ++ request = create_request_wrapper->get(); ++ ++ auto create_response_wrapper = makeUniquePtrCStructWrapper<sandbox_create_response>(free_sandbox_create_response); ++ if (create_response_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ error.SetError("Out of memory"); ++ return false; ++ } ++ response = create_response_wrapper->get(); ++ ++ if (!InitCreateRequest(*request, sandboxId, params)) { ++ ERROR("Failed to init create request for sandboxer create request"); ++ error.SetError("Failed to init create request for sandboxer create request"); ++ return false; ++ } ++ int ret = sandbox_api_create(m_controller_handle, request, response); ++ if (ret != 0) { ++ ERROR("Failed to create sandbox"); ++ error.SetError("Failed to create sandbox"); ++ return false; ++ } ++ ++ return true; ++} ++ ++void SandboxerClient::StartResponseToSandboxInfo(sandbox_start_response &response, ++ ControllerSandboxInfo &sandboxInfo) ++{ ++ sandboxInfo.id = std::string(response.sandbox_id); ++ sandboxInfo.pid = response.pid; ++ sandboxInfo.createdAt = response.created_at; ++ sandboxInfo.taskAddress = std::string(response.address); ++ sandboxInfo.version = response.version; ++ Transform::JsonMapToProtobufMapForString(response.labels, sandboxInfo.labels); ++} ++ ++auto SandboxerClient::Start(const std::string &sandboxId, ControllerSandboxInfo &sandboxInfo, Errors &error) -> bool ++{ ++ sandbox_start_request *request { nullptr }; ++ sandbox_start_response *response { nullptr }; ++ ++ auto start_request_wrapper = makeUniquePtrCStructWrapper<sandbox_start_request>(free_sandbox_start_request); ++ if (start_request_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ error.SetError("Out of memory"); ++ return false; ++ } ++ request = start_request_wrapper->get(); ++ ++ auto start_response_wrapper = makeUniquePtrCStructWrapper<sandbox_start_response>(free_sandbox_start_response); ++ if (start_response_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ error.SetError("Out of memory"); ++ return false; ++ } ++ response = start_response_wrapper->get(); ++ ++ request->sandboxer = util_strdup_s(m_sandboxer.c_str()); ++ request->sandbox_id = util_strdup_s(sandboxId.c_str()); ++ ++ int ret = sandbox_api_start(m_controller_handle, request, response); ++ if (ret != 0) { ++ ERROR("Failed to start sandbox"); ++ error.SetError("Failed to start sandbox"); ++ return false; ++ } ++ ++ StartResponseToSandboxInfo(*response, sandboxInfo); ++ ++ return true; ++} ++ ++void SandboxerClient::InitUpdateRequest(sandbox_update_request &request, ++ sandbox_sandbox *apiSandbox, string_array *fields) ++{ ++ request.sandbox_id = util_strdup_s(apiSandbox->sandbox_id); ++ request.sandboxer = util_strdup_s(apiSandbox->sandboxer); ++ request.sandbox = apiSandbox; ++ request.fields = fields->items; ++ request.fields_len = fields->len; ++} ++ ++auto SandboxerClient::Update(sandbox_sandbox *apiSandbox, string_array *fields, Errors &error) -> bool ++{ ++ sandbox_update_request *request { nullptr }; ++ ++ auto update_request_wrapper = makeUniquePtrCStructWrapper<sandbox_update_request>(free_sandbox_update_request); ++ if (update_request_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ error.SetError("Out of memory"); ++ return false; ++ } ++ request = update_request_wrapper->get(); ++ ++ InitUpdateRequest(*request, apiSandbox, fields); ++ int ret = sandbox_api_update(m_controller_handle, request); ++ request->sandbox = nullptr; ++ request->fields = nullptr; ++ request->fields_len = 0; ++ if (ret != 0) { ++ ERROR("Failed to update sandbox"); ++ error.SetError("Failed to update sandbox"); ++ return false; ++ } ++ ++ return true; ++} ++ ++void SandboxerClient::PlatformResponseToPlatformInfo(sandbox_platform_response &response, ++ ControllerPlatformInfo &platformInfo) ++{ ++ platformInfo.os = std::string(response.os); ++ platformInfo.arch = std::string(response.architecture); ++ platformInfo.variant = std::string(response.variant); ++} ++ ++auto SandboxerClient::Platform(const std::string &sandboxId, ControllerPlatformInfo &platformInfo, ++ Errors &error) -> bool ++{ ++ sandbox_platform_request *request { nullptr }; ++ sandbox_platform_response *response { nullptr }; ++ ++ auto platform_request_wrapper = makeUniquePtrCStructWrapper<sandbox_platform_request>(free_sandbox_platform_request); ++ if (platform_request_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ error.SetError("Out of memory"); ++ return false; ++ } ++ request = platform_request_wrapper->get(); ++ ++ auto platform_response_wrapper = makeUniquePtrCStructWrapper<sandbox_platform_response>(free_sandbox_platform_response); ++ if (platform_response_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ error.SetError("Out of memory"); ++ return false; ++ } ++ response = platform_response_wrapper->get(); ++ ++ request->sandboxer = util_strdup_s(m_sandboxer.c_str()); ++ request->sandbox_id = util_strdup_s(sandboxId.c_str()); ++ int ret = sandbox_api_platform(m_controller_handle, request, response); ++ if (ret != 0) { ++ ERROR("Failed to platform sandbox"); ++ error.SetError("Failed to platform sandbox"); ++ return false; ++ } ++ ++ PlatformResponseToPlatformInfo(*response, platformInfo); ++ ++ return true; ++} ++ ++auto SandboxerClient::Stop(const std::string &sandboxId, uint32_t timeoutSecs, Errors &error) -> bool ++{ ++ sandbox_stop_request *request { nullptr }; ++ ++ auto stop_request_wrapper = makeUniquePtrCStructWrapper<sandbox_stop_request>(free_sandbox_stop_request); ++ if (stop_request_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ error.SetError("Out of memory"); ++ return false; ++ } ++ request = stop_request_wrapper->get(); ++ ++ request->sandboxer = util_strdup_s(m_sandboxer.c_str()); ++ request->sandbox_id = util_strdup_s(sandboxId.c_str()); ++ request->timeout_secs = timeoutSecs; ++ int ret = sandbox_api_stop(m_controller_handle, request); ++ if (ret != 0) { ++ ERROR("Failed to stop sandbox"); ++ error.SetError("Failed to stop sandbox"); ++ return false; ++ } ++ ++ return true; ++} ++ ++void SandboxerClient::StatusResponseToSandboxStatus(sandbox_status_response ++ &response, ++ ControllerSandboxStatus &sandboxStatus) ++{ ++ sandboxStatus.id = std::string(response.sandbox_id); ++ sandboxStatus.pid = response.pid; ++ sandboxStatus.state = std::string(response.state); ++ Transform::JsonMapToProtobufMapForString(response.info, sandboxStatus.info); ++ sandboxStatus.createdAt = response.created_at; ++ sandboxStatus.exitedAt = response.exited_at; ++ if (response.extra != nullptr && response.extra->value != nullptr) { ++ sandboxStatus.extra = std::string(response.extra->value, ++ response.extra->value + response.extra->value_len); ++ } else { ++ sandboxStatus.extra = std::string(""); ++ } ++ sandboxStatus.taskAddress = std::string(response.address); ++ sandboxStatus.version = response.version; ++} ++ ++auto SandboxerClient::Status(const std::string &sandboxId, bool verbose, ControllerSandboxStatus &sandboxStatus, ++ Errors &error) -> bool ++{ ++ sandbox_status_request *request { nullptr }; ++ sandbox_status_response *response { nullptr }; ++ ++ auto status_request_wrapper = makeUniquePtrCStructWrapper<sandbox_status_request>(free_sandbox_status_request); ++ if (status_request_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ error.SetError("Out of memory"); ++ return false; ++ } ++ request = status_request_wrapper->get(); ++ ++ auto status_response_wrapper = makeUniquePtrCStructWrapper<sandbox_status_response>(free_sandbox_status_response); ++ if (status_response_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ error.SetError("Out of memory"); ++ return false; ++ } ++ response = status_response_wrapper->get(); ++ ++ request->sandboxer = util_strdup_s(m_sandboxer.c_str()); ++ request->sandbox_id = util_strdup_s(sandboxId.c_str()); ++ request->verbose = verbose; ++ int ret = sandbox_api_status(m_controller_handle, request, response); ++ if (ret != 0) { ++ ERROR("Failed to status sandbox"); ++ error.SetError("Failed to status sandbox"); ++ return false; ++ } ++ ++ StatusResponseToSandboxStatus(*response, sandboxStatus); ++ ++ return true; ++} ++ ++auto SandboxerClient::Shutdown(const std::string &sandboxId, Errors &error) -> bool ++{ ++ sandbox_shutdown_request *request { nullptr }; ++ ++ auto shutdown_request_wrapper = makeUniquePtrCStructWrapper<sandbox_shutdown_request>(free_sandbox_shutdown_request); ++ if (shutdown_request_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ error.SetError("Out of memory"); ++ return false; ++ } ++ request = shutdown_request_wrapper->get(); ++ ++ request->sandboxer = util_strdup_s(m_sandboxer.c_str()); ++ request->sandbox_id = util_strdup_s(sandboxId.c_str()); ++ int ret = sandbox_api_shutdown(m_controller_handle, request); ++ if (ret != 0) { ++ ERROR("Failed to shutdown sandbox"); ++ error.SetError("Failed to shutdown sandbox"); ++ return false; ++ } ++ ++ return true; ++} ++ ++static int sandbox_api_ready(const char *sandbox_id) ++{ ++ std::string sandboxId = std::string(sandbox_id); ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(sandboxId); ++ if (nullptr == sandbox) { ++ ERROR("Sandbox not found"); ++ return -1; ++ } ++ ++ sandbox->OnSandboxReady(); ++ return 0; ++} ++ ++static int sandbox_api_pending(const char *sandbox_id) ++{ ++ std::string sandboxId = std::string(sandbox_id); ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(sandboxId); ++ if (nullptr == sandbox) { ++ ERROR("Sandbox not found"); ++ return -1; ++ } ++ ++ sandbox->OnSandboxPending(); ++ return 0; ++} ++ ++static int sandbox_api_exit(const char *sandbox_id, const sandbox_wait_response *response) ++{ ++ ControllerExitInfo exitInfo; ++ std::string sandboxId = std::string(sandbox_id); ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(sandboxId); ++ if (nullptr == sandbox) { ++ ERROR("Sandbox not found"); ++ return -1; ++ } ++ ++ exitInfo.exitStatus = response->exit_status; ++ exitInfo.exitedAt = response->exited_at; ++ sandbox->OnSandboxExit(exitInfo); ++ return 0; ++} ++ ++auto SandboxerClient::Wait(std::shared_ptr<SandboxStatusCallback> cb, const std::string &sandboxId, ++ Errors &error) -> bool ++{ ++ sandbox_wait_request *request { nullptr }; ++ sandbox_api_wait_callback callback; ++ ++ auto wait_request_wrapper = makeUniquePtrCStructWrapper<sandbox_wait_request>(free_sandbox_wait_request); ++ if (wait_request_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ error.SetError("Out of memory"); ++ return false; ++ } ++ request = wait_request_wrapper->get(); ++ ++ request->sandboxer = util_strdup_s(m_sandboxer.c_str()); ++ request->sandbox_id = util_strdup_s(sandboxId.c_str()); ++ callback.ready = sandbox_api_ready; ++ callback.pending = sandbox_api_pending; ++ callback.exit = sandbox_api_exit; ++ int ret = sandbox_api_wait(m_controller_handle, request, callback); ++ if (ret != 0) { ++ ERROR("Failed to wait sandbox"); ++ error.SetError("Failed to wait sandbox"); ++ return false; ++ } ++ ++ return true; ++} ++ ++} // namespace +diff --git a/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_client.h b/src/daemon/sandbox/sandboxer/controller/client/grpc_sandboxer_client.h +similarity index 54% +rename from src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_client.h +rename to src/daemon/sandbox/sandboxer/controller/client/grpc_sandboxer_client.h +index eeb5d7f2..d712be04 100644 +--- a/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_client.h ++++ b/src/daemon/sandbox/sandboxer/controller/client/grpc_sandboxer_client.h +@@ -20,15 +20,10 @@ + #include <memory> + #include <future> + +-#include "sandbox/types/mount.pb.h" +-#include "sandbox.pb.h" +-#include "sandbox.grpc.pb.h" +-#include "controller.h" +-#include "grpc_async_wait_call.h" +-#include "grpc_sandboxer_monitor.h" ++#include <isula_sandbox_api.h> + +-using containerd::types::Mount; +-using google::protobuf::Timestamp; ++#include "controller.h" ++#include "utils_array.h" + + namespace sandbox { + +@@ -38,24 +33,13 @@ public: + + ~SandboxerClient() = default; + +- void Init(Errors &error); +- +- void Destroy(); +- + auto Create(const std::string &sandboxId, const ControllerCreateParams ¶ms, Errors &error) -> bool; + + auto Start(const std::string &sandboxId, ControllerSandboxInfo &sandboxInfo, Errors &error) -> bool; + + auto Platform(const std::string &sandboxId, ControllerPlatformInfo &platformInfo, Errors &error) -> bool; + +- auto Prepare(containerd::types::Sandbox &apiSandbox, +- std::vector<std::string> &fields, Errors &error) -> bool; +- +- auto Purge(containerd::types::Sandbox &apiSandbox, +- std::vector<std::string> &fields, Errors &error) -> bool; +- +- auto UpdateResources(const std::string &sandboxId, const ControllerUpdateResourcesParams ¶ms, +- Errors &error) -> bool; ++ auto Update(sandbox_sandbox *apiSandbox, string_array *fields, Errors &error) -> bool; + + auto Stop(const std::string &sandboxId, uint32_t timeoutSecs, Errors &error) -> bool; + +@@ -66,27 +50,23 @@ public: + auto Shutdown(const std::string &sandboxId, Errors &error) -> bool; + + private: +- void InitMountInfo(Mount &mount, const ControllerMountInfo &mountInfo); +- auto InitCreateRequest(containerd::services::sandbox::v1::ControllerCreateRequest &request, ++ auto InitMountInfo(sandbox_mount &m, const ControllerMountInfo &mountInfo) -> int; ++ auto InitCreateRequest(sandbox_create_request &request, + const std::string &sandboxId, + const ControllerCreateParams ¶ms) -> bool; +- void StartResponseToSandboxInfo(const containerd::services::sandbox::v1::ControllerStartResponse &response, ++ void StartResponseToSandboxInfo(sandbox_start_response &response, + ControllerSandboxInfo &sandboxInfo); +- void InitUpdateRequest(containerd::services::sandbox::v1::ControllerUpdateRequest &request, +- containerd::types::Sandbox &apiSandbox, std::vector<std::string> &fields); +- // auto InitUpdateResourcesRequest(containerd::services::sandbox::v1::UpdateResourcesRequest &request, +- // const std::string &sandboxId, +- // const ControllerUpdateResourcesParams ¶ms) -> bool; +- void PlatformResponseToPlatformInfo(const containerd::services::sandbox::v1::ControllerPlatformResponse &response, ++ void InitUpdateRequest(sandbox_update_request &request, ++ sandbox_sandbox *apiSandbox, string_array *fields); ++ ++ void PlatformResponseToPlatformInfo(sandbox_platform_response &response, + ControllerPlatformInfo &platformInfo); +- void StatusResponseToSandboxStatus(const containerd::services::sandbox::v1::ControllerStatusResponse &response, ++ void StatusResponseToSandboxStatus(sandbox_status_response &response, + ControllerSandboxStatus &sandboxStatus); + protected: + std::string m_sandboxer; + std::string m_address; +- std::shared_ptr<grpc::Channel> m_channel; +- std::unique_ptr<containerd::services::sandbox::v1::Controller::StubInterface> m_stub; +- std::unique_ptr<SandboxerClientMonitor> m_monitor; ++ ControllerHandle_t m_controller_handle; + }; + + } // namespace +diff --git a/src/daemon/sandbox/controller/sandboxer/sandboxer_controller.cc b/src/daemon/sandbox/sandboxer/controller/sandboxer_controller.cc +similarity index 78% +rename from src/daemon/sandbox/controller/sandboxer/sandboxer_controller.cc +rename to src/daemon/sandbox/sandboxer/controller/sandboxer_controller.cc +index 70cab015..bbe52c85 100644 +--- a/src/daemon/sandbox/controller/sandboxer/sandboxer_controller.cc ++++ b/src/daemon/sandbox/sandboxer/controller/sandboxer_controller.cc +@@ -27,15 +27,9 @@ SandboxerController::~SandboxerController() {} + + bool SandboxerController::Init(Errors &error) + { +- m_client->Init(error); + return true; + } + +-void SandboxerController::Destroy() +-{ +- m_client->Destroy(); +-} +- + bool SandboxerController::Create(const std::string &sandboxId, + const ControllerCreateParams ¶ms, + Errors &error) +@@ -61,25 +55,10 @@ std::unique_ptr<ControllerPlatformInfo> SandboxerController::Platform(const std: + return platformInfo; + } + +-bool SandboxerController::Prepare(containerd::types::Sandbox &apiSandbox, +- std::vector<std::string> &fields, +- Errors &error) +-{ +- return m_client->Prepare(apiSandbox, fields, error); +-} +- +-bool SandboxerController::Purge(containerd::types::Sandbox &apiSandbox, +- std::vector<std::string> &fields, +- Errors &error) +-{ +- return m_client->Purge(apiSandbox, fields, error); +-} +- +-bool SandboxerController::UpdateResources(const std::string &sandboxId, +- const ControllerUpdateResourcesParams ¶ms, +- Errors &error) ++bool SandboxerController::Update(sandbox_sandbox *apiSandbox, ++ string_array *fields, Errors &error) + { +- return m_client->UpdateResources(sandboxId, params, error); ++ return m_client->Update(apiSandbox, fields, error); + } + + bool SandboxerController::Stop(const std::string &sandboxId, uint32_t timeoutSecs, Errors &error) +diff --git a/src/daemon/sandbox/controller/sandboxer/sandboxer_controller.h b/src/daemon/sandbox/sandboxer/controller/sandboxer_controller.h +similarity index 82% +rename from src/daemon/sandbox/controller/sandboxer/sandboxer_controller.h +rename to src/daemon/sandbox/sandboxer/controller/sandboxer_controller.h +index 8cb7fe7c..13ab86ab 100644 +--- a/src/daemon/sandbox/controller/sandboxer/sandboxer_controller.h ++++ b/src/daemon/sandbox/sandboxer/controller/sandboxer_controller.h +@@ -31,19 +31,12 @@ public: + SandboxerController(const std::string &sandboxer, const std::string &address); + virtual ~SandboxerController(); + bool Init(Errors &error) override; +- void Destroy() override; + bool Create(const std::string &sandboxId, + const ControllerCreateParams ¶ms, + Errors &error) override; + std::unique_ptr<ControllerSandboxInfo> Start(const std::string &sandboxId, Errors &error) override; + std::unique_ptr<ControllerPlatformInfo> Platform(const std::string &sandboxId, Errors &error) override; +- bool Prepare(containerd::types::Sandbox &apiSandbox, std::vector<std::string> &fields, +- Errors &error) override; +- bool Purge(containerd::types::Sandbox &apiSandbox, std::vector<std::string> &fields, +- Errors &error) override; +- bool UpdateResources(const std::string &sandboxId, +- const ControllerUpdateResourcesParams ¶ms, +- Errors &error) override; ++ bool Update(sandbox_sandbox *apiSandbox, string_array *fields, Errors &error) override; + bool Stop(const std::string &sandboxId, uint32_t timeoutSecs, Errors &error) override; + bool Wait(std::shared_ptr<SandboxStatusCallback> cb, const std::string &sandboxId, Errors &error) override; + std::unique_ptr<ControllerSandboxStatus> Status(const std::string &sandboxId, bool verbose, Errors &error) override; +diff --git a/src/daemon/sandbox/sandboxer/sandboxer_sandbox.cc b/src/daemon/sandbox/sandboxer/sandboxer_sandbox.cc +new file mode 100644 +index 00000000..726b7b3a +--- /dev/null ++++ b/src/daemon/sandbox/sandboxer/sandboxer_sandbox.cc +@@ -0,0 +1,254 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-11-20 ++ * Description: provide sandboxer sandbox class definition ++ *********************************************************************************/ ++#include "sandboxer_sandbox.h" ++ ++#include <sys/stat.h> ++#include <unistd.h> ++#include <string> ++#include <mutex> ++#include <memory> ++#include <sys/mount.h> ++ ++#include <isula_libutils/log.h> ++#include <isula_libutils/auto_cleanup.h> ++ ++#include "utils_file.h" ++#include "utils.h" ++#include "cxxutils.h" ++ ++namespace sandbox { ++ ++SandboxerSandbox::SandboxerSandbox(const std::string id, const std::string &rootdir, const std::string &statedir, const std::string name, ++ const RuntimeInfo info, std::string netMode, std::string netNsPath, const runtime::v1::PodSandboxConfig sandboxConfig, ++ const std::string image):Sandbox(id, rootdir, statedir, name, info, netMode, ++ netNsPath, sandboxConfig, image) ++{ ++} ++ ++auto SandboxerSandbox::GetTasksJsonPath() -> std::string ++{ ++ return GetStateDir() + std::string("/") + SANDBOX_TASKS_JSON; ++} ++ ++auto SandboxerSandbox::AddTaskById(const char *task_id, sandbox_task *task) -> bool ++{ ++ ++ std::string taskId = std::string(task_id); ++ auto iter = m_tasks.find(taskId); ++ ++ if (iter != m_tasks.end()) { ++ ERROR("Failed to add existing sandbox task %s for sandbox: %s", ++ task_id, GetId().c_str()); ++ return false; ++ } ++ m_tasks[taskId] = std::make_shared<SandboxTask>(task); ++ return true; ++} ++ ++auto SandboxerSandbox::ReadSandboxTasksJson() -> sandbox_tasks * ++{ ++ const std::string path = GetTasksJsonPath(); ++ __isula_auto_free parser_error err = nullptr; ++ sandbox_tasks *tasksArray = nullptr; ++ ++ ReadGuard<RWMutex> lock(m_tasksMutex); ++ tasksArray = sandbox_tasks_parse_file(path.c_str(), nullptr, &err); ++ if (tasksArray == nullptr) { ++ WARN("Failed to read %s tasks json: %s", path.c_str(), err); ++ } ++ return tasksArray; ++} ++ ++auto SandboxerSandbox::WriteSandboxTasksJson(std::string &tasks_json) -> bool ++{ ++ int nret = 0; ++ const std::string path = GetTasksJsonPath(); ++ ++ WriteGuard<RWMutex> lock(m_tasksMutex); ++ nret = util_atomic_write_file(path.c_str(), tasks_json.c_str(), tasks_json.size(), CONFIG_FILE_MODE, false); ++ if (nret != 0) { ++ SYSERROR("Failed to write file %s", path.c_str()); ++ } ++ return nret == 0; ++} ++ ++auto SandboxerSandbox::DeleteSandboxTasksJson() -> bool ++{ ++ int get_err = 0; ++ const std::string path = GetTasksJsonPath(); ++ ++ WriteGuard<RWMutex> lock(m_tasksMutex); ++ if (util_fileself_exists(path.c_str()) && ++ !util_force_remove_file(path.c_str(), &get_err)) { ++ errno = get_err; ++ SYSERROR("Failed to remove file %s", path.c_str()); ++ return false; ++ } ++ ++ return true; ++} ++ ++void SandboxerSandbox::AddSandboxTasksByArray(sandbox_tasks *tasksArray) ++{ ++ size_t i; ++ ++ WriteGuard<RWMutex> lock(m_tasksMutex); ++ for (i = 0; i < tasksArray->tasks_len; i++) { ++ if (!AddTaskById(tasksArray->tasks[i]->task_id, tasksArray->tasks[i])) { ++ return; ++ } ++ tasksArray->tasks[i] = nullptr; ++ } ++ tasksArray->tasks_len = 0; ++} ++ ++void SandboxerSandbox::LoadSandboxTasks() ++{ ++ sandbox_tasks *tasksArray = nullptr; ++ ++ tasksArray = ReadSandboxTasksJson(); ++ if (tasksArray == nullptr) { ++ return; ++ } ++ ++ AddSandboxTasksByArray(tasksArray); ++ ++ free_sandbox_tasks(tasksArray); ++} ++ ++auto SandboxerSandbox::SaveSandboxTasks() -> bool ++{ ++ std::string tasks_json; ++ ++ if (m_tasks.empty()) { ++ return DeleteSandboxTasksJson(); ++ } ++ ++ tasks_json = GetAnySandboxTasks(); ++ if (tasks_json.empty()) { ++ ERROR("Failed to get sandbox tasks json for sandbox: '%s'", GetId().c_str()); ++ return false; ++ } ++ ++ return WriteSandboxTasksJson(tasks_json); ++} ++ ++auto SandboxerSandbox::AddSandboxTasks(sandbox_task *task) -> bool ++{ ++ if (task == nullptr) { ++ return true; ++ } ++ if (task->task_id == nullptr) { ++ return false; ++ } ++ ++ WriteGuard<RWMutex> lock(m_tasksMutex); ++ ++ return AddTaskById(task->task_id, task); ++} ++ ++auto SandboxerSandbox::GetAnySandboxTasks() -> std::string ++{ ++ __isula_auto_free parser_error err = nullptr; ++ sandbox_tasks tasksArray = { 0 }; ++ size_t i = 0; ++ __isula_auto_free char *tasks_json = nullptr; ++ ++ tasksArray.tasks = (sandbox_task **)util_smart_calloc_s(sizeof(sandbox_task *), m_tasks.size()); ++ if (tasksArray.tasks == nullptr) { ++ SYSERROR("Out of memory."); ++ return std::string(""); ++ } ++ ++ ReadGuard<RWMutex> lock(m_tasksMutex); ++ for (auto const& [_, val] : m_tasks) { ++ /* ++ * We ignore that the processes are modified ++ * when we generate tasks json string. ++ * Because no matter whether a process is deleted or added, ++ * the Update of sandbox api will be called eventually. ++ * ++ * And we ignore that the task is freed after we do GetTask(). ++ * Because the only way to free task is DeleteSandboxTasks() ++ * which needs write lock of m_tasksMutex. ++ */ ++ tasksArray.tasks[i] = val->GetTask(); ++ i++; ++ } ++ tasksArray.tasks_len = m_tasks.size(); ++ ++ tasks_json = sandbox_tasks_generate_json(&tasksArray, nullptr, &(err)); ++ if (tasks_json == nullptr || strlen(tasks_json) == 0) { ++ ERROR("Failed to get sandbox tasks json for sandbox: '%s'", GetId().c_str()); ++ free(tasksArray.tasks); ++ return std::string(""); ++ } ++ ++ free(tasksArray.tasks); ++ return std::string(tasks_json); ++} ++ ++void SandboxerSandbox::DeleteSandboxTasks(const char *containerId) ++{ ++ if (containerId == nullptr) { ++ return; ++ } ++ ++ std::string taskId = std::string(containerId); ++ ++ WriteGuard<RWMutex> lock(m_tasksMutex); ++ auto iter = m_tasks.find(taskId); ++ if (iter == m_tasks.end()) { ++ return; ++ } ++ m_tasks.erase(iter); ++} ++ ++auto SandboxerSandbox::AddSandboxTasksProcess(const char *containerId, sandbox_process *processes) -> bool ++{ ++ if (containerId == nullptr || processes == nullptr) { ++ ERROR("Empty args."); ++ return false; ++ } ++ ++ std::string taskId = std::string(containerId); ++ ++ ReadGuard<RWMutex> lock(m_tasksMutex); ++ auto iter = m_tasks.find(taskId); ++ if (iter == m_tasks.end()) { ++ SYSERROR("Failed to find container %s", containerId); ++ return false; ++ } ++ ++ return iter->second->AddSandboxTasksProcess(processes); ++} ++ ++void SandboxerSandbox::DeleteSandboxTasksProcess(const char *containerId, const char *execId) ++{ ++ if (containerId == nullptr || execId == nullptr) { ++ return; ++ } ++ ++ std::string taskId = std::string(containerId); ++ ++ ReadGuard<RWMutex> lock(m_tasksMutex); ++ auto iter = m_tasks.find(taskId); ++ if (iter == m_tasks.end()) { ++ return; ++ } ++ iter->second->DeleteSandboxTasksProcess(execId); ++} ++ ++} +\ No newline at end of file +diff --git a/src/daemon/sandbox/sandboxer/sandboxer_sandbox.h b/src/daemon/sandbox/sandboxer/sandboxer_sandbox.h +new file mode 100644 +index 00000000..552eb328 +--- /dev/null ++++ b/src/daemon/sandbox/sandboxer/sandboxer_sandbox.h +@@ -0,0 +1,63 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-11-20 ++ * Description: provide sandboxer sandbox class definition ++ *********************************************************************************/ ++ ++#ifndef DAEMON_SANDBOX_SANDBOXER_SANDBOX_H ++#define DAEMON_SANDBOX_SANDBOXER_SANDBOX_H ++ ++#include <string> ++#include <mutex> ++#include <google/protobuf/map.h> ++ ++#include "read_write_lock.h" ++#include "sandbox_task.h" ++#include "sandbox.h" ++ ++namespace sandbox { ++ ++class SandboxerSandbox : public Sandbox { ++public: ++ SandboxerSandbox(const std::string id, const std::string &rootdir, const std::string &statedir, const std::string name = "", ++ const RuntimeInfo info = {"", "", ""}, std::string netMode = DEFAULT_NETMODE, std::string netNsPath = "", ++ const runtime::v1::PodSandboxConfig sandboxConfig = runtime::v1::PodSandboxConfig::default_instance(), ++ const std::string image = ""); ++ virtual ~SandboxerSandbox() = default; ++ ++ // for sandbox api update ++ auto GetTasksJsonPath() -> std::string; ++ void LoadSandboxTasks() override; ++ auto SaveSandboxTasks() -> bool override; ++ auto AddSandboxTasks(sandbox_task *task) -> bool override; ++ auto GetAnySandboxTasks() -> std::string override; ++ void DeleteSandboxTasks(const char *containerId) override; ++ auto AddSandboxTasksProcess(const char *containerId, sandbox_process *processes) -> bool override; ++ void DeleteSandboxTasksProcess(const char *containerId, const char *execId) override; ++ ++private: ++ auto AddTaskById(const char *task_id, sandbox_task *task) -> bool; ++ auto ReadSandboxTasksJson() -> sandbox_tasks *; ++ auto WriteSandboxTasksJson(std::string &tasks_json) -> bool; ++ auto DeleteSandboxTasksJson() -> bool; ++ void AddSandboxTasksByArray(sandbox_tasks *tasksArray); ++ ++private: ++ // use m_tasksMutex to ensure the correctness of the tasks ++ RWMutex m_tasksMutex; ++ // for sandbox api update, containerId --> tasks ++ std::map<std::string, std::shared_ptr<SandboxTask>> m_tasks; ++}; ++ ++} // namespace sandbox ++ ++#endif // DAEMON_SANDBOX_SANDBOXER_SANDBOX_H +\ No newline at end of file +diff --git a/src/daemon/sandbox/shim/CMakeLists.txt b/src/daemon/sandbox/shim/CMakeLists.txt +new file mode 100755 +index 00000000..66fff738 +--- /dev/null ++++ b/src/daemon/sandbox/shim/CMakeLists.txt +@@ -0,0 +1,15 @@ ++# get current directory sources files ++aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} shim_sandbox_srcs) ++ ++add_subdirectory(controller) ++ ++set(SHIM_SANDBOX_SRCS ++ ${shim_sandbox_srcs} ++ ${CONTROLLER_SHIM_SRCS} ++ PARENT_SCOPE ++ ) ++set(SHIM_SANDBOX_INCS ++ ${CMAKE_CURRENT_SOURCE_DIR} ++ ${CONTROLLER_SHIM_INCS} ++ PARENT_SCOPE ++ ) +diff --git a/src/daemon/sandbox/controller/shim/CMakeLists.txt b/src/daemon/sandbox/shim/controller/CMakeLists.txt +similarity index 100% +rename from src/daemon/sandbox/controller/shim/CMakeLists.txt +rename to src/daemon/sandbox/shim/controller/CMakeLists.txt +diff --git a/src/daemon/sandbox/controller/shim/shim_controller.cc b/src/daemon/sandbox/shim/controller/shim_controller.cc +similarity index 97% +rename from src/daemon/sandbox/controller/shim/shim_controller.cc +rename to src/daemon/sandbox/shim/controller/shim_controller.cc +index 14c99168..7e4338f6 100644 +--- a/src/daemon/sandbox/controller/shim/shim_controller.cc ++++ b/src/daemon/sandbox/shim/controller/shim_controller.cc +@@ -45,8 +45,6 @@ bool ShimController::Init(Errors &error) + return true; + } + +-void ShimController::Destroy() {} +- + void ShimController::ApplySandboxLinuxOptions(const runtime::v1::LinuxPodSandboxConfig &lc, host_config *hc, + container_config *custom_config, Errors &error) + { +@@ -340,21 +338,8 @@ std::unique_ptr<ControllerPlatformInfo> ShimController::Platform(const std::stri + return nullptr; + } + +-bool ShimController::Prepare(containerd::types::Sandbox &apiSandbox, +- std::vector<std::string> &fields, Errors &error) +-{ +- return true; +-} +- +-bool ShimController::Purge(containerd::types::Sandbox &apiSandbox, +- std::vector<std::string> &fields, Errors &error) +-{ +- return true; +-} +- +-bool ShimController::UpdateResources(const std::string &sandboxId, +- const ControllerUpdateResourcesParams ¶ms, +- Errors &error) ++bool ShimController::Update(sandbox_sandbox *apiSandbox, ++ string_array *fields, Errors &error) + { + return true; + } +diff --git a/src/daemon/sandbox/controller/shim/shim_controller.h b/src/daemon/sandbox/shim/controller/shim_controller.h +similarity index 89% +rename from src/daemon/sandbox/controller/shim/shim_controller.h +rename to src/daemon/sandbox/shim/controller/shim_controller.h +index 1985ddc0..fc86be98 100644 +--- a/src/daemon/sandbox/controller/shim/shim_controller.h ++++ b/src/daemon/sandbox/shim/controller/shim_controller.h +@@ -32,26 +32,18 @@ + + namespace sandbox { + +-const std::string SHIM_CONTROLLER_NAME = "shim"; +- + class ShimController : public Controller { + public: + ShimController(const std::string &sandboxer); + virtual ~ShimController(); + bool Init(Errors &error) override; +- void Destroy() override; + bool Create(const std::string &sandboxId, + const ControllerCreateParams ¶ms, + Errors &error) override; + std::unique_ptr<ControllerSandboxInfo> Start(const std::string &sandboxId, Errors &error) override; + std::unique_ptr<ControllerPlatformInfo> Platform(const std::string &sandboxId, Errors &error) override; +- bool Prepare(containerd::types::Sandbox &apiSandbox, std::vector<std::string> &fields, +- Errors &error) override; +- bool Purge(containerd::types::Sandbox &apiSandbox, std::vector<std::string> &fields, +- Errors &error) override; +- bool UpdateResources(const std::string &sandboxId, +- const ControllerUpdateResourcesParams ¶ms, +- Errors &error) override; ++ bool Update(sandbox_sandbox *apiSandbox, ++ string_array *fields, Errors &error) override; + bool Stop(const std::string &sandboxId, uint32_t timeoutSecs, Errors &error) override; + bool Wait(std::shared_ptr<SandboxStatusCallback> cb, const std::string &sandboxId, Errors &error) override; + std::unique_ptr<ControllerSandboxStatus> Status(const std::string &sandboxId, bool verbose, Errors &error) override; +diff --git a/src/daemon/sandbox/shim/shim_sandbox.cc b/src/daemon/sandbox/shim/shim_sandbox.cc +new file mode 100644 +index 00000000..2efb8d7c +--- /dev/null ++++ b/src/daemon/sandbox/shim/shim_sandbox.cc +@@ -0,0 +1,65 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-11-20 ++ * Description: provide sandboxer sandbox class definition ++ *********************************************************************************/ ++#include "shim_sandbox.h" ++ ++#include <unistd.h> ++#include <string> ++ ++#include <isula_libutils/log.h> ++#include <isula_libutils/auto_cleanup.h> ++ ++ ++namespace sandbox { ++ ++ShimSandbox::ShimSandbox(const std::string id, const std::string &rootdir, const std::string &statedir, const std::string name, ++ const RuntimeInfo info, std::string netMode, std::string netNsPath, const runtime::v1::PodSandboxConfig sandboxConfig, ++ const std::string image):Sandbox(id, rootdir, statedir, name, info, netMode, ++ netNsPath, sandboxConfig, image) ++{ ++} ++ ++void ShimSandbox::LoadSandboxTasks() ++{ ++} ++ ++auto ShimSandbox::SaveSandboxTasks() -> bool ++{ ++ return true; ++} ++ ++auto ShimSandbox::AddSandboxTasks(sandbox_task *task) -> bool ++{ ++ return true; ++} ++ ++auto ShimSandbox::GetAnySandboxTasks() -> std::string ++{ ++ return std::string("Nothing for shim."); ++} ++ ++void ShimSandbox::DeleteSandboxTasks(const char *containerId) ++{ ++} ++ ++auto ShimSandbox::AddSandboxTasksProcess(const char *containerId, sandbox_process *processes) -> bool ++{ ++ return true; ++} ++ ++void ShimSandbox::DeleteSandboxTasksProcess(const char *containerId, const char *execId) ++{ ++} ++ ++} +\ No newline at end of file +diff --git a/src/daemon/sandbox/shim/shim_sandbox.h b/src/daemon/sandbox/shim/shim_sandbox.h +new file mode 100644 +index 00000000..82da0573 +--- /dev/null ++++ b/src/daemon/sandbox/shim/shim_sandbox.h +@@ -0,0 +1,49 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-11-20 ++ * Description: provide shim sandbox class definition ++ *********************************************************************************/ ++ ++#ifndef DAEMON_SANDBOX_SHIM_SANDBOX_H ++#define DAEMON_SANDBOX_SHIM_SANDBOX_H ++ ++#include <string> ++#include <mutex> ++#include <google/protobuf/map.h> ++ ++#include "read_write_lock.h" ++#include "sandbox_task.h" ++#include "sandbox.h" ++ ++namespace sandbox { ++ ++class ShimSandbox : public Sandbox { ++public: ++ ShimSandbox(const std::string id, const std::string &rootdir, const std::string &statedir, const std::string name = "", ++ const RuntimeInfo info = {"", "", ""}, std::string netMode = DEFAULT_NETMODE, std::string netNsPath = "", ++ const runtime::v1::PodSandboxConfig sandboxConfig = runtime::v1::PodSandboxConfig::default_instance(), ++ const std::string image = ""); ++ virtual ~ShimSandbox() = default; ++ ++ // for sandbox api update ++ void LoadSandboxTasks() override; ++ auto SaveSandboxTasks() -> bool override; ++ auto AddSandboxTasks(sandbox_task *task) -> bool override; ++ auto GetAnySandboxTasks() -> std::string override; ++ void DeleteSandboxTasks(const char *containerId) override; ++ auto AddSandboxTasksProcess(const char *containerId, sandbox_process *processes) -> bool override; ++ void DeleteSandboxTasksProcess(const char *containerId, const char *execId) override; ++}; ++ ++} // namespace sandbox ++ ++#endif // DAEMON_SANDBOX_SHIM_SANDBOX_H +\ No newline at end of file +-- +2.34.1 + diff --git a/0154-add-linux-capability.h-head-file.patch b/0154-add-linux-capability.h-head-file.patch new file mode 100644 index 0000000..24853ac --- /dev/null +++ b/0154-add-linux-capability.h-head-file.patch @@ -0,0 +1,25 @@ +From 94cf20fe53c0a42f703dd6a0122c619c0ee06715 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Sat, 23 Nov 2024 03:45:22 +1400 +Subject: [PATCH 154/156] add linux/capability.h head file + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/utils/cutils/utils_cap.h | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/src/utils/cutils/utils_cap.h b/src/utils/cutils/utils_cap.h +index c7e78ac2..e741b565 100644 +--- a/src/utils/cutils/utils_cap.h ++++ b/src/utils/cutils/utils_cap.h +@@ -18,6 +18,7 @@ + + #include <stdbool.h> + #include <stddef.h> ++#include <linux/capability.h> + + #ifdef __cplusplus + extern "C" { +-- +2.34.1 + diff --git a/0155-sandbox-fix-unused-variables.patch b/0155-sandbox-fix-unused-variables.patch new file mode 100644 index 0000000..5dbc54c --- /dev/null +++ b/0155-sandbox-fix-unused-variables.patch @@ -0,0 +1,34 @@ +From baa25031fcf68443bc1262bebec69a54fed38d88 Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Mon, 25 Nov 2024 15:18:13 +0800 +Subject: [PATCH 155/156] sandbox:fix unused variables + +Signed-off-by: liuxu <liuxu156@huawei.com> +--- + .../sandboxer/controller/client/grpc_sandboxer_client.cc | 3 +-- + 1 file changed, 1 insertion(+), 2 deletions(-) + +diff --git a/src/daemon/sandbox/sandboxer/controller/client/grpc_sandboxer_client.cc b/src/daemon/sandbox/sandboxer/controller/client/grpc_sandboxer_client.cc +index d7ff76d7..5b7f3c1c 100644 +--- a/src/daemon/sandbox/sandboxer/controller/client/grpc_sandboxer_client.cc ++++ b/src/daemon/sandbox/sandboxer/controller/client/grpc_sandboxer_client.cc +@@ -92,7 +92,6 @@ auto SandboxerClient::InitCreateRequest(sandbox_create_request &request, + (void)memcpy(request.options->value, encoded.c_str(), encoded.size()); + request.options->value_len = encoded.size(); + request.netns_path = util_strdup_s(params.netNSPath.c_str()); +- sandbox_mount **mounts = NULL; + size_t mounts_len = params.mounts.size(); + if (mounts_len > 0) { + request.rootfs = (sandbox_mount**)util_common_calloc_s(mounts_len * sizeof(sandbox_mount *)); +@@ -112,7 +111,7 @@ auto SandboxerClient::InitCreateRequest(sandbox_create_request &request, + free(m); + return false; + } +- mounts[request.rootfs_len++] = m; ++ request.rootfs[request.rootfs_len++] = m; + m = NULL; + } + } +-- +2.34.1 + diff --git a/0156-sandbox-sandbox-api-adapt-rust-interface-UT.patch b/0156-sandbox-sandbox-api-adapt-rust-interface-UT.patch new file mode 100644 index 0000000..410fb2d --- /dev/null +++ b/0156-sandbox-sandbox-api-adapt-rust-interface-UT.patch @@ -0,0 +1,3021 @@ +From 6de0c825b7d8dd2d7b1c53524f6ecebcef22bc75 Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Fri, 22 Nov 2024 14:11:42 +0800 +Subject: [PATCH 156/156] sandbox: sandbox api adapt rust interface UT + +Signed-off-by: liuxu <liuxu156@huawei.com> +--- + test/cutils/utils_transform/CMakeLists.txt | 1 - + test/mocks/controller_stub_mock.cc | 314 ---------------- + test/mocks/controller_stub_mock.h | 239 ------------ + test/mocks/grpc_async_wait_call_mock.cc | 92 ----- + test/mocks/grpc_async_wait_call_mock.h | 37 -- + test/mocks/grpc_sandboxer_client_mock.cc | 44 +-- + test/mocks/grpc_sandboxer_client_mock.h | 11 +- + test/mocks/grpc_sandboxer_monitor_mock.cc | 51 --- + test/mocks/rust_sandbox_api_mock.cc | 103 ++++++ + test/mocks/rust_sandbox_api_mock.h | 39 ++ + test/mocks/sandbox_manager_mock.cc | 42 +++ + ..._monitor_mock.h => sandbox_manager_mock.h} | 35 +- + test/mocks/sandboxer_controller_mock.cc | 116 ++++++ + test/mocks/sandboxer_controller_mock.h | 52 +++ + test/mocks/sandboxer_sandbox_mock.cc | 72 ++++ + test/mocks/sandboxer_sandbox_mock.h | 43 +++ + test/mocks/shim_controller_mock.cc | 36 +- + test/mocks/shim_controller_mock.h | 11 +- + test/mocks/shim_sandbox_mock.cc | 72 ++++ + test/mocks/shim_sandbox_mock.h | 43 +++ + test/sandbox/controller/CMakeLists.txt | 2 - + test/sandbox/controller/controller_common.cc | 51 ++- + test/sandbox/controller/controller_common.h | 9 +- + .../sandbox/controller/manager/CMakeLists.txt | 15 +- + .../controller/sandboxer/CMakeLists.txt | 1 - + .../sandboxer/async_wait_call/CMakeLists.txt | 37 -- + .../async_wait_call/async_wait_call_ut.cc | 115 ------ + .../async_wait_call/dummy_monitor_utils.h | 121 ------ + .../sandboxer/sandboxer_client/CMakeLists.txt | 22 +- + .../sandboxer_client/sandboxer_client_ut.cc | 344 ++++++++---------- + .../sandboxer_controller/CMakeLists.txt | 11 +- + .../sandboxer_controller_ut.cc | 62 +--- + test/sandbox/controller/shim/CMakeLists.txt | 7 +- + test/sandbox/sandbox/CMakeLists.txt | 21 +- + test/sandbox/sandbox/sandbox_ut.cc | 5 +- + test/sandbox/sandbox_manager/CMakeLists.txt | 21 +- + 36 files changed, 861 insertions(+), 1436 deletions(-) + delete mode 100644 test/mocks/controller_stub_mock.cc + delete mode 100644 test/mocks/controller_stub_mock.h + delete mode 100644 test/mocks/grpc_async_wait_call_mock.cc + delete mode 100644 test/mocks/grpc_async_wait_call_mock.h + delete mode 100644 test/mocks/grpc_sandboxer_monitor_mock.cc + create mode 100644 test/mocks/rust_sandbox_api_mock.cc + create mode 100644 test/mocks/rust_sandbox_api_mock.h + create mode 100644 test/mocks/sandbox_manager_mock.cc + rename test/mocks/{grpc_sandboxer_monitor_mock.h => sandbox_manager_mock.h} (55%) + create mode 100644 test/mocks/sandboxer_controller_mock.cc + create mode 100644 test/mocks/sandboxer_controller_mock.h + create mode 100644 test/mocks/sandboxer_sandbox_mock.cc + create mode 100644 test/mocks/sandboxer_sandbox_mock.h + create mode 100644 test/mocks/shim_sandbox_mock.cc + create mode 100644 test/mocks/shim_sandbox_mock.h + delete mode 100644 test/sandbox/controller/sandboxer/async_wait_call/CMakeLists.txt + delete mode 100644 test/sandbox/controller/sandboxer/async_wait_call/async_wait_call_ut.cc + delete mode 100644 test/sandbox/controller/sandboxer/async_wait_call/dummy_monitor_utils.h + +diff --git a/test/cutils/utils_transform/CMakeLists.txt b/test/cutils/utils_transform/CMakeLists.txt +index fde9c9f9..23c870fa 100644 +--- a/test/cutils/utils_transform/CMakeLists.txt ++++ b/test/cutils/utils_transform/CMakeLists.txt +@@ -15,7 +15,6 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cpputils +- ${CMAKE_BINARY_DIR}/grpc/src/api/services/sandbox/google/protobuf + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lprotobuf -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) +diff --git a/test/mocks/controller_stub_mock.cc b/test/mocks/controller_stub_mock.cc +deleted file mode 100644 +index 712540bb..00000000 +--- a/test/mocks/controller_stub_mock.cc ++++ /dev/null +@@ -1,314 +0,0 @@ +-#include "controller_stub_mock.h" +- +-static std::shared_ptr<MockControllerStub> g_controller_stub_mock = NULL; +- +-std::unique_ptr<DummyControllerStub> NewDummyControllerStub() +-{ +- std::unique_ptr<DummyControllerStub> stub(new DummyControllerStub()); +- return stub; +-} +- +-void MockControllerStub_SetMock(std::shared_ptr<MockControllerStub> mock) +-{ +- g_controller_stub_mock = mock; +-} +- +-::grpc::Status DummyControllerStub::Create(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerCreateRequest &request, +- ::containerd::services::sandbox::v1::ControllerCreateResponse* response) +-{ +- if (g_controller_stub_mock == NULL) { +- return ::grpc::Status::OK; +- } +- return g_controller_stub_mock->Create(context, request, response); +-} +- +-::grpc::Status DummyControllerStub::Start(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerStartRequest &request, +- ::containerd::services::sandbox::v1::ControllerStartResponse* response) +-{ +- if (g_controller_stub_mock == NULL) { +- return ::grpc::Status::OK; +- } +- return g_controller_stub_mock->Start(context, request, response); +-} +- +-::grpc::Status DummyControllerStub::Platform(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerPlatformRequest &request, +- ::containerd::services::sandbox::v1::ControllerPlatformResponse* response) +-{ +- if (g_controller_stub_mock == NULL) { +- return ::grpc::Status::OK; +- } +- return g_controller_stub_mock->Platform(context, request, response); +-} +- +-::grpc::Status DummyControllerStub::Prepare(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::PrepareRequest &request, +- ::containerd::services::sandbox::v1::PrepareResponse* response) +-{ +- if (g_controller_stub_mock == NULL) { +- return ::grpc::Status::OK; +- } +- return g_controller_stub_mock->Prepare(context, request, response); +-} +- +-::grpc::Status DummyControllerStub::Purge(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::PurgeRequest &request, +- ::containerd::services::sandbox::v1::PurgeResponse* response) +-{ +- if (g_controller_stub_mock == NULL) { +- return ::grpc::Status::OK; +- } +- return g_controller_stub_mock->Purge(context, request, response); +-} +- +-::grpc::Status DummyControllerStub::UpdateResources(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::UpdateResourcesRequest &request, +- ::containerd::services::sandbox::v1::UpdateResourcesResponse* response) +-{ +- if (g_controller_stub_mock == NULL) { +- return ::grpc::Status::OK; +- } +- return g_controller_stub_mock->UpdateResources(context, request, response); +-} +- +-::grpc::Status DummyControllerStub::Stop(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerStopRequest &request, +- ::containerd::services::sandbox::v1::ControllerStopResponse* response) +-{ +- if (g_controller_stub_mock == NULL) { +- return ::grpc::Status::OK; +- } +- return g_controller_stub_mock->Stop(context, request, response); +-} +- +-::grpc::Status DummyControllerStub::Wait(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerWaitRequest &request, +- ::containerd::services::sandbox::v1::ControllerWaitResponse* response) +-{ +- if (g_controller_stub_mock == NULL) { +- return ::grpc::Status::OK; +- } +- return g_controller_stub_mock->Wait(context, request, response); +-} +- +-::grpc::Status DummyControllerStub::Status(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerStatusRequest &request, +- ::containerd::services::sandbox::v1::ControllerStatusResponse* response) +-{ +- if (g_controller_stub_mock == NULL) { +- return ::grpc::Status::OK; +- } +- return g_controller_stub_mock->Status(context, request, response); +-} +- +-::grpc::Status DummyControllerStub::Shutdown(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerShutdownRequest &request, +- ::containerd::services::sandbox::v1::ControllerShutdownResponse* response) +-{ +- if (g_controller_stub_mock == NULL) { +- return ::grpc::Status::OK; +- } +- return g_controller_stub_mock->Shutdown(context, request, response); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerCreateResponse>* +-DummyControllerStub::AsyncCreateRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerCreateRequest &request, ::grpc::CompletionQueue* cq) +-{ +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->AsyncCreateRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerCreateResponse>* +-DummyControllerStub::PrepareAsyncCreateRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerCreateRequest &request, ::grpc::CompletionQueue* cq) +-{ +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->PrepareAsyncCreateRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStartResponse>* +-DummyControllerStub::AsyncStartRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerStartRequest &request, ::grpc::CompletionQueue* cq) +-{ +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->AsyncStartRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStartResponse>* +-DummyControllerStub::PrepareAsyncStartRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerStartRequest &request, ::grpc::CompletionQueue* cq) +-{ +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->PrepareAsyncStartRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerPlatformResponse>* +-DummyControllerStub::AsyncPlatformRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerPlatformRequest &request, ::grpc::CompletionQueue* cq) +-{ +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->AsyncPlatformRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerPlatformResponse>* +-DummyControllerStub::PrepareAsyncPlatformRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerPlatformRequest &request, ::grpc::CompletionQueue* cq) +-{ +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->PrepareAsyncPlatformRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PrepareResponse>* +-DummyControllerStub::AsyncPrepareRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::PrepareRequest &request, ::grpc::CompletionQueue* cq) +-{ +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->AsyncPrepareRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PrepareResponse>* +-DummyControllerStub::PrepareAsyncPrepareRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::PrepareRequest &request, ::grpc::CompletionQueue* cq) +-{ +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->PrepareAsyncPrepareRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PurgeResponse>* +-DummyControllerStub::AsyncPurgeRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::PurgeRequest &request, ::grpc::CompletionQueue* cq) +-{ +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->AsyncPurgeRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PurgeResponse>* +-DummyControllerStub::PrepareAsyncPurgeRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::PurgeRequest &request, ::grpc::CompletionQueue* cq) +-{ +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->PrepareAsyncPurgeRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::UpdateResourcesResponse>* +-DummyControllerStub::AsyncUpdateResourcesRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::UpdateResourcesRequest &request, ::grpc::CompletionQueue* cq) +-{ +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->AsyncUpdateResourcesRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::UpdateResourcesResponse>* +-DummyControllerStub::PrepareAsyncUpdateResourcesRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::UpdateResourcesRequest &request, ::grpc::CompletionQueue* cq) +-{ +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->PrepareAsyncUpdateResourcesRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStopResponse>* +-DummyControllerStub::AsyncStopRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerStopRequest &request, ::grpc::CompletionQueue* cq) +-{ +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->AsyncStopRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStopResponse>* +-DummyControllerStub::PrepareAsyncStopRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerStopRequest &request, ::grpc::CompletionQueue* cq) +-{ +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->PrepareAsyncStopRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerWaitResponse>* +-DummyControllerStub::AsyncWaitRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerWaitRequest &request, ::grpc::CompletionQueue* cq) +-{ +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->AsyncWaitRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerWaitResponse>* +-DummyControllerStub::PrepareAsyncWaitRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerWaitRequest &request, ::grpc::CompletionQueue* cq) +-{ +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->PrepareAsyncWaitRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStatusResponse>* +-DummyControllerStub::AsyncStatusRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerStatusRequest &request, ::grpc::CompletionQueue* cq) +-{ +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->AsyncStatusRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStatusResponse>* +-DummyControllerStub::PrepareAsyncStatusRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerStatusRequest &request, ::grpc::CompletionQueue* cq) +-{ +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->PrepareAsyncStatusRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerShutdownResponse>* +-DummyControllerStub::AsyncShutdownRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerShutdownRequest &request, ::grpc::CompletionQueue* cq) +-{ +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->AsyncShutdownRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerShutdownResponse>* +-DummyControllerStub::PrepareAsyncShutdownRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerShutdownRequest &request, ::grpc::CompletionQueue* cq) +-{ +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->PrepareAsyncShutdownRaw(context, request, cq); +-} +diff --git a/test/mocks/controller_stub_mock.h b/test/mocks/controller_stub_mock.h +deleted file mode 100644 +index 85cb82bb..00000000 +--- a/test/mocks/controller_stub_mock.h ++++ /dev/null +@@ -1,239 +0,0 @@ +-/****************************************************************************** +- * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +- * iSulad licensed under the Mulan PSL v2. +- * You can use this software according to the terms and conditions of the Mulan PSL v2. +- * You may obtain a copy of Mulan PSL v2 at: +- * http://license.coscl.org.cn/MulanPSL2 +- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +- * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +- * PURPOSE. +- * See the Mulan PSL v2 for more details. +- * Author: xuxuepeng +- * Create: 2023-07-15 +- * Description: provide grpc controller stub mock +- ******************************************************************************/ +- +-#ifndef _ISULAD_TEST_MOCKS_CONTROLLER_STUB_MOCK_H +-#define _ISULAD_TEST_MOCKS_CONTROLLER_STUB_MOCK_H +- +- +-#include <gmock/gmock.h> +-#include "sandbox.grpc.pb.h" +- +-// MockControllerStub is a mock implementation of the Controller::StubInterface interface. +-class MockControllerStub { +-public: +- MOCK_METHOD3(Create, ::grpc::Status(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerCreateRequest &request, +- ::containerd::services::sandbox::v1::ControllerCreateResponse* response)); +- MOCK_METHOD3(Start, ::grpc::Status(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerStartRequest &request, +- ::containerd::services::sandbox::v1::ControllerStartResponse* response)); +- MOCK_METHOD3(Platform, ::grpc::Status(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerPlatformRequest &request, +- ::containerd::services::sandbox::v1::ControllerPlatformResponse* response)); +- MOCK_METHOD3(Prepare, ::grpc::Status(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::PrepareRequest &request, +- ::containerd::services::sandbox::v1::PrepareResponse* response)); +- MOCK_METHOD3(Purge, ::grpc::Status(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::PurgeRequest &request, +- ::containerd::services::sandbox::v1::PurgeResponse* response)); +- MOCK_METHOD3(UpdateResources, ::grpc::Status(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::UpdateResourcesRequest &request, +- ::containerd::services::sandbox::v1::UpdateResourcesResponse* response)); +- MOCK_METHOD3(Stop, ::grpc::Status(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerStopRequest &request, +- ::containerd::services::sandbox::v1::ControllerStopResponse* response)); +- MOCK_METHOD3(Wait, ::grpc::Status(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerWaitRequest &request, +- ::containerd::services::sandbox::v1::ControllerWaitResponse* response)); +- MOCK_METHOD3(Status, ::grpc::Status(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerStatusRequest &request, +- ::containerd::services::sandbox::v1::ControllerStatusResponse* response)); +- MOCK_METHOD3(Shutdown, ::grpc::Status(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerShutdownRequest &request, +- ::containerd::services::sandbox::v1::ControllerShutdownResponse* response)); +- MOCK_METHOD3(AsyncCreateRaw, +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerCreateResponse>* +- (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerCreateRequest &request, +- ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(PrepareAsyncCreateRaw, +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerCreateResponse>* +- (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerCreateRequest &request, +- ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(AsyncStartRaw, +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStartResponse>* +- (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStartRequest &request, +- ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(PrepareAsyncStartRaw, +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStartResponse>* +- (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStartRequest &request, +- ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(AsyncPlatformRaw, +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerPlatformResponse>* +- (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerPlatformRequest &request, +- ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(PrepareAsyncPlatformRaw, +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerPlatformResponse>* +- (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerPlatformRequest &request, +- ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(AsyncPrepareRaw, +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PrepareResponse>* +- (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PrepareRequest &request, +- ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(PrepareAsyncPrepareRaw, +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PrepareResponse>* +- (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PrepareRequest &request, +- ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(AsyncPurgeRaw, +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PurgeResponse>* +- (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PurgeRequest &request, +- ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(PrepareAsyncPurgeRaw, +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PurgeResponse>* +- (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PurgeRequest &request, +- ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(AsyncUpdateResourcesRaw, +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::UpdateResourcesResponse>* +- (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::UpdateResourcesRequest &request, +- ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(PrepareAsyncUpdateResourcesRaw, +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::UpdateResourcesResponse>* +- (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::UpdateResourcesRequest &request, +- ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(AsyncStopRaw, +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStopResponse>* +- (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStopRequest &request, +- ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(PrepareAsyncStopRaw, +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStopResponse>* +- (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStopRequest &request, +- ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(AsyncWaitRaw, +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerWaitResponse>* +- (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerWaitRequest &request, +- ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(PrepareAsyncWaitRaw, +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerWaitResponse>* +- (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerWaitRequest &request, +- ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(AsyncStatusRaw, +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStatusResponse>* +- (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStatusRequest &request, +- ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(PrepareAsyncStatusRaw, +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStatusResponse>* +- (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStatusRequest &request, +- ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(AsyncShutdownRaw, +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerShutdownResponse>* +- (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerShutdownRequest &request, +- ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(PrepareAsyncShutdownRaw, +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerShutdownResponse>* +- (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerShutdownRequest &request, +- ::grpc::CompletionQueue* cq)); +-}; +- +-class DummyControllerStub: public containerd::services::sandbox::v1::Controller::StubInterface { +-public: +- DummyControllerStub() = default; +- ~DummyControllerStub() = default; +- ::grpc::Status Create(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerCreateRequest &request, +- ::containerd::services::sandbox::v1::ControllerCreateResponse* response) override; +- ::grpc::Status Start(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerStartRequest &request, +- ::containerd::services::sandbox::v1::ControllerStartResponse* response) override; +- ::grpc::Status Platform(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerPlatformRequest &request, +- ::containerd::services::sandbox::v1::ControllerPlatformResponse* response) override; +- ::grpc::Status Prepare(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::PrepareRequest &request, +- ::containerd::services::sandbox::v1::PrepareResponse* response) override; +- ::grpc::Status Purge(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PurgeRequest &request, +- ::containerd::services::sandbox::v1::PurgeResponse* response) override; +- ::grpc::Status UpdateResources(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::UpdateResourcesRequest &request, +- ::containerd::services::sandbox::v1::UpdateResourcesResponse* response) override; +- ::grpc::Status Stop(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerStopRequest &request, +- ::containerd::services::sandbox::v1::ControllerStopResponse* response) override; +- ::grpc::Status Wait(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerWaitRequest &request, +- ::containerd::services::sandbox::v1::ControllerWaitResponse* response) override; +- ::grpc::Status Status(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerStatusRequest &request, +- ::containerd::services::sandbox::v1::ControllerStatusResponse* response) override; +- ::grpc::Status Shutdown(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerShutdownRequest &request, +- ::containerd::services::sandbox::v1::ControllerShutdownResponse* response) override; +-private: +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerCreateResponse>* +- AsyncCreateRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerCreateRequest &request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerCreateResponse>* +- PrepareAsyncCreateRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerCreateRequest &request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStartResponse>* +- AsyncStartRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerStartRequest &request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStartResponse>* +- PrepareAsyncStartRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerStartRequest &request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerPlatformResponse>* +- AsyncPlatformRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerPlatformRequest &request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerPlatformResponse>* +- PrepareAsyncPlatformRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerPlatformRequest &request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PrepareResponse>* AsyncPrepareRaw( +- ::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PrepareRequest &request, +- ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PrepareResponse>* +- PrepareAsyncPrepareRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::PrepareRequest &request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PurgeResponse>* AsyncPurgeRaw( +- ::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PurgeRequest &request, +- ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PurgeResponse>* PrepareAsyncPurgeRaw( +- ::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PurgeRequest &request, +- ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::UpdateResourcesResponse>* +- AsyncUpdateResourcesRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::UpdateResourcesRequest &request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::UpdateResourcesResponse>* +- PrepareAsyncUpdateResourcesRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::UpdateResourcesRequest &request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStopResponse>* AsyncStopRaw( +- ::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStopRequest &request, +- ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStopResponse>* +- PrepareAsyncStopRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerStopRequest &request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerWaitResponse>* AsyncWaitRaw( +- ::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerWaitRequest &request, +- ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerWaitResponse>* +- PrepareAsyncWaitRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerWaitRequest &request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStatusResponse>* +- AsyncStatusRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerStatusRequest &request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStatusResponse>* +- PrepareAsyncStatusRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerStatusRequest &request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerShutdownResponse>* +- AsyncShutdownRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerShutdownRequest &request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerShutdownResponse>* +- PrepareAsyncShutdownRaw(::grpc::ClientContext* context, +- const ::containerd::services::sandbox::v1::ControllerShutdownRequest &request, ::grpc::CompletionQueue* cq) override; +-}; +- +-std::unique_ptr<DummyControllerStub> NewDummyControllerStub(); +- +-void MockControllerStub_SetMock(std::shared_ptr<MockControllerStub> stub); +- +- +-#endif // _ISULAD_TEST_MOCKS_CONTROLLER_STUB_MOCK_H +\ No newline at end of file +diff --git a/test/mocks/grpc_async_wait_call_mock.cc b/test/mocks/grpc_async_wait_call_mock.cc +deleted file mode 100644 +index 5eef1794..00000000 +--- a/test/mocks/grpc_async_wait_call_mock.cc ++++ /dev/null +@@ -1,92 +0,0 @@ +-/****************************************************************************** +- * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +- * iSulad licensed under the Mulan PSL v2. +- * You can use this software according to the terms and conditions of the Mulan PSL v2. +- * You may obtain a copy of Mulan PSL v2 at: +- * http://license.coscl.org.cn/MulanPSL2 +- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +- * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +- * PURPOSE. +- * See the Mulan PSL v2 for more details. +- * Author: xuxuepeng +- * Create: 2023-07-31 +- * Description: provide grpc sandboxer async wait call mock +- ******************************************************************************/ +- +-#include "grpc_async_wait_call_mock.h" +- +-static std::shared_ptr<SandboxerAsyncWaitCallMock> g_sandboxer_async_wait_call_mock = NULL; +- +-void MockSandboxerAsyncWaitCall_SetMock(std::shared_ptr<SandboxerAsyncWaitCallMock> mock) +-{ +- g_sandboxer_async_wait_call_mock = mock; +-} +- +-SandboxerAsyncWaitCall::SandboxerAsyncWaitCall(std::shared_ptr<SandboxStatusCallback> cb, +- const std::string &sandboxId, const std::string &sandboxer) +-{ +- m_cb = cb; +- m_sandboxId = sandboxId; +- m_sandboxer = sandboxer; +- m_status = grpc::Status::OK; +- m_retryTimes = 0; +- m_retryCounter = 0; +- m_remove = false; +-} +- +-auto SandboxerAsyncWaitCall::Call(containerd::services::sandbox::v1::Controller::StubInterface &stub, +- grpc::CompletionQueue &cq) -> bool +-{ +- if (g_sandboxer_async_wait_call_mock == NULL) { +- return true; +- } +- return g_sandboxer_async_wait_call_mock->Call(stub, cq); +-} +- +-auto SandboxerAsyncWaitCall::HandleResponse() -> SandboxerAsyncWaitStatus +-{ +- if (g_sandboxer_async_wait_call_mock == NULL) { +- return SANDBOXER_ASYNC_WAIT_STATUS_OK; +- } +- return g_sandboxer_async_wait_call_mock->HandleResponse(); +-} +- +-auto SandboxerAsyncWaitCall::Timeout() -> bool +-{ +- if (g_sandboxer_async_wait_call_mock == NULL) { +- return true; +- } +- return g_sandboxer_async_wait_call_mock->Timeout(); +-} +- +-void SandboxerAsyncWaitCall::SandboxExitCallback(bool statusOK, const ControllerExitInfo &exitInfo) +-{ +- if (g_sandboxer_async_wait_call_mock == NULL) { +- return; +- } +- return g_sandboxer_async_wait_call_mock->SandboxExitCallback(statusOK, exitInfo); +-} +- +-void SandboxerAsyncWaitCall::SandboxPendingCallback() +-{ +- if (g_sandboxer_async_wait_call_mock == NULL) { +- return; +- } +- return g_sandboxer_async_wait_call_mock->SandboxPendingCallback(); +-} +- +-void SandboxerAsyncWaitCall::SandboxReadyCallback() +-{ +- if (g_sandboxer_async_wait_call_mock == NULL) { +- return; +- } +- return g_sandboxer_async_wait_call_mock->SandboxReadyCallback(); +-} +- +-auto SandboxerAsyncWaitCall::GetSandboxId() -> const std::string & +-{ +- if (g_sandboxer_async_wait_call_mock == NULL) { +- return m_sandboxId; +- } +- return g_sandboxer_async_wait_call_mock->GetSandboxId(); +-} +diff --git a/test/mocks/grpc_async_wait_call_mock.h b/test/mocks/grpc_async_wait_call_mock.h +deleted file mode 100644 +index c79f998e..00000000 +--- a/test/mocks/grpc_async_wait_call_mock.h ++++ /dev/null +@@ -1,37 +0,0 @@ +-/****************************************************************************** +- * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +- * iSulad licensed under the Mulan PSL v2. +- * You can use this software according to the terms and conditions of the Mulan PSL v2. +- * You may obtain a copy of Mulan PSL v2 at: +- * http://license.coscl.org.cn/MulanPSL2 +- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +- * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +- * PURPOSE. +- * See the Mulan PSL v2 for more details. +- * Author: xuxuepeng +- * Create: 2023-07-31 +- * Description: provide grpc sandboxer async wait call mock +- ******************************************************************************/ +- +-#ifndef _ISULAD_TEST_MOCKS_GRPC_SANDBOXER_ASYNC_WAIT_CALL_MOCK_H +-#define _ISULAD_TEST_MOCKS_GRPC_SANDBOXER_ASYNC_WAIT_CALL_MOCK_H +- +-#include <gmock/gmock.h> +-#include "grpc_async_wait_call.h" +-using namespace sandbox; +- +-// Mock above class +-class SandboxerAsyncWaitCallMock { +-public: +- MOCK_METHOD2(Call, bool(containerd::services::sandbox::v1::Controller::StubInterface &stub, grpc::CompletionQueue &cq)); +- MOCK_METHOD0(HandleResponse, SandboxerAsyncWaitStatus()); +- MOCK_METHOD0(Timeout, bool()); +- MOCK_METHOD2(SandboxExitCallback, void(bool statusOK, const ControllerExitInfo &exitInfo)); +- MOCK_METHOD0(SandboxPendingCallback, void()); +- MOCK_METHOD0(SandboxReadyCallback, void()); +- MOCK_METHOD0(GetSandboxId, const std::string & ()); +-}; +- +-void MockSandboxerAsyncWaitCall_SetMock(std::shared_ptr<SandboxerAsyncWaitCallMock> mock); +- +-#endif // _ISULAD_TEST_MOCKS_GRPC_SANDBOXER_ASYNC_WAIT_CALL_MOCK_H +\ No newline at end of file +diff --git a/test/mocks/grpc_sandboxer_client_mock.cc b/test/mocks/grpc_sandboxer_client_mock.cc +index 03df9048..1f03a00e 100644 +--- a/test/mocks/grpc_sandboxer_client_mock.cc ++++ b/test/mocks/grpc_sandboxer_client_mock.cc +@@ -14,7 +14,6 @@ + ******************************************************************************/ + + #include "grpc_sandboxer_client_mock.h" +-#include "controller_stub_mock.h" + + static std::shared_ptr<SandboxerClientMock> g_sandboxer_client_mock = NULL; + +@@ -22,24 +21,6 @@ SandboxerClient::SandboxerClient(const std::string &sandboxer, const std::string + { + m_sandboxer = sandboxer; + m_address = address; +- m_channel = grpc::CreateChannel(m_address, grpc::InsecureChannelCredentials()); +- m_stub = NewDummyControllerStub(); +-} +- +-void SandboxerClient::Init(Errors &error) +-{ +- if (g_sandboxer_client_mock == NULL) { +- return; +- } +- g_sandboxer_client_mock->Init(error); +-} +- +-void SandboxerClient::Destroy() +-{ +- if (g_sandboxer_client_mock == NULL) { +- return; +- } +- return g_sandboxer_client_mock->Destroy(); + } + + auto SandboxerClient::Create(const std::string &sandboxId, const ControllerCreateParams ¶ms, Errors &error) -> bool +@@ -67,31 +48,12 @@ auto SandboxerClient::Platform(const std::string &sandboxId, ControllerPlatformI + return g_sandboxer_client_mock->Platform(sandboxId, platformInfo, error); + } + +-auto SandboxerClient::Prepare(const std::string &sandboxId, const ControllerPrepareParams ¶ms, std::string &bundle, +- Errors &error) -> bool ++auto SandboxerClient::Update(sandbox_sandbox *apiSandbox, string_array *fields, Errors &error) -> bool + { + if (g_sandboxer_client_mock == NULL) { + return true; + } +- return g_sandboxer_client_mock->Prepare(sandboxId, params, bundle, error); +-} +- +-auto SandboxerClient::Purge(const std::string &sandboxId, const std::string &containerId, +- const std::string &execId, Errors &error) -> bool +-{ +- if (g_sandboxer_client_mock == NULL) { +- return true; +- } +- return g_sandboxer_client_mock->Purge(sandboxId, containerId, execId, error); +-} +- +-auto SandboxerClient::UpdateResources(const std::string &sandboxId, const ControllerUpdateResourcesParams ¶ms, +- Errors &error) -> bool +-{ +- if (g_sandboxer_client_mock == NULL) { +- return true; +- } +- return g_sandboxer_client_mock->UpdateResources(sandboxId, params, error); ++ return g_sandboxer_client_mock->Update(apiSandbox, fields, error); + } + + auto SandboxerClient::Stop(const std::string &sandboxId, uint32_t timeoutSecs, Errors &error) -> bool +@@ -131,4 +93,4 @@ auto SandboxerClient::Shutdown(const std::string &sandboxId, Errors &error) -> b + void MockSandboxerClient_SetMock(std::shared_ptr<SandboxerClientMock> mock) + { + g_sandboxer_client_mock = mock; +-} +\ No newline at end of file ++} +diff --git a/test/mocks/grpc_sandboxer_client_mock.h b/test/mocks/grpc_sandboxer_client_mock.h +index ac06462a..8d535013 100644 +--- a/test/mocks/grpc_sandboxer_client_mock.h ++++ b/test/mocks/grpc_sandboxer_client_mock.h +@@ -17,23 +17,18 @@ + #define _ISULAD_TEST_MOCKS_GRPC_SANDBOXER_CLIENT_MOCK_H + + #include <gmock/gmock.h> ++#include <isula_sandbox_api.h> ++ + #include "grpc_sandboxer_client.h" + using namespace sandbox; + + class SandboxerClientMock { + public: + SandboxerClientMock() = default; +- MOCK_METHOD1(Init, void(Errors &error)); +- MOCK_METHOD0(Destroy, void()); + MOCK_METHOD3(Create, bool(const std::string &sandboxId, const ControllerCreateParams ¶ms, Errors &error)); + MOCK_METHOD3(Start, bool(const std::string &sandboxId, ControllerSandboxInfo &sandboxInfo, Errors &error)); + MOCK_METHOD3(Platform, bool(const std::string &sandboxId, ControllerPlatformInfo &platformInfo, Errors &error)); +- MOCK_METHOD4(Prepare, bool(const std::string &sandboxId, const ControllerPrepareParams ¶ms, std::string &bundle, +- Errors &error)); +- MOCK_METHOD4(Purge, bool(const std::string &sandboxId, const std::string &containerId, const std::string &execId, +- Errors &error)); +- MOCK_METHOD3(UpdateResources, bool(const std::string &sandboxId, const ControllerUpdateResourcesParams ¶ms, +- Errors &error)); ++ MOCK_METHOD3(Update, bool(sandbox_sandbox *apiSandbox, string_array *fields, Errors &error)); + MOCK_METHOD3(Stop, bool(const std::string &sandboxId, uint32_t timeoutSecs, Errors &error)); + MOCK_METHOD3(Wait, bool(std::shared_ptr<SandboxStatusCallback> cb, const std::string &sandboxId, Errors &error)); + MOCK_METHOD4(Status, bool(const std::string &sandboxId, bool verbose, ControllerSandboxStatus &sandboxStatus, +diff --git a/test/mocks/grpc_sandboxer_monitor_mock.cc b/test/mocks/grpc_sandboxer_monitor_mock.cc +deleted file mode 100644 +index b3ddda87..00000000 +--- a/test/mocks/grpc_sandboxer_monitor_mock.cc ++++ /dev/null +@@ -1,51 +0,0 @@ +-/****************************************************************************** +- * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +- * iSulad licensed under the Mulan PSL v2. +- * You can use this software according to the terms and conditions of the Mulan PSL v2. +- * You may obtain a copy of Mulan PSL v2 at: +- * http://license.coscl.org.cn/MulanPSL2 +- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +- * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +- * PURPOSE. +- * See the Mulan PSL v2 for more details. +- * Author: xuxuepeng +- * Create: 2023-07-31 +- * Description: provide grpc sandboxer client monitor mock +- ******************************************************************************/ +- +-#include "grpc_sandboxer_monitor_mock.h" +- +-static std::shared_ptr<SandboxerClientMonitorMock> g_sandboxer_client_monitor_mock = NULL; +- +- +-SandboxerClientMonitor::SandboxerClientMonitor(std::shared_ptr<grpc::Channel> channel, const std::string &sandboxer): +- m_channel(channel), m_sandboxer(sandboxer), m_teardown(false) {} +- +-void SandboxerClientMonitor::Start() +-{ +- if (g_sandboxer_client_monitor_mock == NULL) { +- return; +- } +- return g_sandboxer_client_monitor_mock->Start(); +-} +- +-void SandboxerClientMonitor::Stop() +-{ +- if (g_sandboxer_client_monitor_mock == NULL) { +- return; +- } +- return g_sandboxer_client_monitor_mock->Stop(); +-} +- +-bool SandboxerClientMonitor::Monitor(SandboxerAsyncWaitCall *call) +-{ +- if (g_sandboxer_client_monitor_mock == NULL) { +- return true; +- } +- return g_sandboxer_client_monitor_mock->Monitor(call); +-} +- +-void MockSandboxerMonitor_SetMock(std::shared_ptr<SandboxerClientMonitorMock> mock) +-{ +- g_sandboxer_client_monitor_mock = mock; +-} +diff --git a/test/mocks/rust_sandbox_api_mock.cc b/test/mocks/rust_sandbox_api_mock.cc +new file mode 100644 +index 00000000..b595e4b8 +--- /dev/null ++++ b/test/mocks/rust_sandbox_api_mock.cc +@@ -0,0 +1,103 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-11-22 ++ * Description: provide rust sandbox api mock ++ ******************************************************************************/ ++ ++#include "rust_sandbox_api_mock.h" ++ ++static std::shared_ptr<RustSandboxApiMock> g_rust_sandbox_api_mock = NULL; ++ ++void RustSandboxApiMock_SetMock(std::shared_ptr<RustSandboxApiMock> mock) ++{ ++ g_rust_sandbox_api_mock = mock; ++} ++ ++ControllerHandle_t sandbox_api_build_controller(const char *sandboxer, const char *address) ++{ ++ if (g_rust_sandbox_api_mock != nullptr) { ++ return g_rust_sandbox_api_mock->sandbox_api_build_controller(sandboxer, address); ++ } ++ return nullptr; ++} ++ ++int sandbox_api_create(ControllerHandle_t chandle, const sandbox_create_request *request, sandbox_create_response *response) ++{ ++ if (g_rust_sandbox_api_mock != nullptr) { ++ return g_rust_sandbox_api_mock->sandbox_api_create(chandle, request, response); ++ } ++ return 0; ++} ++ ++int sandbox_api_start(ControllerHandle_t chandle, const sandbox_start_request *request, sandbox_start_response *response) ++{ ++ if (g_rust_sandbox_api_mock != nullptr) { ++ return g_rust_sandbox_api_mock->sandbox_api_start(chandle, request, response); ++ } ++ return 0; ++} ++ ++int sandbox_api_platform(ControllerHandle_t chandle, const sandbox_platform_request *request, sandbox_platform_response *response) ++{ ++ if (g_rust_sandbox_api_mock != nullptr) { ++ return g_rust_sandbox_api_mock->sandbox_api_platform(chandle, request, response); ++ } ++ return 0; ++} ++ ++int sandbox_api_stop(ControllerHandle_t chandle, const sandbox_stop_request *request) ++{ ++ if (g_rust_sandbox_api_mock != nullptr) { ++ return g_rust_sandbox_api_mock->sandbox_api_stop(chandle, request); ++ } ++ return 0; ++} ++ ++int sandbox_api_wait(ControllerHandle_t chandle, const sandbox_wait_request *request, sandbox_api_wait_callback callback) ++{ ++ if (g_rust_sandbox_api_mock != nullptr) { ++ return g_rust_sandbox_api_mock->sandbox_api_wait(chandle, request, callback); ++ } ++ return 0; ++} ++ ++int sandbox_api_status(ControllerHandle_t chandle, const sandbox_status_request *request, sandbox_status_response *response) ++{ ++ if (g_rust_sandbox_api_mock != nullptr) { ++ return g_rust_sandbox_api_mock->sandbox_api_status(chandle, request, response); ++ } ++ return 0; ++} ++ ++int sandbox_api_shutdown(ControllerHandle_t chandle, const sandbox_shutdown_request *request) ++{ ++ if (g_rust_sandbox_api_mock != nullptr) { ++ return g_rust_sandbox_api_mock->sandbox_api_shutdown(chandle, request); ++ } ++ return 0; ++} ++ ++int sandbox_api_metrics(ControllerHandle_t chandle, const sandbox_metrics_request *request, sandbox_metrics_response *response) ++{ ++ if (g_rust_sandbox_api_mock != nullptr) { ++ return g_rust_sandbox_api_mock->sandbox_api_metrics(chandle, request, response); ++ } ++ return 0; ++} ++ ++int sandbox_api_update(ControllerHandle_t chandle, const sandbox_update_request *request) ++{ ++ if (g_rust_sandbox_api_mock != nullptr) { ++ return g_rust_sandbox_api_mock->sandbox_api_update(chandle, request); ++ } ++ return 0; ++} +diff --git a/test/mocks/rust_sandbox_api_mock.h b/test/mocks/rust_sandbox_api_mock.h +new file mode 100644 +index 00000000..134d71e9 +--- /dev/null ++++ b/test/mocks/rust_sandbox_api_mock.h +@@ -0,0 +1,39 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-11-22 ++ * Description: provide rust sandbox api mock ++ ******************************************************************************/ ++ ++#ifndef _ISULAD_TEST_MOCKS_RUST_SANDBOX_API_MOCK_H ++#define _ISULAD_TEST_MOCKS_RUST_SANDBOX_API_MOCK_H ++ ++#include <gmock/gmock.h> ++#include <isula_sandbox_api.h> ++ ++class RustSandboxApiMock { ++public: ++ RustSandboxApiMock() = default; ++ MOCK_METHOD2(sandbox_api_build_controller, ControllerHandle_t(const char *sandboxer, const char *address)); ++ MOCK_METHOD3(sandbox_api_create, int(ControllerHandle_t chandle, const sandbox_create_request *request, sandbox_create_response *response)); ++ MOCK_METHOD3(sandbox_api_start, int(ControllerHandle_t chandle, const sandbox_start_request *request, sandbox_start_response *response)); ++ MOCK_METHOD2(sandbox_api_stop, int(ControllerHandle_t chandle, const sandbox_stop_request *request)); ++ MOCK_METHOD3(sandbox_api_wait, int(ControllerHandle_t chandle, const sandbox_wait_request *request, sandbox_api_wait_callback callback)); ++ MOCK_METHOD3(sandbox_api_status, int(ControllerHandle_t chandle, const sandbox_status_request *request, sandbox_status_response *response)); ++ MOCK_METHOD2(sandbox_api_shutdown, int(ControllerHandle_t chandle, const sandbox_shutdown_request *request)); ++ MOCK_METHOD3(sandbox_api_metrics, int(ControllerHandle_t chandle, const sandbox_metrics_request *request, sandbox_metrics_response *response)); ++ MOCK_METHOD3(sandbox_api_platform, int(ControllerHandle_t chandle, const sandbox_platform_request *request, sandbox_platform_response *response)); ++ MOCK_METHOD2(sandbox_api_update, int(ControllerHandle_t chandle, const sandbox_update_request *request)); ++}; ++ ++void RustSandboxApiMock_SetMock(std::shared_ptr<RustSandboxApiMock> mock); ++ ++#endif // _ISULAD_TEST_MOCKS_RUST_SANDBOX_API_MOCK_H +diff --git a/test/mocks/sandbox_manager_mock.cc b/test/mocks/sandbox_manager_mock.cc +new file mode 100644 +index 00000000..c3c32e52 +--- /dev/null ++++ b/test/mocks/sandbox_manager_mock.cc +@@ -0,0 +1,42 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2023-10-20 ++ * Description: provide sandboxer controller mock ++ ******************************************************************************/ ++ ++#include "sandbox_manager_mock.h" ++ ++namespace sandbox { ++static std::shared_ptr<MockSandboxManager> g_sandbox_manager_mock = nullptr; ++ ++void MockSandboxManager_SetMock(std::shared_ptr<MockSandboxManager> mock) ++{ ++ g_sandbox_manager_mock = mock; ++} ++ ++SandboxManager *SandboxManager::GetInstance() noexcept ++{ ++ if (g_sandbox_manager_mock != nullptr) { ++ return g_sandbox_manager_mock->GetInstance(); ++ } ++ return nullptr; ++} ++ ++std::shared_ptr<Sandbox> SandboxManager::GetSandbox(const std::string &idOrName) ++{ ++ if (g_sandbox_manager_mock != nullptr) { ++ return g_sandbox_manager_mock->GetSandbox(idOrName); ++ } ++ return nullptr; ++} ++ ++} +diff --git a/test/mocks/grpc_sandboxer_monitor_mock.h b/test/mocks/sandbox_manager_mock.h +similarity index 55% +rename from test/mocks/grpc_sandboxer_monitor_mock.h +rename to test/mocks/sandbox_manager_mock.h +index 0ab21e00..1378d5b7 100644 +--- a/test/mocks/grpc_sandboxer_monitor_mock.h ++++ b/test/mocks/sandbox_manager_mock.h +@@ -8,26 +8,35 @@ + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. +- * Author: xuxuepeng +- * Create: 2023-07-31 +- * Description: provide grpc sandboxer client monitor mock ++ * Author: jikai ++ * Create: 2023-10-20 ++ * Description: provide sandbox manager mock + ******************************************************************************/ + +-#ifndef _ISULAD_TEST_MOCKS_GRPC_SANDBOXER_MONITOR_MOCK_H +-#define _ISULAD_TEST_MOCKS_GRPC_SANDBOXER_MONITOR_MOCK_H ++#ifndef _ISULAD_TEST_MOCKS_SANDBOX_MANAGER_MOCK_H ++#define _ISULAD_TEST_MOCKS_SANDBOX_MANAGER_MOCK_H + + #include <gmock/gmock.h> +-#include "grpc_sandboxer_monitor.h" ++#include <memory> + +-using namespace sandbox; ++#include "sandbox.h" ++#include "sandbox_manager.h" + +-class SandboxerClientMonitorMock { ++namespace sandbox { ++ ++class MockSandboxManager { + public: +- MOCK_METHOD1(Monitor, bool(SandboxerAsyncWaitCall *call)); +- MOCK_METHOD0(Start, void()); +- MOCK_METHOD0(Stop, void()); ++ MockSandboxManager() = default; ++ virtual ~MockSandboxManager() = default; ++ ++ MOCK_METHOD0(GetInstance, SandboxManager *()); ++ ++ MOCK_METHOD1(GetSandbox, std::shared_ptr<Sandbox>(const std::string &idOrName)); ++ + }; + +-void MockSandboxerMonitor_SetMock(std::shared_ptr<SandboxerClientMonitorMock> mock); ++void MockSandboxManager_SetMock(std::shared_ptr<MockSandboxManager> mock); ++ ++} + +-#endif // _ISULAD_TEST_MOCKS_GRPC_SANDBOXER_MONITOR_MOCK_H ++#endif +\ No newline at end of file +diff --git a/test/mocks/sandboxer_controller_mock.cc b/test/mocks/sandboxer_controller_mock.cc +new file mode 100644 +index 00000000..4f7258d5 +--- /dev/null ++++ b/test/mocks/sandboxer_controller_mock.cc +@@ -0,0 +1,116 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2023-10-20 ++ * Description: provide sandboxer controller mock ++ ******************************************************************************/ ++ ++#include "sandboxer_controller_mock.h" ++ ++namespace sandbox { ++static std::shared_ptr<MockSandboxerController> g_sandboxer_controller_mock = nullptr; ++ ++SandboxerController::SandboxerController(const std::string &sandboxer, const std::string &address) ++{ ++} ++ ++SandboxerController::~SandboxerController() ++{ ++} ++ ++void MockSandboxerController_SetMock(std::shared_ptr<MockSandboxerController> mock) ++{ ++ g_sandboxer_controller_mock = mock; ++} ++ ++bool SandboxerController::Init(Errors &error) ++{ ++ if (g_sandboxer_controller_mock != nullptr) { ++ return g_sandboxer_controller_mock->Init(error); ++ } ++ return true; ++} ++ ++bool SandboxerController::Create(const std::string &sandboxId, ++ const ControllerCreateParams ¶ms, ++ Errors &error) ++{ ++ if (g_sandboxer_controller_mock != nullptr) { ++ return g_sandboxer_controller_mock->Create(sandboxId, params, error); ++ } ++ return true; ++} ++ ++std::unique_ptr<ControllerSandboxInfo> SandboxerController::Start(const std::string &sandboxId, Errors &error) ++{ ++ if (g_sandboxer_controller_mock != nullptr) { ++ return g_sandboxer_controller_mock->Start(sandboxId, error); ++ } ++ return nullptr; ++} ++ ++std::unique_ptr<ControllerPlatformInfo> SandboxerController::Platform(const std::string &sandboxId, Errors &error) ++{ ++ if (g_sandboxer_controller_mock != nullptr) { ++ return g_sandboxer_controller_mock->Platform(sandboxId, error); ++ } ++ return nullptr; ++} ++ ++bool SandboxerController::Update(sandbox_sandbox *apiSandbox, ++ string_array *fields, Errors &error) ++{ ++ return g_sandboxer_controller_mock->Update(apiSandbox, fields, error); ++} ++ ++bool SandboxerController::Stop(const std::string &sandboxId, uint32_t timeoutSecs, Errors &error) ++{ ++ if (g_sandboxer_controller_mock != nullptr) { ++ return g_sandboxer_controller_mock->Stop(sandboxId, timeoutSecs, error); ++ } ++ return true; ++} ++ ++bool SandboxerController::Wait(std::shared_ptr<SandboxStatusCallback> cb, const std::string &sandboxId, Errors &error) ++{ ++ if (g_sandboxer_controller_mock != nullptr) { ++ return g_sandboxer_controller_mock->Wait(cb, sandboxId, error); ++ } ++ return true; ++} ++ ++std::unique_ptr<ControllerSandboxStatus> SandboxerController::Status(const std::string &sandboxId, bool verbose, ++ Errors &error) ++{ ++ if (g_sandboxer_controller_mock != nullptr) { ++ return g_sandboxer_controller_mock->Status(sandboxId, verbose, error); ++ } ++ return nullptr; ++} ++ ++bool SandboxerController::Shutdown(const std::string &sandboxId, Errors &error) ++{ ++ if (g_sandboxer_controller_mock != nullptr) { ++ return g_sandboxer_controller_mock->Shutdown(sandboxId, error); ++ } ++ return true; ++} ++ ++bool SandboxerController::UpdateNetworkSettings(const std::string &sandboxId, const std::string &networkSettings, ++ Errors &error) ++{ ++ if (g_sandboxer_controller_mock != nullptr) { ++ return g_sandboxer_controller_mock->UpdateNetworkSettings(sandboxId, networkSettings, error); ++ } ++ return true; ++} ++ ++} +diff --git a/test/mocks/sandboxer_controller_mock.h b/test/mocks/sandboxer_controller_mock.h +new file mode 100644 +index 00000000..be0f4016 +--- /dev/null ++++ b/test/mocks/sandboxer_controller_mock.h +@@ -0,0 +1,52 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2023-10-20 ++ * Description: provide sandboxer controller mock ++ ******************************************************************************/ ++ ++#ifndef _ISULAD_TEST_MOCKS_SANDBOXER_CONTROLLER_MOCK_H ++#define _ISULAD_TEST_MOCKS_SANDBOXER_CONTROLLER_MOCK_H ++ ++#include <gmock/gmock.h> ++#include <memory> ++ ++#include "sandboxer_controller.h" ++ ++namespace sandbox { ++ ++class MockSandboxerController { ++public: ++ MockSandboxerController() = default; ++ virtual ~MockSandboxerController() = default; ++ ++ MOCK_METHOD1(Init, bool(Errors &error)); ++ MOCK_METHOD3(Create, bool(const std::string &sandboxId, ++ const ControllerCreateParams ¶ms, ++ Errors &error)); ++ MOCK_METHOD2(Start, std::unique_ptr<ControllerSandboxInfo>(const std::string &sandboxId, Errors &error)); ++ MOCK_METHOD2(Platform, std::unique_ptr<ControllerPlatformInfo>(const std::string &sandboxId, Errors &error)); ++ MOCK_METHOD3(Update, bool(sandbox_sandbox *apiSandbox, ++ string_array *fields, Errors &error)); ++ MOCK_METHOD3(Stop, bool(const std::string &sandboxId, uint32_t timeoutSecs, Errors &error)); ++ MOCK_METHOD3(Wait, bool(std::shared_ptr<SandboxStatusCallback> cb, const std::string &sandboxId, Errors &error)); ++ MOCK_METHOD3(Status, std::unique_ptr<ControllerSandboxStatus>(const std::string &sandboxId, bool verbose, ++ Errors &error)); ++ MOCK_METHOD2(Shutdown, bool(const std::string &sandboxId, Errors &error)); ++ MOCK_METHOD3(UpdateNetworkSettings, bool(const std::string &sandboxId, const std::string &networkSettings, ++ Errors &error)); ++}; ++ ++void MockSandboxerController_SetMock(std::shared_ptr<MockSandboxerController> mock); ++ ++} ++ ++#endif +\ No newline at end of file +diff --git a/test/mocks/sandboxer_sandbox_mock.cc b/test/mocks/sandboxer_sandbox_mock.cc +new file mode 100644 +index 00000000..cce58842 +--- /dev/null ++++ b/test/mocks/sandboxer_sandbox_mock.cc +@@ -0,0 +1,72 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-11-21 ++ * Description: provide sandbox mock ++ ******************************************************************************/ ++ ++#include <gmock/gmock.h> ++#include "sandboxer_sandbox_mock.h" ++ ++namespace sandbox { ++MockSandboxerSandbox *g_sandboxer_sandbox_mock = nullptr; ++ ++SandboxerSandbox::SandboxerSandbox(const std::string id, const std::string &rootdir, const std::string &statedir, const std::string name, ++ const RuntimeInfo info, std::string netMode, std::string netNsPath, const runtime::v1::PodSandboxConfig sandboxConfig, ++ std::string image):Sandbox(id, rootdir, statedir, name, info, netMode, ++ netNsPath, sandboxConfig, image) ++{ ++} ++ ++void MockSandboxerSandbox_SetMock(MockSandboxerSandbox *mock) ++{ ++ g_sandboxer_sandbox_mock = mock; ++} ++ ++void SandboxerSandbox::LoadSandboxTasks() {} ++ ++auto SandboxerSandbox::SaveSandboxTasks() -> bool ++{ ++ if (g_sandboxer_sandbox_mock != nullptr) { ++ return g_sandboxer_sandbox_mock->SaveSandboxTasks(); ++ } ++ return true; ++} ++ ++auto SandboxerSandbox::AddSandboxTasks(sandbox_task *task) -> bool ++{ ++ if (g_sandboxer_sandbox_mock != nullptr) { ++ return g_sandboxer_sandbox_mock->AddSandboxTasks(task); ++ } ++ return true; ++} ++ ++auto SandboxerSandbox::GetAnySandboxTasks() -> std::string ++{ ++ if (g_sandboxer_sandbox_mock != nullptr) { ++ return g_sandboxer_sandbox_mock->GetAnySandboxTasks(); ++ } ++ return std::string("Nothing for sandboxer."); ++} ++ ++void SandboxerSandbox::DeleteSandboxTasks(const char *containerId) {} ++ ++auto SandboxerSandbox::AddSandboxTasksProcess(const char *containerId, sandbox_process *processes) -> bool ++{ ++ if (g_sandboxer_sandbox_mock != nullptr) { ++ return g_sandboxer_sandbox_mock->AddSandboxTasksProcess(containerId, processes); ++ } ++ return true; ++} ++ ++void SandboxerSandbox::DeleteSandboxTasksProcess(const char *containerId, const char *execId) {} ++ ++} +\ No newline at end of file +diff --git a/test/mocks/sandboxer_sandbox_mock.h b/test/mocks/sandboxer_sandbox_mock.h +new file mode 100644 +index 00000000..4f76e5fc +--- /dev/null ++++ b/test/mocks/sandboxer_sandbox_mock.h +@@ -0,0 +1,43 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-11-21 ++ * Description: provide sandbox mock ++ ******************************************************************************/ ++ ++#ifndef _ISULAD_TEST_MOCKS_SANDBOXER_SANDBOX_MOCK_H ++#define _ISULAD_TEST_MOCKS_SANDBOXER_SANDBOX_MOCK_H ++ ++#include <gmock/gmock.h> ++#include "sandbox_mock.h" ++#include "sandboxer_sandbox.h" ++ ++namespace sandbox { ++ ++class MockSandboxerSandbox : public MockSandbox { ++public: ++ MockSandboxerSandbox() = default; ++ virtual ~MockSandboxerSandbox() = default; ++ ++ MOCK_METHOD0(LoadSandboxTasks, void()); ++ MOCK_METHOD0(SaveSandboxTasks, bool()); ++ MOCK_METHOD1(AddSandboxTasks, bool(sandbox_task *task)); ++ MOCK_METHOD0(GetAnySandboxTasks, std::string()); ++ MOCK_METHOD1(DeleteSandboxTasks, void(const char *containerId)); ++ MOCK_METHOD2(AddSandboxTasksProcess, bool(const char *containerId, sandbox_process *processes)); ++ MOCK_METHOD2(DeleteSandboxTasksProcess, void(const char *containerId, const char *execId)); ++}; ++ ++void MockSandboxerSandbox_SetMock(MockSandboxerSandbox *mock); ++ ++} ++ ++#endif +diff --git a/test/mocks/shim_controller_mock.cc b/test/mocks/shim_controller_mock.cc +index e0ffc563..d9067e22 100644 +--- a/test/mocks/shim_controller_mock.cc ++++ b/test/mocks/shim_controller_mock.cc +@@ -39,13 +39,6 @@ bool ShimController::Init(Errors &error) + return true; + } + +-void ShimController::Destroy() +-{ +- if (g_shim_controller_mock != nullptr) { +- g_shim_controller_mock->Destroy(); +- } +-} +- + bool ShimController::Create(const std::string &sandboxId, + const ControllerCreateParams ¶ms, + Errors &error) +@@ -72,33 +65,10 @@ std::unique_ptr<ControllerPlatformInfo> ShimController::Platform(const std::stri + return nullptr; + } + +-std::string ShimController::Prepare(const std::string &sandboxId, +- const ControllerPrepareParams ¶ms, +- Errors &error) +-{ +- if (g_shim_controller_mock != nullptr) { +- return g_shim_controller_mock->Prepare(sandboxId, params, error); +- } +- return ""; +-} +- +-bool ShimController::Purge(const std::string &sandboxId, const std::string &containerId, +- const std::string &execId, Errors &error) ++bool ShimController::Update(sandbox_sandbox *apiSandbox, ++ string_array *fields, Errors &error) + { +- if (g_shim_controller_mock != nullptr) { +- return g_shim_controller_mock->Purge(sandboxId, containerId, execId, error); +- } +- return true; +-} +- +-bool ShimController::UpdateResources(const std::string &sandboxId, +- const ControllerUpdateResourcesParams ¶ms, +- Errors &error) +-{ +- if (g_shim_controller_mock != nullptr) { +- return g_shim_controller_mock->UpdateResources(sandboxId, params, error); +- } +- return true; ++ return g_shim_controller_mock->Update(apiSandbox, fields, error); + } + + bool ShimController::Stop(const std::string &sandboxId, uint32_t timeoutSecs, Errors &error) +diff --git a/test/mocks/shim_controller_mock.h b/test/mocks/shim_controller_mock.h +index 6d0de591..d3a1d0a5 100644 +--- a/test/mocks/shim_controller_mock.h ++++ b/test/mocks/shim_controller_mock.h +@@ -29,20 +29,13 @@ public: + virtual ~MockShimController() = default; + + MOCK_METHOD1(Init, bool(Errors &error)); +- MOCK_METHOD0(Destroy, void()); + MOCK_METHOD3(Create, bool(const std::string &sandboxId, + const ControllerCreateParams ¶ms, + Errors &error)); + MOCK_METHOD2(Start, std::unique_ptr<ControllerSandboxInfo>(const std::string &sandboxId, Errors &error)); + MOCK_METHOD2(Platform, std::unique_ptr<ControllerPlatformInfo>(const std::string &sandboxId, Errors &error)); +- MOCK_METHOD3(Prepare, std::string(const std::string &sandboxId, +- const ControllerPrepareParams ¶ms, +- Errors &error)); +- MOCK_METHOD4(Purge, bool(const std::string &sandboxId, const std::string &containerId, +- const std::string &execId, Errors &error)); +- MOCK_METHOD3(UpdateResources, bool(const std::string &sandboxId, +- const ControllerUpdateResourcesParams ¶ms, +- Errors &error)); ++ MOCK_METHOD3(Update, bool(sandbox_sandbox *apiSandbox, ++ string_array *fields, Errors &error)); + MOCK_METHOD3(Stop, bool(const std::string &sandboxId, uint32_t timeoutSecs, Errors &error)); + MOCK_METHOD3(Wait, bool(std::shared_ptr<SandboxStatusCallback> cb, const std::string &sandboxId, Errors &error)); + MOCK_METHOD3(Status, std::unique_ptr<ControllerSandboxStatus>(const std::string &sandboxId, bool verbose, +diff --git a/test/mocks/shim_sandbox_mock.cc b/test/mocks/shim_sandbox_mock.cc +new file mode 100644 +index 00000000..ccefb424 +--- /dev/null ++++ b/test/mocks/shim_sandbox_mock.cc +@@ -0,0 +1,72 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-11-21 ++ * Description: provide sandbox mock ++ ******************************************************************************/ ++ ++#include <gmock/gmock.h> ++#include "shim_sandbox_mock.h" ++ ++namespace sandbox { ++MockShimSandbox *g_shim_sandbox_mock = nullptr; ++ ++ShimSandbox::ShimSandbox(const std::string id, const std::string &rootdir, const std::string &statedir, const std::string name, ++ const RuntimeInfo info, std::string netMode, std::string netNsPath, const runtime::v1::PodSandboxConfig sandboxConfig, ++ std::string image):Sandbox(id, rootdir, statedir, name, info, netMode, ++ netNsPath, sandboxConfig, image) ++{ ++} ++ ++void MockShimSandbox_SetMock(MockShimSandbox *mock) ++{ ++ g_shim_sandbox_mock = mock; ++} ++ ++void ShimSandbox::LoadSandboxTasks() {} ++ ++auto ShimSandbox::SaveSandboxTasks() -> bool ++{ ++ if (g_shim_sandbox_mock != nullptr) { ++ return g_shim_sandbox_mock->SaveSandboxTasks(); ++ } ++ return true; ++} ++ ++auto ShimSandbox::AddSandboxTasks(sandbox_task *task) -> bool ++{ ++ if (g_shim_sandbox_mock != nullptr) { ++ return g_shim_sandbox_mock->AddSandboxTasks(task); ++ } ++ return true; ++} ++ ++auto ShimSandbox::GetAnySandboxTasks() -> std::string ++{ ++ if (g_shim_sandbox_mock != nullptr) { ++ return g_shim_sandbox_mock->GetAnySandboxTasks(); ++ } ++ return std::string("Nothing for shim."); ++} ++ ++void ShimSandbox::DeleteSandboxTasks(const char *containerId) {} ++ ++auto ShimSandbox::AddSandboxTasksProcess(const char *containerId, sandbox_process *processes) -> bool ++{ ++ if (g_shim_sandbox_mock != nullptr) { ++ return g_shim_sandbox_mock->AddSandboxTasksProcess(containerId, processes); ++ } ++ return true; ++} ++ ++void ShimSandbox::DeleteSandboxTasksProcess(const char *containerId, const char *execId) {} ++ ++} +\ No newline at end of file +diff --git a/test/mocks/shim_sandbox_mock.h b/test/mocks/shim_sandbox_mock.h +new file mode 100644 +index 00000000..1b16a4cc +--- /dev/null ++++ b/test/mocks/shim_sandbox_mock.h +@@ -0,0 +1,43 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-11-21 ++ * Description: provide sandbox mock ++ ******************************************************************************/ ++ ++#ifndef _ISULAD_TEST_MOCKS_SHIM_SANDBOX_MOCK_H ++#define _ISULAD_TEST_MOCKS_SHIM_SANDBOX_MOCK_H ++ ++#include <gmock/gmock.h> ++#include "sandbox_mock.h" ++#include "shim_sandbox.h" ++ ++namespace sandbox { ++ ++class MockShimSandbox : public MockSandbox { ++public: ++ MockShimSandbox() = default; ++ virtual ~MockShimSandbox() = default; ++ ++ MOCK_METHOD0(LoadSandboxTasks, void()); ++ MOCK_METHOD0(SaveSandboxTasks, bool()); ++ MOCK_METHOD1(AddSandboxTasks, bool(sandbox_task *task)); ++ MOCK_METHOD0(GetAnySandboxTasks, std::string()); ++ MOCK_METHOD1(DeleteSandboxTasks, void(const char *containerId)); ++ MOCK_METHOD2(AddSandboxTasksProcess, bool(const char *containerId, sandbox_process *processes)); ++ MOCK_METHOD2(DeleteSandboxTasksProcess, void(const char *containerId, const char *execId)); ++}; ++ ++void MockShimSandbox_SetMock(MockShimSandbox *mock); ++ ++} ++ ++#endif +diff --git a/test/sandbox/controller/CMakeLists.txt b/test/sandbox/controller/CMakeLists.txt +index 0ba23735..d201872b 100644 +--- a/test/sandbox/controller/CMakeLists.txt ++++ b/test/sandbox/controller/CMakeLists.txt +@@ -1,7 +1,5 @@ + project(iSulad_UT) + +-aux_source_directory(${CMAKE_BINARY_DIR}/grpc/src/api/services/sandbox/sandbox/types grpc_sandbox_type_srcs) +-list(APPEND grpc_sandbox_type_srcs ${CMAKE_BINARY_DIR}/grpc/src/api/services/sandbox/sandbox.pb.cc) + list(APPEND grpc_sandbox_type_srcs ${CMAKE_BINARY_DIR}/grpc/src/api/services/cri/v1/api_v1.pb.cc) + list(APPEND grpc_sandbox_type_srcs ${CMAKE_BINARY_DIR}/grpc/src/api/services/cri/gogo.pb.cc) + add_subdirectory(sandboxer) +diff --git a/test/sandbox/controller/controller_common.cc b/test/sandbox/controller/controller_common.cc +index 5f870c34..8621dfd9 100644 +--- a/test/sandbox/controller/controller_common.cc ++++ b/test/sandbox/controller/controller_common.cc +@@ -14,6 +14,7 @@ + ******************************************************************************/ + + #include "controller_common.h" ++#include "utils.h" + + std::unique_ptr<sandbox::ControllerMountInfo> CreateTestMountInfo() + { +@@ -43,24 +44,40 @@ std::unique_ptr<sandbox::ControllerStreamInfo> CreateTestStreamInfo() + return streamInfo; + } + +-std::unique_ptr<sandbox::ControllerPrepareParams> CreateTestPrepareParams() ++std::unique_ptr<CStructWrapper<sandbox_sandbox>> CreateTestUpdateApiSandbox() + { +- std::unique_ptr<sandbox::ControllerPrepareParams> params(new sandbox::ControllerPrepareParams()); +- params->containerId = DUMMY_CONTAINER_ID; +- params->execId = DUMMY_EXEC_ID; +- params->spec = std::unique_ptr<std::string>(new std::string("{spec: test}")); +- params->rootfs.push_back(std::move(CreateTestMountInfo())); +- params->rootfs.push_back(std::move(CreateTestMountInfo())); +- params->streamInfo = CreateTestStreamInfo(); +- return params; ++ sandbox_sandbox *apiSandbox = nullptr; ++ ++ auto apiSandbox_wrapper = makeUniquePtrCStructWrapper<sandbox_sandbox>(free_sandbox_sandbox); ++ if (apiSandbox_wrapper == nullptr) { ++ return nullptr; ++ } ++ apiSandbox = apiSandbox_wrapper->get(); ++ ++ apiSandbox->sandbox_id = util_strdup_s(DUMMY_SANDBOX_ID.c_str()); ++ apiSandbox->sandboxer = util_strdup_s(DUMMY_SANDBOXER.c_str()); ++ ++ return apiSandbox_wrapper; + } + +-std::unique_ptr<sandbox::ControllerUpdateResourcesParams> CreateTestUpdateResourcesParams( +- google::protobuf::Map<std::string, std::string> &annotations) ++std::unique_ptr<CStructWrapper<string_array>> CreateTestFields() + { +- std::unique_ptr<std::string> resources(new std::string("{cpu: 12}")); +- std::unique_ptr<sandbox::ControllerUpdateResourcesParams> params( +- new sandbox::ControllerUpdateResourcesParams{DUMMY_SANDBOX_ID, std::move(resources), annotations} +- ); +- return params; +-} ++ size_t fields_len = 1; ++ string_array *fields = nullptr; ++ ++ auto fields_wrapper = makeUniquePtrCStructWrapper<string_array>(util_free_string_array); ++ if (fields_wrapper == nullptr) { ++ return nullptr; ++ } ++ fields = fields_wrapper->get(); ++ ++ fields = util_string_array_new(fields_len); ++ if (fields == nullptr) { ++ return nullptr; ++ } ++ if (util_append_string_array(fields, DUMMY_SANDBOX_EXTENSIONS_TASKS.c_str())) { ++ return nullptr; ++ } ++ ++ return fields_wrapper; ++} +\ No newline at end of file +diff --git a/test/sandbox/controller/controller_common.h b/test/sandbox/controller/controller_common.h +index c01ae83c..b41f24e2 100644 +--- a/test/sandbox/controller/controller_common.h ++++ b/test/sandbox/controller/controller_common.h +@@ -16,6 +16,8 @@ + #ifndef _ISULAD_TEST_SANDBOX_CONTROLLER_CONTROLLER_COMMON_H + #define _ISULAD_TEST_SANDBOX_CONTROLLER_CONTROLLER_COMMON_H + #include "controller.h" ++#include "cstruct_wrapper.h" ++#include "utils_array.h" + + const std::string DUMMY_SANDBOX_ID = "604db93a33ec4c7787e4f369338f5887"; + const std::string DUMMY_CONTAINER_ID = "504db93a32ec4c9789e4d369a38f3889"; +@@ -24,6 +26,8 @@ const uint64_t SECOND_TO_NANOS = 1000000000; + const uint64_t DUMMY_CREATE_AT = 1588 * SECOND_TO_NANOS + 1588; + const uint64_t DUMMY_EXITED_AT = 1688 * SECOND_TO_NANOS + 1588; + const std::string DUMMY_TASK_ADDRESS = "vsock://18982:1"; ++const std::string DUMMY_SANDBOXER = "vmm"; ++const std::string DUMMY_SANDBOX_EXTENSIONS_TASKS = "extensions.tasks"; + + std::unique_ptr<sandbox::ControllerMountInfo> CreateTestMountInfo(); + +@@ -31,9 +35,8 @@ std::unique_ptr<sandbox::ControllerCreateParams> CreateTestCreateParams(); + + std::unique_ptr<sandbox::ControllerStreamInfo> CreateTestStreamInfo(); + +-std::unique_ptr<sandbox::ControllerPrepareParams> CreateTestPrepareParams(); ++std::unique_ptr<CStructWrapper<sandbox_sandbox>> CreateTestUpdateApiSandbox(); + +-std::unique_ptr<sandbox::ControllerUpdateResourcesParams> CreateTestUpdateResourcesParams( +- google::protobuf::Map<std::string, std::string> &annotations); ++std::unique_ptr<CStructWrapper<string_array>> CreateTestFields(); + + #endif // _ISULAD_TEST_SANDBOX_CONTROLLER_CONTROLLER_COMMON_H +\ No newline at end of file +diff --git a/test/sandbox/controller/manager/CMakeLists.txt b/test/sandbox/controller/manager/CMakeLists.txt +index 6e8c9052..1d8ca372 100644 +--- a/test/sandbox/controller/manager/CMakeLists.txt ++++ b/test/sandbox/controller/manager/CMakeLists.txt +@@ -4,14 +4,12 @@ SET(EXE controller_manager_ut) + + add_executable(${EXE} + ${grpc_sandbox_type_srcs} +- ${CMAKE_BINARY_DIR}/grpc/src/api/services/sandbox/sandbox.pb.cc + ${CMAKE_BINARY_DIR}/grpc/src/api/services/cri/v1/api_v1.pb.cc + ${CMAKE_BINARY_DIR}/grpc/src/api/services/cri/gogo.pb.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/utils/cpputils/errors.cc +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/sandbox/controller/sandboxer/sandboxer_controller.cc +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/sandbox/controller/controller_manager.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/sandbox/sandboxer/controller/sandboxer_controller.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/sandbox/controller_manager.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/grpc_sandboxer_client_mock.cc +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/controller_stub_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/isulad_config_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/shim_controller_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/sandbox/controller/controller_common.cc +@@ -25,10 +23,11 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/entry/cri + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/executor +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/sandbox/controller +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/sandbox/controller/sandboxer +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/sandbox/controller/sandboxer/client +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/sandbox/controller/shim ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/sandbox ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/sandbox/sandboxer ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/sandbox/sandboxer/controller ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/sandbox/sandboxer/controller/client ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/sandbox/shim/controller + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/common + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/utils/cutils + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/utils/cutils/map +diff --git a/test/sandbox/controller/sandboxer/CMakeLists.txt b/test/sandbox/controller/sandboxer/CMakeLists.txt +index e77c2e6d..11689cae 100644 +--- a/test/sandbox/controller/sandboxer/CMakeLists.txt ++++ b/test/sandbox/controller/sandboxer/CMakeLists.txt +@@ -2,4 +2,3 @@ project(iSulad_UT) + + add_subdirectory(sandboxer_client) + add_subdirectory(sandboxer_controller) +-add_subdirectory(async_wait_call) +diff --git a/test/sandbox/controller/sandboxer/async_wait_call/CMakeLists.txt b/test/sandbox/controller/sandboxer/async_wait_call/CMakeLists.txt +deleted file mode 100644 +index 0631988a..00000000 +--- a/test/sandbox/controller/sandboxer/async_wait_call/CMakeLists.txt ++++ /dev/null +@@ -1,37 +0,0 @@ +-project(iSulad_UT) +- +-SET(EXE sandboxer_client_async_wait_call_ut) +- +-add_executable(${EXE} +- ${grpc_sandbox_type_srcs} +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/sandbox/controller/sandboxer/client/grpc_async_wait_call.cc +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/sandbox/controller/sandboxer/client/grpc_client_utils.cc +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cpputils/errors.cc +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../test/mocks/controller_stub_mock.cc +- ${CMAKE_CURRENT_SOURCE_DIR}/async_wait_call_ut.cc) +- +-target_include_directories(${EXE} PUBLIC +- ${GTEST_INCLUDE_DIR} +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../test/sandbox/controller +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../test/sandbox/controller/sandboxer/async_wait_call +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../test/mocks +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/entry/cri +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/sandbox/controller +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/sandbox/controller/sandboxer +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/sandbox/controller/sandboxer/client +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/common +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/map +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/sha256 +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cpputils +- ${CMAKE_BINARY_DIR}/grpc/src/api/services/cri +- ${CMAKE_BINARY_DIR}/grpc/src/api/services/cri/v1 +- ${CMAKE_BINARY_DIR}/grpc/src/api/services/sandbox +- ${CMAKE_BINARY_DIR}/grpc/src/api/services/sandbox/google/protobuf +- ${CMAKE_BINARY_DIR}/grpc/src/api/services/sandbox/sandbox/types +-) +- +-target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lgrpc -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) +-target_link_libraries(${EXE} -Wl,--as-needed ${ISULAD_ABSL_USED_TARGETS}) +-add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) +-set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/sandbox/controller/sandboxer/async_wait_call/async_wait_call_ut.cc b/test/sandbox/controller/sandboxer/async_wait_call/async_wait_call_ut.cc +deleted file mode 100644 +index 1a58344c..00000000 +--- a/test/sandbox/controller/sandboxer/async_wait_call/async_wait_call_ut.cc ++++ /dev/null +@@ -1,115 +0,0 @@ +-/****************************************************************************** +- * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +- * iSulad licensed under the Mulan PSL v2. +- * You can use this software according to the terms and conditions of the Mulan PSL v2. +- * You may obtain a copy of Mulan PSL v2 at: +- * http://license.coscl.org.cn/MulanPSL2 +- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +- * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +- * PURPOSE. +- * See the Mulan PSL v2 for more details. +- * Author: xuxuepeng +- * Create: 2023-08-01 +- * Description: Async wait call UT +- ******************************************************************************/ +- +-#include "gtest/gtest.h" +-#include "grpc_async_wait_call.h" +-#include "dummy_monitor_utils.h" +-#include "controller_stub_mock.h" +-#include "grpc_client_utils.h" +-#include "controller.h" +- +-class SandboxerAsyncWaitCallWrapper : public sandbox::SandboxerAsyncWaitCall { +-public: +- SandboxerAsyncWaitCallWrapper(std::shared_ptr<sandbox::SandboxStatusCallback> cb, +- const std::string &sandboxId, const std::string &sandboxer) +- : sandbox::SandboxerAsyncWaitCall(cb, sandboxId, sandboxer) {} +- ~SandboxerAsyncWaitCallWrapper() = default; +- +- grpc::ClientAsyncResponseReaderInterface<containerd::services::sandbox::v1::ControllerWaitResponse> &GetReader() +- { +- return *m_responseReader; +- } +-}; +- +-class AsyncWaitCallTest : public testing::Test { +-protected: +- void SetUp() override +- { +- m_sandboxId = "8040f13d54889ad4cd"; +- m_sandboxer = "test_sandboxer"; +- m_callback = std::shared_ptr<DummyCallback>(new DummyCallback()); +- m_call = std::unique_ptr<sandbox::SandboxerAsyncWaitCall>(new sandbox::SandboxerAsyncWaitCall(m_callback, m_sandboxId, +- m_sandboxer)); +- m_stub = std::unique_ptr<DummyControllerStub>(NewDummyControllerStub()); +- m_stub_mock = std::make_shared<MockControllerStub>(); +- MockControllerStub_SetMock(m_stub_mock); +- } +- +- void TearDown() override +- { +- MockControllerStub_SetMock(nullptr); +- } +- +- std::string m_sandboxId; +- std::string m_sandboxer; +- std::shared_ptr<DummyCallback> m_callback; +- std::unique_ptr<sandbox::SandboxerAsyncWaitCall> m_call; +- std::unique_ptr<DummyControllerStub> m_stub; +- std::shared_ptr<MockControllerStub> m_stub_mock; +- grpc::CompletionQueue m_cq; +-}; +- +-TEST_F(AsyncWaitCallTest, CallStatusOK) +-{ +- auto reader = new DummyClientAsyncResponseReader(); +- auto timestamp = google::protobuf::Timestamp(); +- timestamp.set_seconds(123456); +- reader->SetExitAt(timestamp); +- reader->SetExitStatus(1); +- reader->SetStatus(grpc::Status::OK); +- EXPECT_CALL(*m_stub_mock, PrepareAsyncWaitRaw).WillOnce(testing::Return(reader)); +- EXPECT_TRUE(m_call->Call(*m_stub, m_cq)); +- EXPECT_EQ(m_call->HandleResponse(), sandbox::SANDBOXER_ASYNC_WAIT_STATUS_OK); +- EXPECT_EQ(m_callback->GetStatus(), ASYNC_WAIT_CALL_STATUS_EXIT); +- EXPECT_EQ(m_callback->GetExitStatus(), 1); +- EXPECT_EQ(m_callback->GetExitedAt(), TimestampToNanos(timestamp)); +-} +- +-TEST_F(AsyncWaitCallTest, CallStatusError) +-{ +- auto reader = new DummyClientAsyncResponseReader(); +- reader->SetStatus(grpc::Status(grpc::StatusCode::PERMISSION_DENIED, "Permission denied")); +- EXPECT_CALL(*m_stub_mock, PrepareAsyncWaitRaw).WillOnce(testing::Return(reader)); +- EXPECT_TRUE(m_call->Call(*m_stub, m_cq)); +- EXPECT_EQ(m_call->HandleResponse(), sandbox::SANDBOXER_ASYNC_WAIT_STATUS_ERROR); +- EXPECT_EQ(m_callback->GetStatus(), ASYNC_WAIT_CALL_STATUS_EXIT); +-} +- +-TEST_F(AsyncWaitCallTest, CallStatusNotFound) +-{ +- auto reader = new DummyClientAsyncResponseReader(); +- reader->SetStatus(grpc::Status(grpc::StatusCode::NOT_FOUND, "Not found")); +- EXPECT_CALL(*m_stub_mock, PrepareAsyncWaitRaw).WillOnce(testing::Return(reader)); +- EXPECT_TRUE(m_call->Call(*m_stub, m_cq)); +- EXPECT_EQ(m_call->HandleResponse(), sandbox::SANDBOXER_ASYNC_WAIT_STATUS_NOT_FOUND); +- EXPECT_EQ(m_callback->GetStatus(), ASYNC_WAIT_CALL_STATUS_EXIT); +-} +- +-TEST_F(AsyncWaitCallTest, CallStatusUnavailable) +-{ +- auto reader = new DummyClientAsyncResponseReader(); +- reader->SetStatus(grpc::Status(grpc::StatusCode::UNAVAILABLE, "Unavailable")); +- EXPECT_CALL(*m_stub_mock, PrepareAsyncWaitRaw).WillOnce(testing::Return(reader)); +- EXPECT_TRUE(m_call->Call(*m_stub, m_cq)); +- EXPECT_EQ(m_call->HandleResponse(), sandbox::SANDBOXER_ASYNC_WAIT_STATUS_RETRY); +- EXPECT_EQ(m_callback->GetStatus(), ASYNC_WAIT_CALL_STATUS_PENDING); +-} +- +-TEST_F(AsyncWaitCallTest, CallStatusPrepareAsyncWaitFailed) +-{ +- EXPECT_CALL(*m_stub_mock, PrepareAsyncWaitRaw).WillOnce(testing::Return(nullptr)); +- EXPECT_FALSE(m_call->Call(*m_stub, m_cq)); +- EXPECT_EQ(m_callback->GetStatus(), ASYNC_WAIT_CALL_STATUS_EXIT); +-} +diff --git a/test/sandbox/controller/sandboxer/async_wait_call/dummy_monitor_utils.h b/test/sandbox/controller/sandboxer/async_wait_call/dummy_monitor_utils.h +deleted file mode 100644 +index 7e98d844..00000000 +--- a/test/sandbox/controller/sandboxer/async_wait_call/dummy_monitor_utils.h ++++ /dev/null +@@ -1,121 +0,0 @@ +-/****************************************************************************** +- * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +- * iSulad licensed under the Mulan PSL v2. +- * You can use this software according to the terms and conditions of the Mulan PSL v2. +- * You may obtain a copy of Mulan PSL v2 at: +- * http://license.coscl.org.cn/MulanPSL2 +- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +- * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +- * PURPOSE. +- * See the Mulan PSL v2 for more details. +- * Author: xuxuepeng +- * Create: 2023-08-01 +- * Description: provide dummy class definition for monitor unit tests +- *********************************************************************************/ +- +-#ifndef DUMMY_MONITOR_UTILS_H_ +-#define DUMMY_MONITOR_UTILS_H_ +- +-#include "sandbox.pb.h" +-#include "sandbox.grpc.pb.h" +- +-#include "controller.h" +- +-class DummyClientAsyncResponseReader: public +- grpc::ClientAsyncResponseReaderInterface<containerd::services::sandbox::v1::ControllerWaitResponse> { +-public: +- DummyClientAsyncResponseReader() = default; +- ~DummyClientAsyncResponseReader() = default; +- +- void StartCall() override {} +- +- void ReadInitialMetadata(void *tag) override {} +- +- void Finish(containerd::services::sandbox::v1::ControllerWaitResponse *response, grpc::Status *status, +- void *tag) override +- { +- response->set_exit_status(m_exitStatus); +- response->mutable_exited_at()->CopyFrom(m_exitedAt); +- *status = m_status; +- m_tag = tag; +- } +- +- void SetExitAt(const google::protobuf::Timestamp &exitAt) +- { +- m_exitedAt = exitAt; +- } +- +- void SetExitStatus(uint32_t status) +- { +- m_exitStatus = status; +- } +- +- void SetStatus(grpc::Status status) +- { +- m_status = status; +- } +- +- void *GetTag() +- { +- return m_tag; +- } +- +-private: +- google::protobuf::Timestamp m_exitedAt; +- uint32_t m_exitStatus; +- containerd::services::sandbox::v1::ControllerWaitResponse *m_response; +- grpc::Status m_status; +- void *m_tag; +-}; +- +-enum AsyncWaitCallStatus { +- ASYNC_WAIT_CALL_STATUS_UNKNOWN, +- ASYNC_WAIT_CALL_STATUS_READY, +- ASYNC_WAIT_CALL_STATUS_EXIT, +- ASYNC_WAIT_CALL_STATUS_PENDING, +-}; +- +-class DummyCallback: public sandbox::SandboxStatusCallback { +-public: +- DummyCallback() +- { +- m_status = ASYNC_WAIT_CALL_STATUS_UNKNOWN; +- } +- ~DummyCallback() = default; +- +- void OnSandboxReady() override +- { +- m_status = ASYNC_WAIT_CALL_STATUS_READY; +- } +- void OnSandboxPending() override +- { +- m_status = ASYNC_WAIT_CALL_STATUS_PENDING; +- } +- void OnSandboxExit(const sandbox::ControllerExitInfo &exitInfo) override +- { +- m_status = ASYNC_WAIT_CALL_STATUS_EXIT; +- m_exitStatus = exitInfo.exitStatus; +- m_exitedAt = exitInfo.exitedAt; +- } +- +- AsyncWaitCallStatus GetStatus() +- { +- return m_status; +- } +- +- uint32_t GetExitStatus() +- { +- return m_exitStatus; +- } +- +- uint64_t GetExitedAt() +- { +- return m_exitedAt; +- } +-private: +- AsyncWaitCallStatus m_status; +- uint32_t m_exitStatus; +- uint64_t m_exitedAt; +-}; +- +-#endif // DUMMY_MONITOR_UTILS_H_ +\ No newline at end of file +diff --git a/test/sandbox/controller/sandboxer/sandboxer_client/CMakeLists.txt b/test/sandbox/controller/sandboxer/sandboxer_client/CMakeLists.txt +index 881797c6..3a136489 100644 +--- a/test/sandbox/controller/sandboxer/sandboxer_client/CMakeLists.txt ++++ b/test/sandbox/controller/sandboxer/sandboxer_client/CMakeLists.txt +@@ -4,14 +4,11 @@ SET(EXE controller_sandboxer_client_ut) + + add_executable(${EXE} + ${grpc_sandbox_type_srcs} +- ${CMAKE_BINARY_DIR}/grpc/src/api/services/sandbox/sandbox.pb.cc +- ${CMAKE_BINARY_DIR}/grpc/src/api/services/sandbox/sandbox.grpc.pb.cc +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_client.cc +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/sandbox/controller/sandboxer/client/grpc_client_utils.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/sandbox/sandboxer/controller/client/grpc_sandboxer_client.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cpputils/errors.cc +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../test/mocks/controller_stub_mock.cc +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../test/mocks/grpc_sandboxer_monitor_mock.cc +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../test/mocks/grpc_async_wait_call_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cpputils/transform.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../test/mocks/sandbox_manager_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../test/mocks/rust_sandbox_api_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../test/sandbox/controller/controller_common.cc + ${CMAKE_CURRENT_SOURCE_DIR}/sandboxer_client_ut.cc) + +@@ -19,10 +16,12 @@ target_include_directories(${EXE} PUBLIC + ${GTEST_INCLUDE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../test/sandbox/controller + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../test/mocks ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/entry/cri +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/sandbox/controller +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/sandbox/controller/sandboxer +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/sandbox/controller/sandboxer/client ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/sandbox ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/sandbox/sandboxer ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/sandbox/sandboxer/controller ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/sandbox/sandboxer/controller/client + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/common + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/map +@@ -30,9 +29,6 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cpputils + ${CMAKE_BINARY_DIR}/grpc/src/api/services/cri + ${CMAKE_BINARY_DIR}/grpc/src/api/services/cri/v1 +- ${CMAKE_BINARY_DIR}/grpc/src/api/services/sandbox +- ${CMAKE_BINARY_DIR}/grpc/src/api/services/sandbox/google/protobuf +- ${CMAKE_BINARY_DIR}/grpc/src/api/services/sandbox/sandbox/types + ) + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lgrpc -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) +diff --git a/test/sandbox/controller/sandboxer/sandboxer_client/sandboxer_client_ut.cc b/test/sandbox/controller/sandboxer/sandboxer_client/sandboxer_client_ut.cc +index 3b02d95d..019e777c 100644 +--- a/test/sandbox/controller/sandboxer/sandboxer_client/sandboxer_client_ut.cc ++++ b/test/sandbox/controller/sandboxer/sandboxer_client/sandboxer_client_ut.cc +@@ -13,21 +13,16 @@ + * Description: Sandboxer client UT + ******************************************************************************/ + ++#include <isula_libutils/json_common.h> ++ + #include "gtest/gtest.h" +-#include "controller_stub_mock.h" + #include "grpc_sandboxer_client.h" ++#include "grpc_sandboxer_client_mock.h" ++#include "rust_sandbox_api_mock.h" ++#include "sandbox_manager_mock.h" + #include "controller_common.h" + #include "controller.h" +- +-class SandboxerClientWrapper : public sandbox::SandboxerClient { +-public: +- SandboxerClientWrapper(const std::string &sandboxer, const std::string &address) : SandboxerClient(sandboxer, address) +- { +- m_stub = NewDummyControllerStub(); +- } +- +- ~SandboxerClientWrapper() = default; +-}; ++#include "utils.h" + + class ControllerSandboxerClientTest : public testing::Test { + protected: +@@ -35,78 +30,38 @@ protected: + { + m_sandboxer = "sandboxer"; + m_address = "/tmp/sandboxer.sock"; ++ ControllerHandle_t handle_ptr = (ControllerHandle_t)(0x1); // just not nullptr ++ ++ m_sandboxManagerMock = std::make_shared<MockSandboxManager>(); ++ MockSandboxManager_SetMock(m_sandboxManagerMock); ++ m_rustSandboxApiMock = std::make_shared<RustSandboxApiMock>(); ++ RustSandboxApiMock_SetMock(m_rustSandboxApiMock); + +- m_sandboxerClient = std::make_shared<SandboxerClientWrapper>(m_sandboxer, m_address); +- m_stub = std::make_shared<MockControllerStub>(); +- MockControllerStub_SetMock(m_stub); ++ EXPECT_CALL(*m_rustSandboxApiMock, sandbox_api_build_controller).Times(1).WillOnce(testing::DoAll(testing::Return(handle_ptr))); ++ m_sandboxerClient = std::make_shared<SandboxerClient>(m_sandboxer, m_address); + } + + void TearDown() override + { +- MockControllerStub_SetMock(nullptr); ++ MockSandboxManager_SetMock(nullptr); ++ RustSandboxApiMock_SetMock(nullptr); + } + + std::string m_sandboxer; + std::string m_address; +- +- std::shared_ptr<MockControllerStub> m_stub; +- std::shared_ptr<SandboxerClientWrapper> m_sandboxerClient; ++ std::shared_ptr<SandboxerClient> m_sandboxerClient; ++ std::shared_ptr<MockSandboxManager> m_sandboxManagerMock = nullptr; ++ std::shared_ptr<RustSandboxApiMock> m_rustSandboxApiMock = nullptr; + }; + +-static std::unique_ptr<containerd::services::sandbox::v1::ControllerStartResponse> CreateTestGrpcStartResponse() +-{ +- std::unique_ptr<containerd::services::sandbox::v1::ControllerStartResponse> response( +- new containerd::services::sandbox::v1::ControllerStartResponse()); +- response->set_sandbox_id(DUMMY_SANDBOX_ID); +- response->set_pid(1); +- response->mutable_created_at()->set_seconds(DUMMY_CREATE_AT / SECOND_TO_NANOS); +- response->mutable_created_at()->set_nanos(DUMMY_CREATE_AT % SECOND_TO_NANOS); +- response->mutable_labels()->insert({"label1", "value1"}); +- return response; +-} +- +-// Create platform response for test. +-static std::unique_ptr<containerd::services::sandbox::v1::ControllerPlatformResponse> CreateTestPlatformResponse() +-{ +- std::unique_ptr<containerd::services::sandbox::v1::ControllerPlatformResponse> response( +- new containerd::services::sandbox::v1::ControllerPlatformResponse() +- ); +- response->mutable_platform()->set_os("linux"); +- response->mutable_platform()->set_architecture("amd64"); +- response->mutable_platform()->set_variant("ubuntu"); +- return response; +-} +- +-// Create status response for test +-static std::unique_ptr<containerd::services::sandbox::v1::ControllerStatusResponse> CreateTestStatusResponse() +-{ +- std::unique_ptr<containerd::services::sandbox::v1::ControllerStatusResponse> response( +- new containerd::services::sandbox::v1::ControllerStatusResponse() +- ); +- response->set_sandbox_id(DUMMY_SANDBOX_ID); +- response->set_state("running"); +- response->set_pid(1); +- response->set_task_address(DUMMY_TASK_ADDRESS); +- response->mutable_created_at()->set_seconds(DUMMY_CREATE_AT / SECOND_TO_NANOS); +- response->mutable_created_at()->set_nanos(DUMMY_CREATE_AT % SECOND_TO_NANOS); +- response->mutable_exited_at()->set_seconds(DUMMY_CREATE_AT / SECOND_TO_NANOS); +- response->mutable_exited_at()->set_nanos(DUMMY_CREATE_AT % SECOND_TO_NANOS); +- response->mutable_info()->insert({"info1", "value1"}); +- response->mutable_extra()->set_value("{extra: test}"); +- return response; +-} +- + /************* Unit tests for Create *************/ + TEST_F(ControllerSandboxerClientTest, CreateTestSucceed) + { + Errors err; + std::unique_ptr<sandbox::ControllerCreateParams> params = CreateTestCreateParams(); +- // Fake a grpc create response. +- containerd::services::sandbox::v1::ControllerCreateResponse response; +- response.set_sandbox_id(DUMMY_SANDBOX_ID); +- // Set response to return sandbox_id, and return OK for stub_->Create(). +- EXPECT_CALL(*m_stub, Create).Times(1).WillOnce(testing::DoAll(testing::SetArgPointee<2>(response), +- testing::Return(grpc::Status::OK))); ++ ++ // Set response to return sandbox_id, and return OK for sandbox_api_create(). ++ EXPECT_CALL(*m_rustSandboxApiMock, sandbox_api_create).Times(1).WillOnce(testing::DoAll(testing::Return(0))); + EXPECT_TRUE(m_sandboxerClient->Create(DUMMY_SANDBOX_ID, *params, err)); + EXPECT_TRUE(err.Empty()); + } +@@ -117,7 +72,7 @@ TEST_F(ControllerSandboxerClientTest, CreateTestNullConfig) + std::unique_ptr<sandbox::ControllerCreateParams> params(new sandbox::ControllerCreateParams()); + params->config = nullptr; + // Stub should not be called +- EXPECT_CALL(*m_stub, Create).Times(0); ++ EXPECT_CALL(*m_rustSandboxApiMock, sandbox_api_create).Times(0); + EXPECT_FALSE(m_sandboxerClient->Create(DUMMY_SANDBOX_ID, *params, err)); + EXPECT_FALSE(err.Empty()); + EXPECT_THAT(err.GetCMessage(), testing::HasSubstr("Failed to init create request for sandboxer create request")); +@@ -128,13 +83,9 @@ TEST_F(ControllerSandboxerClientTest, CreateTestNullMount) + Errors err; + std::unique_ptr<sandbox::ControllerCreateParams> params = CreateTestCreateParams(); + params->mounts.push_back(nullptr); +- containerd::services::sandbox::v1::ControllerCreateRequest request; + // Save request to check mount size. +- EXPECT_CALL(*m_stub, Create).Times(1).WillOnce(testing::DoAll(testing::SaveArg<1>(&request), +- testing::Return(grpc::Status::OK))); ++ EXPECT_CALL(*m_rustSandboxApiMock, sandbox_api_create).Times(1).WillOnce(testing::DoAll(testing::Return(0))); + EXPECT_TRUE(m_sandboxerClient->Create(DUMMY_SANDBOX_ID, *params, err)); +- // The nullptr pushed in params should not be counted. +- EXPECT_EQ(request.rootfs_size(), 1); + EXPECT_TRUE(err.Empty()); + } + +@@ -142,22 +93,65 @@ TEST_F(ControllerSandboxerClientTest, CreateTestStatusNotOK) + { + Errors err; + std::unique_ptr<sandbox::ControllerCreateParams> params = CreateTestCreateParams(); +- // Fake a grpc create response. +- containerd::services::sandbox::v1::ControllerCreateResponse response; +- response.set_sandbox_id(DUMMY_SANDBOX_ID); +- EXPECT_CALL(*m_stub, Create).Times(1).WillOnce(testing::Return(grpc::Status(grpc::StatusCode::ABORTED, "gRPC Abort"))); ++ ++ EXPECT_CALL(*m_rustSandboxApiMock, sandbox_api_create).Times(1).WillOnce(testing::Return(-1)); + EXPECT_FALSE(m_sandboxerClient->Create(DUMMY_SANDBOX_ID, *params, err)); +- EXPECT_THAT(err.GetCMessage(), testing::HasSubstr("gRPC Abort")); + } + + /************* Unit tests for Start *************/ ++static std::unique_ptr<CStructWrapper<json_map_string_string>> GetMockLabels() ++{ ++ json_map_string_string *labels = nullptr; ++ size_t len = 1; ++ ++ auto labels_wrapper = makeUniquePtrCStructWrapper<json_map_string_string>(free_json_map_string_string); ++ if (labels_wrapper == nullptr) { ++ return nullptr; ++ } ++ labels = labels_wrapper->get(); ++ ++ labels->keys = (char **)util_smart_calloc_s(sizeof(char *), len); ++ if (labels->keys == nullptr) { ++ return nullptr; ++ } ++ labels->keys[0] = util_strdup_s("label1"); ++ labels->values = (char **)util_smart_calloc_s(sizeof(char *), len); ++ if (labels->values == nullptr) { ++ return nullptr; ++ } ++ labels->values[0] = util_strdup_s("value1"); ++ labels->len = len; ++ ++ return labels_wrapper; ++} ++ ++static std::unique_ptr<CStructWrapper<sandbox_start_response>> GetMockSandboxStartResponse() ++{ ++ sandbox_start_response *reponse = nullptr; ++ ++ auto reponse_wrapper = makeUniquePtrCStructWrapper<sandbox_start_response>(free_sandbox_start_response); ++ if (reponse_wrapper == nullptr) { ++ return nullptr; ++ } ++ reponse = reponse_wrapper->get(); ++ ++ reponse->sandbox_id = util_strdup_s(DUMMY_SANDBOX_ID.c_str()); ++ reponse->pid = 1; ++ reponse->created_at = DUMMY_CREATE_AT; ++ reponse->address = util_strdup_s(DUMMY_TASK_ADDRESS.c_str()); ++ reponse->version = 0; ++ reponse->labels = GetMockLabels()->move(); ++ ++ return reponse_wrapper; ++} ++ + TEST_F(ControllerSandboxerClientTest, StartTestSucceed) + { + Errors err; + sandbox::ControllerSandboxInfo sandboxInfo; +- std::unique_ptr<containerd::services::sandbox::v1::ControllerStartResponse> response = CreateTestGrpcStartResponse(); +- EXPECT_CALL(*m_stub, Start).Times(1).WillOnce(testing::DoAll(testing::SetArgPointee<2>(*response), +- testing::Return(grpc::Status::OK))); ++ ++ EXPECT_CALL(*m_rustSandboxApiMock, sandbox_api_start).Times(1).WillOnce(testing::DoAll(testing::SetArgPointee<2>(*(GetMockSandboxStartResponse()->move())), ++ testing::Return(0))); + EXPECT_TRUE(m_sandboxerClient->Start(DUMMY_SANDBOX_ID, sandboxInfo, err)); + EXPECT_TRUE(err.Empty()); + EXPECT_EQ(sandboxInfo.id, DUMMY_SANDBOX_ID); +@@ -171,128 +165,55 @@ TEST_F(ControllerSandboxerClientTest, StartTestStatusNotOK) + { + Errors err; + sandbox::ControllerSandboxInfo sandboxInfo; +- EXPECT_CALL(*m_stub, Start).Times(1).WillOnce(testing::Return(grpc::Status(grpc::StatusCode::ABORTED, "gRPC Abort"))); ++ EXPECT_CALL(*m_rustSandboxApiMock, sandbox_api_start).Times(1).WillOnce(testing::Return(-1)); + EXPECT_FALSE(m_sandboxerClient->Start(DUMMY_SANDBOX_ID, sandboxInfo, err)); +- EXPECT_THAT(err.GetCMessage(), testing::HasSubstr("gRPC Abort")); +-} +- +-/************* Unit tests for Prepare *************/ +-TEST_F(ControllerSandboxerClientTest, PrepareTestSucceed) +-{ +- Errors err; +- std::string bundle; +- std::unique_ptr<sandbox::ControllerPrepareParams> params = CreateTestPrepareParams(); +- // Fake a grpc prepare response. +- containerd::services::sandbox::v1::PrepareResponse response; +- response.set_bundle("/tmp/bundle"); +- // Set response to return bundle, and return OK for stub_->Prepare(). +- EXPECT_CALL(*m_stub, Prepare).Times(1).WillOnce(testing::DoAll(testing::SetArgPointee<2>(response), +- testing::Return(grpc::Status::OK))); +- EXPECT_TRUE(m_sandboxerClient->Prepare(DUMMY_SANDBOX_ID, *params, bundle, err)); +- EXPECT_TRUE(err.Empty()); +- EXPECT_EQ(bundle, "/tmp/bundle"); +-} +- +-TEST_F(ControllerSandboxerClientTest, PrepareTestNullSpec) +-{ +- Errors err; +- std::string bundle; +- std::unique_ptr<sandbox::ControllerPrepareParams> params = CreateTestPrepareParams(); +- params->spec = nullptr; +- // Stub should not be called +- EXPECT_CALL(*m_stub, Prepare).Times(0); +- EXPECT_FALSE(m_sandboxerClient->Prepare(DUMMY_SANDBOX_ID, *params, bundle, err)); +- EXPECT_THAT(err.GetCMessage(), testing::HasSubstr("Failed to init prepare request for sandboxer prepare request")); + } + +-TEST_F(ControllerSandboxerClientTest, PrepareTestNullMount) ++/************* Unit tests for Update *************/ ++TEST_F(ControllerSandboxerClientTest, UpdateTestSucceed) + { + Errors err; +- std::string bundle; +- std::unique_ptr<sandbox::ControllerPrepareParams> params = CreateTestPrepareParams(); +- params->rootfs.push_back(nullptr); +- containerd::services::sandbox::v1::PrepareRequest request; +- // Save request to check mount size. +- EXPECT_CALL(*m_stub, Prepare).Times(1).WillOnce(testing::DoAll(testing::SaveArg<1>(&request), +- testing::Return(grpc::Status::OK))); +- EXPECT_TRUE(m_sandboxerClient->Prepare(DUMMY_SANDBOX_ID, *params, bundle, err)); +- // The nullptr pushed in params should not be counted. +- EXPECT_EQ(request.rootfs_size(), 2); ++ auto apiSandbox = CreateTestUpdateApiSandbox(); ++ auto fields = CreateTestFields(); ++ // Set response to return bundle, and return OK for sandbox_api_update(). ++ EXPECT_CALL(*m_rustSandboxApiMock, sandbox_api_update).Times(1).WillOnce(testing::DoAll(testing::Return(0))); ++ EXPECT_TRUE(m_sandboxerClient->Update(apiSandbox->get(), fields->get(), err)); + EXPECT_TRUE(err.Empty()); + } + +-TEST_F(ControllerSandboxerClientTest, PrepareTestStatusNotOK) ++TEST_F(ControllerSandboxerClientTest, UpdateTestStatusNotOK) + { + Errors err; +- std::string bundle; +- std::unique_ptr<sandbox::ControllerPrepareParams> params = CreateTestPrepareParams(); +- EXPECT_CALL(*m_stub, Prepare).Times(1).WillOnce(testing::Return(grpc::Status(grpc::StatusCode::ABORTED, "gRPC Abort"))); +- EXPECT_FALSE(m_sandboxerClient->Prepare(DUMMY_SANDBOX_ID, *params, bundle, err)); +- EXPECT_THAT(err.GetCMessage(), testing::HasSubstr("gRPC Abort")); ++ auto apiSandbox = CreateTestUpdateApiSandbox(); ++ auto fields = CreateTestFields(); ++ EXPECT_CALL(*m_rustSandboxApiMock, sandbox_api_update).Times(1).WillOnce(testing::Return(-1)); ++ EXPECT_FALSE(m_sandboxerClient->Update(apiSandbox->get(), fields->get(), err)); + } + +-/************* Unit tests for Purge *************/ +-TEST_F(ControllerSandboxerClientTest, PurgeTestSucceed) +-{ +- Errors err; +- // Set response to return OK for stub_->Purge(). +- EXPECT_CALL(*m_stub, Purge).Times(1).WillOnce(testing::Return(grpc::Status::OK)); +- EXPECT_TRUE(m_sandboxerClient->Purge(DUMMY_SANDBOX_ID, DUMMY_CONTAINER_ID, DUMMY_EXEC_ID, err)); +- EXPECT_TRUE(err.Empty()); +-} +- +-TEST_F(ControllerSandboxerClientTest, PurgeTestStatusNotOK) ++/************* Unit tests for Platform *************/ ++static std::unique_ptr<CStructWrapper<sandbox_platform_response>> GetMockSandboxPlatformResponse() + { +- Errors err; +- EXPECT_CALL(*m_stub, Purge).Times(1).WillOnce(testing::Return(grpc::Status(grpc::StatusCode::ABORTED, "gRPC Abort"))); +- EXPECT_FALSE(m_sandboxerClient->Purge(DUMMY_SANDBOX_ID, DUMMY_CONTAINER_ID, DUMMY_EXEC_ID, err)); +- EXPECT_THAT(err.GetCMessage(), testing::HasSubstr("gRPC Abort")); +-} ++ sandbox_platform_response *reponse = nullptr; + +-/************* Unit tests for UpdateResources *************/ +-TEST_F(ControllerSandboxerClientTest, UpdateResourcesTestSucceed) +-{ +- Errors err; +- google::protobuf::Map<std::string, std::string> annotations; +- std::unique_ptr<sandbox::ControllerUpdateResourcesParams> params = CreateTestUpdateResourcesParams(annotations); +- // Set response to return OK for stub_->UpdateResources(). +- EXPECT_CALL(*m_stub, UpdateResources).Times(1).WillOnce(testing::Return(grpc::Status::OK)); +- EXPECT_TRUE(m_sandboxerClient->UpdateResources(DUMMY_SANDBOX_ID, *params, err)); +- EXPECT_TRUE(err.Empty()); +-} ++ auto reponse_wrapper = makeUniquePtrCStructWrapper<sandbox_platform_response>(free_sandbox_platform_response); ++ if (reponse_wrapper == nullptr) { ++ return nullptr; ++ } ++ reponse = reponse_wrapper->get(); + +-TEST_F(ControllerSandboxerClientTest, UpdateResourcesTestNullResources) +-{ +- Errors err; +- google::protobuf::Map<std::string, std::string> annotations; +- std::unique_ptr<sandbox::ControllerUpdateResourcesParams> params = CreateTestUpdateResourcesParams(annotations); +- params->resources = nullptr; +- // Stub should not be called +- EXPECT_CALL(*m_stub, UpdateResources).Times(0); +- EXPECT_FALSE(m_sandboxerClient->UpdateResources(DUMMY_SANDBOX_ID, *params, err)); +- EXPECT_THAT(err.GetCMessage(), +- testing::HasSubstr("Failed to init update-resources request for sandboxer update-resources request")); +-} ++ reponse->os = util_strdup_s("linux"); ++ reponse->architecture = util_strdup_s("amd64"); ++ reponse->variant = util_strdup_s("ubuntu"); + +-TEST_F(ControllerSandboxerClientTest, UpdateResourcesTestStatusNotOK) +-{ +- Errors err; +- google::protobuf::Map<std::string, std::string> annotations; +- std::unique_ptr<sandbox::ControllerUpdateResourcesParams> params = CreateTestUpdateResourcesParams(annotations); +- EXPECT_CALL(*m_stub, UpdateResources).Times(1).WillOnce(testing::Return(grpc::Status(grpc::StatusCode::ABORTED, +- "gRPC Abort"))); +- EXPECT_FALSE(m_sandboxerClient->UpdateResources(DUMMY_SANDBOX_ID, *params, err)); +- EXPECT_THAT(err.GetCMessage(), testing::HasSubstr("gRPC Abort")); ++ return reponse_wrapper; + } + +-/************* Unit tests for Platform *************/ + TEST_F(ControllerSandboxerClientTest, PlatformTestSucceed) + { + Errors err; + sandbox::ControllerPlatformInfo platformInfo; +- std::unique_ptr<containerd::services::sandbox::v1::ControllerPlatformResponse> response = CreateTestPlatformResponse(); +- EXPECT_CALL(*m_stub, Platform).Times(1).WillOnce(testing::DoAll(testing::SetArgPointee<2>(*response), +- testing::Return(grpc::Status::OK))); ++ EXPECT_CALL(*m_rustSandboxApiMock, sandbox_api_platform).Times(1).WillOnce(testing::DoAll(testing::SetArgPointee<2>(*(GetMockSandboxPlatformResponse()->move())), ++ testing::Return(0))); + EXPECT_TRUE(m_sandboxerClient->Platform(DUMMY_SANDBOX_ID, platformInfo, err)); + EXPECT_TRUE(err.Empty()); + EXPECT_EQ(platformInfo.os, "linux"); +@@ -304,18 +225,16 @@ TEST_F(ControllerSandboxerClientTest, PlatformTestStatusNotOK) + { + Errors err; + sandbox::ControllerPlatformInfo platformInfo; +- EXPECT_CALL(*m_stub, Platform).Times(1).WillOnce(testing::Return(grpc::Status(grpc::StatusCode::ABORTED, +- "gRPC Abort"))); ++ EXPECT_CALL(*m_rustSandboxApiMock, sandbox_api_platform).Times(1).WillOnce(testing::Return(-1)); + EXPECT_FALSE(m_sandboxerClient->Platform(DUMMY_SANDBOX_ID, platformInfo, err)); +- EXPECT_THAT(err.GetCMessage(), testing::HasSubstr("gRPC Abort")); + } + + /************* Unit tests for Stop *************/ + TEST_F(ControllerSandboxerClientTest, StopTestSucceed) + { + Errors err; +- // Set response to return OK for stub_->Stop(). +- EXPECT_CALL(*m_stub, Stop).Times(1).WillOnce(testing::Return(grpc::Status::OK)); ++ // Set response to return OK for sandbox_api_stop(). ++ EXPECT_CALL(*m_rustSandboxApiMock, sandbox_api_stop).Times(1).WillOnce(testing::Return(0)); + EXPECT_TRUE(m_sandboxerClient->Stop(DUMMY_SANDBOX_ID, 0, err)); + EXPECT_TRUE(err.Empty()); + } +@@ -323,19 +242,49 @@ TEST_F(ControllerSandboxerClientTest, StopTestSucceed) + TEST_F(ControllerSandboxerClientTest, StopTestStatusNotOK) + { + Errors err; +- EXPECT_CALL(*m_stub, Stop).Times(1).WillOnce(testing::Return(grpc::Status(grpc::StatusCode::ABORTED, "gRPC Abort"))); ++ EXPECT_CALL(*m_rustSandboxApiMock, sandbox_api_stop).Times(1).WillOnce(testing::Return(-1)); + EXPECT_FALSE(m_sandboxerClient->Stop(DUMMY_SANDBOX_ID, 0, err)); +- EXPECT_THAT(err.GetCMessage(), testing::HasSubstr("gRPC Abort")); + } + + /************* Unit tests for Status *************/ ++static std::unique_ptr<CStructWrapper<sandbox_status_response>> GetMockSandboxStatusResponse() ++{ ++ sandbox_status_response *reponse = nullptr; ++ ++ auto reponse_wrapper = makeUniquePtrCStructWrapper<sandbox_status_response>(free_sandbox_status_response); ++ if (reponse_wrapper == nullptr) { ++ return nullptr; ++ } ++ reponse = reponse_wrapper->get(); ++ ++ reponse->sandbox_id = util_strdup_s(DUMMY_SANDBOX_ID.c_str()); ++ reponse->pid = 1; ++ reponse->state = util_strdup_s("running"); ++ reponse->info = GetMockLabels()->move(); ++ if (reponse->info == nullptr) { ++ return nullptr; ++ } ++ reponse->created_at = DUMMY_CREATE_AT; ++ reponse->exited_at = DUMMY_CREATE_AT; ++ reponse->extra = (defs_any *)util_common_calloc_s(sizeof(defs_any)); ++ if (reponse->extra == nullptr) { ++ return nullptr; ++ } ++ reponse->extra->value = (uint8_t*)util_strdup_s("{extra: test}"); ++ reponse->extra->value_len = 13; ++ reponse->address = util_strdup_s(DUMMY_TASK_ADDRESS.c_str()); ++ reponse->version = 0; ++ ++ return reponse_wrapper; ++} ++ + TEST_F(ControllerSandboxerClientTest, StatusTestSucceed) + { + Errors err; + sandbox::ControllerSandboxStatus sandboxStatus; +- std::unique_ptr<containerd::services::sandbox::v1::ControllerStatusResponse> response = CreateTestStatusResponse(); +- EXPECT_CALL(*m_stub, Status).Times(1).WillOnce(testing::DoAll(testing::SetArgPointee<2>(*response), +- testing::Return(grpc::Status::OK))); ++ ++ EXPECT_CALL(*m_rustSandboxApiMock, sandbox_api_status).Times(1).WillOnce(testing::DoAll(testing::SetArgPointee<2>(*(GetMockSandboxStatusResponse()->move())), ++ testing::Return(0))); + EXPECT_TRUE(m_sandboxerClient->Status(DUMMY_SANDBOX_ID, false, sandboxStatus, err)); + EXPECT_TRUE(err.Empty()); + EXPECT_EQ(sandboxStatus.id, DUMMY_SANDBOX_ID); +@@ -345,7 +294,7 @@ TEST_F(ControllerSandboxerClientTest, StatusTestSucceed) + EXPECT_EQ(sandboxStatus.createdAt, DUMMY_CREATE_AT); + EXPECT_EQ(sandboxStatus.exitedAt, DUMMY_CREATE_AT); + EXPECT_EQ(sandboxStatus.info.size(), 1); +- EXPECT_EQ(sandboxStatus.info["info1"], "value1"); ++ EXPECT_EQ(sandboxStatus.info["label1"], "value1"); + EXPECT_EQ(sandboxStatus.extra, "{extra: test}"); + } + +@@ -353,17 +302,16 @@ TEST_F(ControllerSandboxerClientTest, StatusTestStatusNotOK) + { + Errors err; + sandbox::ControllerSandboxStatus sandboxStatus; +- EXPECT_CALL(*m_stub, Status).Times(1).WillOnce(testing::Return(grpc::Status(grpc::StatusCode::ABORTED, "gRPC Abort"))); ++ EXPECT_CALL(*m_rustSandboxApiMock, sandbox_api_status).Times(1).WillOnce(testing::Return(-1)); + EXPECT_FALSE(m_sandboxerClient->Status(DUMMY_SANDBOX_ID, false, sandboxStatus, err)); +- EXPECT_THAT(err.GetCMessage(), testing::HasSubstr("gRPC Abort")); + } + + /************* Unit tests for Shutdown *************/ + TEST_F(ControllerSandboxerClientTest, ShutdownTestSucceed) + { + Errors err; +- // Set response to return OK for stub_->Shutdown(). +- EXPECT_CALL(*m_stub, Shutdown).Times(1).WillOnce(testing::Return(grpc::Status::OK)); ++ // Set response to return OK for sandbox_api_shutdown(). ++ EXPECT_CALL(*m_rustSandboxApiMock, sandbox_api_shutdown).Times(1).WillOnce(testing::Return(0)); + EXPECT_TRUE(m_sandboxerClient->Shutdown(DUMMY_SANDBOX_ID, err)); + EXPECT_TRUE(err.Empty()); + } +@@ -371,8 +319,6 @@ TEST_F(ControllerSandboxerClientTest, ShutdownTestSucceed) + TEST_F(ControllerSandboxerClientTest, ShutdownTestStatusNotOK) + { + Errors err; +- EXPECT_CALL(*m_stub, Shutdown).Times(1).WillOnce(testing::Return(grpc::Status(grpc::StatusCode::ABORTED, +- "gRPC Abort"))); ++ EXPECT_CALL(*m_rustSandboxApiMock, sandbox_api_shutdown).Times(1).WillOnce(testing::Return(-1)); + EXPECT_FALSE(m_sandboxerClient->Shutdown(DUMMY_SANDBOX_ID, err)); +- EXPECT_THAT(err.GetCMessage(), testing::HasSubstr("gRPC Abort")); + } +diff --git a/test/sandbox/controller/sandboxer/sandboxer_controller/CMakeLists.txt b/test/sandbox/controller/sandboxer/sandboxer_controller/CMakeLists.txt +index 963ce9a5..e03cc58e 100644 +--- a/test/sandbox/controller/sandboxer/sandboxer_controller/CMakeLists.txt ++++ b/test/sandbox/controller/sandboxer/sandboxer_controller/CMakeLists.txt +@@ -4,13 +4,11 @@ SET(EXE sandbox_controller_ut) + + add_executable(${EXE} + ${grpc_sandbox_type_srcs} +- ${CMAKE_BINARY_DIR}/grpc/src/api/services/sandbox/sandbox.pb.cc + ${CMAKE_BINARY_DIR}/grpc/src/api/services/cri/v1/api_v1.pb.cc + ${CMAKE_BINARY_DIR}/grpc/src/api/services/cri/gogo.pb.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cpputils/errors.cc +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/sandbox/controller/sandboxer/sandboxer_controller.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/sandbox/sandboxer/controller/sandboxer_controller.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../test/mocks/grpc_sandboxer_client_mock.cc +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../test/mocks/controller_stub_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../test/sandbox/controller/controller_common.cc + ${CMAKE_CURRENT_SOURCE_DIR}/sandboxer_controller_ut.cc) + +@@ -19,9 +17,10 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../test/sandbox/controller + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../test/mocks + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/entry/cri +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/sandbox/controller +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/sandbox/controller/sandboxer +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/sandbox/controller/sandboxer/client ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/sandbox ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/sandbox/sandboxer ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/sandbox/sandboxer/controller ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/sandbox/sandboxer/controller/client + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/common + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/map +diff --git a/test/sandbox/controller/sandboxer/sandboxer_controller/sandboxer_controller_ut.cc b/test/sandbox/controller/sandboxer/sandboxer_controller/sandboxer_controller_ut.cc +index f49d7cc5..726464e8 100644 +--- a/test/sandbox/controller/sandboxer/sandboxer_controller/sandboxer_controller_ut.cc ++++ b/test/sandbox/controller/sandboxer/sandboxer_controller/sandboxer_controller_ut.cc +@@ -27,8 +27,6 @@ protected: + m_contoller = std::move(std::unique_ptr<SandboxerController>(new SandboxerController(m_sandboxer, m_address))); + m_sandboxerClientMock = std::make_shared<SandboxerClientMock>(); + MockSandboxerClient_SetMock(m_sandboxerClientMock); +- EXPECT_CALL(*m_sandboxerClientMock, Init).Times(1); +- m_contoller->Init(err); + } + + void TearDown() override +@@ -119,61 +117,27 @@ TEST_F(SandboxerControllerTest, PlatformTestFailed) + EXPECT_EQ(ret, nullptr); + } + +-/************* Unit tests for Prepare *************/ +-TEST_F(SandboxerControllerTest, PrepareTestSucceed) ++/************* Unit tests for Update *************/ ++TEST_F(SandboxerControllerTest, UpdateTestSucceed) + { + Errors err; +- std::string bundle = "/tmp/bundle"; +- // Set response to return sandbox_id, and return OK for stub_->Prepare(). +- EXPECT_CALL(*m_sandboxerClientMock, Prepare).Times(1).WillOnce(testing::DoAll(testing::SetArgReferee<2>(bundle), +- testing::Return(true))); +- std::string ret = m_contoller->Prepare(DUMMY_SANDBOX_ID, *CreateTestPrepareParams(), err); +- EXPECT_EQ(ret, bundle); +-} +- +-TEST_F(SandboxerControllerTest, PrepareTestFailed) +-{ +- Errors err; +- // Set response to return sandbox_id, and return OK for stub_->Prepare(). +- EXPECT_CALL(*m_sandboxerClientMock, Prepare).Times(1).WillOnce(testing::Return(false)); +- std::string ret = m_contoller->Prepare(DUMMY_SANDBOX_ID, *CreateTestPrepareParams(), err); +- EXPECT_EQ(ret, ""); +-} +- +-/************* Unit tests for Purge *************/ +-TEST_F(SandboxerControllerTest, PurgeTestSucceed) +-{ +- Errors err; +- // Set response to return sandbox_id, and return OK for stub_->Purge(). +- EXPECT_CALL(*m_sandboxerClientMock, Purge).Times(1).WillOnce(testing::Return(true)); +- EXPECT_TRUE(m_contoller->Purge(DUMMY_SANDBOX_ID, DUMMY_CONTAINER_ID, DUMMY_EXEC_ID, err)); +-} ++ auto apiSandbox = CreateTestUpdateApiSandbox(); ++ auto fields = CreateTestFields(); + +-TEST_F(SandboxerControllerTest, PurgeTestFailed) +-{ +- Errors err; +- // Set response to return sandbox_id, and return OK for stub_->Purge(). +- EXPECT_CALL(*m_sandboxerClientMock, Purge).Times(1).WillOnce(testing::Return(false)); +- EXPECT_FALSE(m_contoller->Purge(DUMMY_SANDBOX_ID, DUMMY_CONTAINER_ID, DUMMY_EXEC_ID, err)); ++ // return OK for stub_->Update(). ++ EXPECT_CALL(*m_sandboxerClientMock, Update).Times(1).WillOnce(testing::DoAll(testing::Return(true))); ++ EXPECT_TRUE(m_contoller->Update(apiSandbox->get(), fields->get(), err)); + } + +-/************* Unit tests for UpdateResources *************/ +-TEST_F(SandboxerControllerTest, UpdateResourcesTestSucceed) ++TEST_F(SandboxerControllerTest, UpdateTestFailed) + { + Errors err; +- google::protobuf::Map<std::string, std::string> annotations; +- // Set response to return sandbox_id, and return OK for stub_->UpdateResources(). +- EXPECT_CALL(*m_sandboxerClientMock, UpdateResources).Times(1).WillOnce(testing::Return(true)); +- EXPECT_TRUE(m_contoller->UpdateResources(DUMMY_SANDBOX_ID, *CreateTestUpdateResourcesParams(annotations), err)); +-} ++ auto apiSandbox = CreateTestUpdateApiSandbox(); ++ auto fields = CreateTestFields(); + +-TEST_F(SandboxerControllerTest, UpdateResourcesTestFailed) +-{ +- Errors err; +- google::protobuf::Map<std::string, std::string> annotations; +- // Set response to return sandbox_id, and return OK for stub_->UpdateResources(). +- EXPECT_CALL(*m_sandboxerClientMock, UpdateResources).Times(1).WillOnce(testing::Return(false)); +- EXPECT_FALSE(m_contoller->UpdateResources(DUMMY_SANDBOX_ID, *CreateTestUpdateResourcesParams(annotations), err)); ++ // return OK for stub_->Update(). ++ EXPECT_CALL(*m_sandboxerClientMock, Update).Times(1).WillOnce(testing::Return(false)); ++ EXPECT_FALSE(m_contoller->Update(apiSandbox->get(), fields->get(), err)); + } + + /************* Unit tests for Stop *************/ +diff --git a/test/sandbox/controller/shim/CMakeLists.txt b/test/sandbox/controller/shim/CMakeLists.txt +index d18d1861..069312c9 100644 +--- a/test/sandbox/controller/shim/CMakeLists.txt ++++ b/test/sandbox/controller/shim/CMakeLists.txt +@@ -14,7 +14,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/err_msg.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/config/daemon_arguments.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/config/isulad_config.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/sandbox/controller/shim/shim_controller.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/sandbox/shim/controller/shim_controller.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/utils/cpputils/errors.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/utils/cpputils/cxxutils.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/utils/sha256/sha256.c +@@ -41,8 +41,9 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2 + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/sandbox/controller +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/sandbox/controller/shim ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/sandbox ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/sandbox/shim ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/sandbox/shim/controller + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/utils/cpputils + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/utils/sha256 + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/sandbox/controller +diff --git a/test/sandbox/sandbox/CMakeLists.txt b/test/sandbox/sandbox/CMakeLists.txt +index 2a35388f..9ee67033 100644 +--- a/test/sandbox/sandbox/CMakeLists.txt ++++ b/test/sandbox/sandbox/CMakeLists.txt +@@ -6,7 +6,6 @@ aux_source_directory(${CMAKE_BINARY_DIR}/grpc/src/api/services/sandbox/sandbox/t + + add_executable(${EXE} + ${sandbox_type_srcs} +- ${CMAKE_BINARY_DIR}/grpc/src/api/services/sandbox/sandbox.pb.cc + ${CMAKE_BINARY_DIR}/grpc/src/api/services/cri/v1/api_v1.pb.cc + ${CMAKE_BINARY_DIR}/grpc/src/api/services/cri/gogo.pb.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cpputils/errors.cc +@@ -14,14 +13,15 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cpputils/transform.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cpputils/cxxutils.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/sandbox.cc +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/controller/controller_manager.cc +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/controller/sandboxer/sandboxer_controller.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/sandbox_task.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/controller_manager.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/sandboxer/controller/sandboxer_controller.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/shim/shim_sandbox.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/id_name_manager.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/isulad_config.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/sandbox/controller/controller_common.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/daemon_arguments.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/grpc_sandboxer_client_mock.cc +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/controller_stub_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/shim_controller_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/mailbox_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c +@@ -35,18 +35,17 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/entry/cri + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/executor + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/mailbox +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/controller +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/controller/shim +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/controller/sandboxer +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/controller/sandboxer/client ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/shim ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/shim/controller ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/sandboxer ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/sandboxer/controller ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/sandboxer/controller/client + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cpputils + ${CMAKE_BINARY_DIR}/grpc/src/api/services/cri + ${CMAKE_BINARY_DIR}/grpc/src/api/services/cri/v1 +- ${CMAKE_BINARY_DIR}/grpc/src/api/services/sandbox +- ${CMAKE_BINARY_DIR}/grpc/src/api/services/sandbox/google/protobuf +- ${CMAKE_BINARY_DIR}/grpc/src/api/services/sandbox/sandbox/types + ) + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lgrpc -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) +diff --git a/test/sandbox/sandbox/sandbox_ut.cc b/test/sandbox/sandbox/sandbox_ut.cc +index dd84d8fb..192d46ef 100644 +--- a/test/sandbox/sandbox/sandbox_ut.cc ++++ b/test/sandbox/sandbox/sandbox_ut.cc +@@ -16,6 +16,7 @@ + #include <gtest/gtest.h> + + #include "sandbox.h" ++#include "shim_sandbox.h" + + namespace sandbox { + +@@ -40,7 +41,7 @@ TEST_F(SandboxTest, TestDefaultGetters) + std::string name = "test"; + RuntimeInfo info = {"runc", "shim", "kuasar"}; + +- auto sandbox = new Sandbox(id, rootdir, statedir, name, info); ++ auto sandbox = new ShimSandbox(id, rootdir, statedir, name, info); + ASSERT_NE(sandbox, nullptr); + + ASSERT_EQ(sandbox->IsReady(), false); +@@ -66,7 +67,7 @@ TEST_F(SandboxTest, TestGettersAndSetters) + std::string statedir = "/test2/statedir"; + std::string mode = "host"; + +- auto sandbox = new Sandbox(id, rootdir, statedir); ++ auto sandbox = new ShimSandbox(id, rootdir, statedir); + ASSERT_NE(sandbox, nullptr); + + sandbox->SetNetMode(mode); +diff --git a/test/sandbox/sandbox_manager/CMakeLists.txt b/test/sandbox/sandbox_manager/CMakeLists.txt +index 5a7cb2ea..9254263c 100644 +--- a/test/sandbox/sandbox_manager/CMakeLists.txt ++++ b/test/sandbox/sandbox_manager/CMakeLists.txt +@@ -6,22 +6,22 @@ aux_source_directory(${CMAKE_BINARY_DIR}/grpc/src/api/services/sandbox/sandbox/t + + add_executable(${EXE} + ${sandbox_type_srcs} +- ${CMAKE_BINARY_DIR}/grpc/src/api/services/sandbox/sandbox.pb.cc + ${CMAKE_BINARY_DIR}/grpc/src/api/services/cri/v1/api_v1.pb.cc + ${CMAKE_BINARY_DIR}/grpc/src/api/services/cri/gogo.pb.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cpputils/errors.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cpputils/read_write_lock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cpputils/transform.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sandbox_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/shim_sandbox_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sandboxer_sandbox_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/sandbox_manager.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/id_name_manager.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/isulad_config_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/daemon_arguments.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/controller/controller_manager.cc +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/controller/sandboxer/sandboxer_controller.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/controller_manager.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/grpc_sandboxer_client_mock.cc +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/controller_stub_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sandboxer_controller_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/shim_controller_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/sandbox_manager_ut.cc) + +@@ -33,18 +33,17 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/entry/cri + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/executor +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/controller +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/controller/shim +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/controller/sandboxer +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/controller/sandboxer/client ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/shim ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/shim/controller ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/sandboxer ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/sandboxer/controller ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/sandboxer/controller/client + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cpputils + ${CMAKE_BINARY_DIR}/grpc/src/api/services/cri + ${CMAKE_BINARY_DIR}/grpc/src/api/services/cri/v1 +- ${CMAKE_BINARY_DIR}/grpc/src/api/services/sandbox +- ${CMAKE_BINARY_DIR}/grpc/src/api/services/sandbox/google/protobuf +- ${CMAKE_BINARY_DIR}/grpc/src/api/services/sandbox/sandbox/types + ) + set_target_properties(${EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,util_list_all_subdir") + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lgrpc -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) +-- +2.34.1 + diff --git a/0157-bugfix-for-nri-init.patch b/0157-bugfix-for-nri-init.patch new file mode 100644 index 0000000..4751173 --- /dev/null +++ b/0157-bugfix-for-nri-init.patch @@ -0,0 +1,80 @@ +From 08b996a54c6330e704cbc9271f348a62e24fe880 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Wed, 18 Dec 2024 22:43:26 +1400 +Subject: [PATCH 157/158] bugfix for nri init + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/common/nri/nri_convert.cc | 7 ++++--- + src/daemon/nri/nri_plugin_ops.cc | 28 +++++++++++++--------------- + 2 files changed, 17 insertions(+), 18 deletions(-) + +diff --git a/src/daemon/common/nri/nri_convert.cc b/src/daemon/common/nri/nri_convert.cc +index d862d992..6e571d9a 100644 +--- a/src/daemon/common/nri/nri_convert.cc ++++ b/src/daemon/common/nri/nri_convert.cc +@@ -196,7 +196,6 @@ auto PodSandboxToNRI(const std::shared_ptr<const sandbox::Sandbox> &sandbox, nri + pod._namespace = util_strdup_s(sandbox->GetSandboxConfig().metadata().namespace_().c_str()); + } + +- + pod.labels = Transform::ProtobufMapToJsonMapForString(sandbox->GetSandboxConfig().labels(), tmpError); + if (pod.labels == nullptr) { + ERROR("Failed to transform labels to nri for pod : %s, : %s", pod.name, tmpError.GetMessage().c_str()); +@@ -971,9 +970,11 @@ auto ContainerToNRIByID(const std::string &id, nri_container &con) -> bool + goto out; + } + +- con.pod_sandbox_id = util_strdup_s(cont->common_config->sandbox_info->id); +- ret = true; ++ if (cont->common_config->sandbox_info!= nullptr && cont->common_config->sandbox_info->id != nullptr) { ++ con.pod_sandbox_id = util_strdup_s(cont->common_config->sandbox_info->id); ++ } + ++ ret = true; + out: + container_unref(cont); + return ret; +diff --git a/src/daemon/nri/nri_plugin_ops.cc b/src/daemon/nri/nri_plugin_ops.cc +index e2f88b63..7953f7de 100644 +--- a/src/daemon/nri/nri_plugin_ops.cc ++++ b/src/daemon/nri/nri_plugin_ops.cc +@@ -42,22 +42,20 @@ bool nri_adaption_init(void) + { + Errors error; + +- if (!conf_get_nri_support()) { +- return true; +- } +- +- nri_runtime_callbacks callbacks; +- callbacks.register_plugin = nri_registry_containers; +- callbacks.update_containers = nri_update_containers; +- if (nri_runtime_service_init(callbacks) != 0) { +- ERROR("Failed to init runtime service\n"); +- return false; +- } ++ if (conf_get_nri_support()) { ++ nri_runtime_callbacks callbacks; ++ callbacks.register_plugin = nri_registry_containers; ++ callbacks.update_containers = nri_update_containers; ++ if (nri_runtime_service_init(callbacks) != 0) { ++ ERROR("Failed to init runtime service\n"); ++ return false; ++ } + +- if (conf_get_nri_external_support()) { +- if (!start_external_listener()) { +- ERROR("Failed to start external listener\n"); +- goto clean_out; ++ if (conf_get_nri_external_support()) { ++ if (!start_external_listener()) { ++ ERROR("Failed to start external listener\n"); ++ goto clean_out; ++ } + } + } + +-- +2.25.1 + diff --git a/0158-Revert-move-nri-call-in-stop-and-remove-con.patch b/0158-Revert-move-nri-call-in-stop-and-remove-con.patch new file mode 100644 index 0000000..e2a0d61 --- /dev/null +++ b/0158-Revert-move-nri-call-in-stop-and-remove-con.patch @@ -0,0 +1,87 @@ +From e160e82c1a7eff3c9cca9794a4db04508e9ffb05 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Wed, 18 Dec 2024 22:54:32 +1400 +Subject: [PATCH 158/158] Revert "move nri call in stop and remove con" + +This reverts commit db60c64138b45539fe70282c853ac2dae5954924. +--- + src/daemon/common/cri/cri_helpers.cc | 18 ------------------ + .../cri/v1/v1_cri_container_manager_service.cc | 14 ++++++++++++++ + 2 files changed, 14 insertions(+), 18 deletions(-) + +diff --git a/src/daemon/common/cri/cri_helpers.cc b/src/daemon/common/cri/cri_helpers.cc +index aa8e3c19..a8cbd996 100644 +--- a/src/daemon/common/cri/cri_helpers.cc ++++ b/src/daemon/common/cri/cri_helpers.cc +@@ -32,10 +32,6 @@ + #include "isulad_config.h" + #include "sha256.h" + +-#ifdef ENABLE_NRI +-#include "nri_adaption.h" +-#endif +- + namespace CRIHelpers { + const std::string Constants::POD_NETWORK_ANNOTATION_KEY { "network.alpha.kubernetes.io/network" }; + const std::string Constants::CONTAINER_TYPE_LABEL_KEY { "cri.isulad.type" }; +@@ -664,13 +660,6 @@ void RemoveContainerHelper(service_executor_t *cb, const std::string &containerI + goto cleanup; + } + +-#ifdef ENABLE_NRI +- if (!NRIAdaptation::GetInstance()->RemoveContainer(containerID, error)) { +- ERROR("NRI RemoveContainer notification failed: %s", error.GetCMessage()); +- } +- error.Clear(); +-#endif +- + if (cb->container.remove(request, &response) != 0) { + if (response != nullptr && response->errmsg != nullptr) { + error.SetError(response->errmsg); +@@ -730,13 +719,6 @@ void StopContainerHelper(service_executor_t *cb, const std::string &containerID, + error.SetError(msg); + } + +-#ifdef ENABLE_NRI +- if (!NRIAdaptation::GetInstance()->StopContainer(containerID, error)) { +- ERROR("NRI StopContainer notification failed: %s", error.GetCMessage()); +- } +- error.Clear(); +-#endif +- + free_container_stop_request(request); + free_container_stop_response(response); + } +diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +index b585b49c..1e84d14c 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +@@ -618,11 +618,25 @@ cleanup: + + void ContainerManagerService::StopContainer(const std::string &containerID, int64_t timeout, Errors &error) + { ++#ifdef ENABLE_NRI ++ Errors nriErr; ++#endif + CRIHelpers::StopContainer(m_cb, containerID, timeout, error); ++#ifdef ENABLE_NRI ++ if (!NRIAdaptation::GetInstance()->StopContainer(containerID, nriErr)) { ++ ERROR("NRI StopContainer notification failed: %s", nriErr.GetCMessage()); ++ } ++#endif + } + + void ContainerManagerService::RemoveContainer(const std::string &containerID, Errors &error) + { ++#ifdef ENABLE_NRI ++ Errors nriErr; ++ if (!NRIAdaptation::GetInstance()->RemoveContainer(containerID, nriErr)) { ++ ERROR("NRI RemoveContainer notification failed: %s", nriErr.GetCMessage()); ++ } ++#endif + CRIHelpers::RemoveContainer(m_cb, containerID, error); + if (error.NotEmpty()) { + WARN("Failed to remove container %s", containerID.c_str()); +-- +2.25.1 + diff --git a/0159-bugfix-overwriting-when-i-is-len-1.patch b/0159-bugfix-overwriting-when-i-is-len-1.patch new file mode 100644 index 0000000..057da86 --- /dev/null +++ b/0159-bugfix-overwriting-when-i-is-len-1.patch @@ -0,0 +1,31 @@ +From 9006b2901cb17b5cf0fa3579334ddb765ac3fe11 Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Mon, 25 Nov 2024 19:42:02 +0800 +Subject: [PATCH 01/19] bugfix:overwriting when i is len-1 + +Signed-off-by: liuxu <liuxu156@huawei.com> +--- + src/daemon/modules/spec/specs.c | 7 +++++-- + 1 file changed, 5 insertions(+), 2 deletions(-) + +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index 36e89343..67a47257 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -2779,8 +2779,11 @@ void spec_remove_mount(oci_runtime_spec *oci_spec, const char *dest) + for (i = 0; i < oci_spec->mounts_len; i++) { + if (strcmp(oci_spec->mounts[i]->destination, dest) == 0) { + free_defs_mount(oci_spec->mounts[i]); +- (void)memcpy((void **)&oci_spec->mounts[i], (void **)&oci_spec->mounts[i + 1], +- (oci_spec->mounts_len - i - 1) * sizeof(void *)); ++ oci_spec->mounts[i] = NULL; ++ if (i != oci_spec->mounts_len - 1) { ++ (void)memcpy((void **)&oci_spec->mounts[i], (void **)&oci_spec->mounts[i + 1], ++ (oci_spec->mounts_len - i - 1) * sizeof(void *)); ++ } + oci_spec->mounts_len--; + return; + } +-- +2.23.0 + diff --git a/0160-bug-fix-Isula-ps-not-display-N-A-when-ports-empty.patch b/0160-bug-fix-Isula-ps-not-display-N-A-when-ports-empty.patch new file mode 100644 index 0000000..9551235 --- /dev/null +++ b/0160-bug-fix-Isula-ps-not-display-N-A-when-ports-empty.patch @@ -0,0 +1,25 @@ +From f55efffaeb1f28ac4e4e1b60b820b2b4694067b4 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?=E6=AD=A6=E7=A7=AF=E8=B6=85?= <wujichao1@huawei.com> +Date: Tue, 26 Nov 2024 20:29:29 +0800 +Subject: [PATCH 02/19] bug fix: Isula ps not display N/A when ports empty + +--- + src/cmd/isula/information/ps.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/cmd/isula/information/ps.c b/src/cmd/isula/information/ps.c +index 6bdc157f..5f46ef22 100644 +--- a/src/cmd/isula/information/ps.c ++++ b/src/cmd/isula/information/ps.c +@@ -332,7 +332,7 @@ static void print_basic_container_info_item(const struct isula_container_summary + } else if (strcmp(name, "Created") == 0) { + print_created_field(in->created, length->created_length); + } else if (strcmp(name, "Ports") == 0) { +- const char *ports = (in->ports != NULL ? in->ports : "N/A"); ++ const char *ports = (in->ports != NULL ? in->ports : " "); + printf("%-*s", (int)length->ports_length, ports); + } + } +-- +2.23.0 + diff --git a/0161-bugfix-for-workdir-len-verify.patch b/0161-bugfix-for-workdir-len-verify.patch new file mode 100644 index 0000000..238e9b5 --- /dev/null +++ b/0161-bugfix-for-workdir-len-verify.patch @@ -0,0 +1,28 @@ +From 3c9d51bfdf0e71ac846946d746003139bf3a02c8 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Tue, 26 Nov 2024 23:46:52 +1400 +Subject: [PATCH 03/19] bugfix for workdir len verify + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/modules/runtime/isula/isula_rt_ops.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index e628c3fe..22608e4c 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -786,8 +786,8 @@ static int shielded_output_check(const char *output, const char *workdir) + return 0; + } + +- if (sizeof(chdir_pattern) > PATH_MAX - strlen("chdir ") - strlen(" failed")) { +- INFO("chdir_pattern is too long"); ++ if (strlen(workdir) > PATH_MAX - strlen("chdir ") - strlen(" failed")) { ++ INFO("workdir is too long"); + return -1; + } + +-- +2.23.0 + diff --git a/0162-bugfix-fix-exec-detach-for-shim-v2.patch b/0162-bugfix-fix-exec-detach-for-shim-v2.patch new file mode 100644 index 0000000..ef821d9 --- /dev/null +++ b/0162-bugfix-fix-exec-detach-for-shim-v2.patch @@ -0,0 +1,115 @@ +From 6766ace88ac35f217b5e6b6ddab7c30e126a7956 Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Tue, 26 Nov 2024 15:37:46 +0800 +Subject: [PATCH 04/19] bugfix:fix exec --detach for shim v2 + +Signed-off-by: liuxu <liuxu156@huawei.com> +--- + .../modules/runtime/isula/isula_rt_ops.c | 11 ++---- + src/daemon/modules/runtime/runtime_common.h | 35 +++++++++++++++++++ + src/daemon/modules/runtime/shim/shim_rt_ops.c | 4 ++- + 3 files changed, 40 insertions(+), 10 deletions(-) + create mode 100644 src/daemon/modules/runtime/runtime_common.h + +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index e628c3fe..11288310 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -56,6 +56,7 @@ + #include "utils_file.h" + #include "console.h" + #include "shim_constants.h" ++#include "runtime_common.h" + + #define SHIM_BINARY "isulad-shim" + #define RESIZE_FIFO_NAME "resize_fifo" +@@ -1393,14 +1394,6 @@ int rt_isula_rm(const char *id, const char *runtime, const rt_rm_params_t *param + return 0; + } + +-static bool fg_exec(const rt_exec_params_t *params) +-{ +- if (params->console_fifos[0] != NULL || params->console_fifos[1] != NULL || params->console_fifos[2] != NULL) { +- return true; +- } +- return false; +-} +- + static char *try_generate_random_id() + { + char *id = NULL; +@@ -1536,7 +1529,7 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p + } + } + +- args.fg = fg_exec(params); ++ args.fg = rt_fg_exec(params); + args.id = id; + args.workdir = workdir; + args.bundle = bundle; +diff --git a/src/daemon/modules/runtime/runtime_common.h b/src/daemon/modules/runtime/runtime_common.h +new file mode 100644 +index 00000000..dde47110 +--- /dev/null ++++ b/src/daemon/modules/runtime/runtime_common.h +@@ -0,0 +1,35 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-11-28 ++ * Description: runtime common definition ++ ******************************************************************************/ ++ ++#ifndef DAEMON_MODULES_RUNTIME_COMMON_H ++#define DAEMON_MODULES_RUNTIME_COMMON_H ++ ++#include "runtime_api.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++static inline bool rt_fg_exec(const rt_exec_params_t *params) ++{ ++ return params->console_fifos[0] != NULL || params->console_fifos[1] != NULL || params->console_fifos[2] != NULL; ++} ++ ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif // DAEMON_MODULES_RUNTIME_COMMON_H +diff --git a/src/daemon/modules/runtime/shim/shim_rt_ops.c b/src/daemon/modules/runtime/shim/shim_rt_ops.c +index fc4d8e3a..fce18ade 100644 +--- a/src/daemon/modules/runtime/shim/shim_rt_ops.c ++++ b/src/daemon/modules/runtime/shim/shim_rt_ops.c +@@ -36,6 +36,7 @@ + #include "shim_rt_monitor.h" + #include "supervisor.h" + #include "isulad_config.h" ++#include "runtime_common.h" + + #define EXIT_SIGNAL_OFFSET_X 128 + +@@ -589,7 +590,8 @@ int rt_shim_exec(const char *id, const char *runtime, const rt_exec_params_t *pa + goto out; + } + +- if (shim_v2_wait(id, params->suffix, exit_code) != 0) { ++ if (rt_fg_exec(params) && ++ shim_v2_wait(id, params->suffix, exit_code) != 0) { + ERROR("%s: failed to wait exec process", id); + ret = -1; + goto out; +-- +2.23.0 + diff --git a/0163-image-layer-fix-code-style.patch b/0163-image-layer-fix-code-style.patch new file mode 100644 index 0000000..e3ba0a4 --- /dev/null +++ b/0163-image-layer-fix-code-style.patch @@ -0,0 +1,114 @@ +From 255fada49877e854690d628dc7832c3e459fd5aa Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Thu, 5 Dec 2024 19:59:15 +0800 +Subject: [PATCH 05/19] image layer:fix code style + +Signed-off-by: liuxu <liuxu156@huawei.com> +--- + .../modules/image/oci/storage/layer_store/layer.c | 9 +++++---- + .../modules/image/oci/storage/layer_store/layer.h | 6 +++--- + .../image/oci/storage/layer_store/layer_store.c | 11 ++++++----- + .../image/oci/storage/layer_store/layer_store.h | 5 +++-- + 4 files changed, 17 insertions(+), 14 deletions(-) + +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer.c b/src/daemon/modules/image/oci/storage/layer_store/layer.c +index 4beb3d10..8fd9aa5b 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer.c +@@ -15,16 +15,17 @@ + + #include "layer.h" + +-#include <isula_libutils/json_common.h> +-#include <isula_libutils/storage_layer.h> + #include <stdlib.h> + #include <string.h> + ++#include <isula_libutils/json_common.h> ++#include <isula_libutils/storage_layer.h> ++#include <isula_libutils/storage_mount_point.h> ++#include <isula_libutils/log.h> ++ + #include "constants.h" +-#include "isula_libutils/storage_mount_point.h" + #include "util_atomic.h" + #include "utils.h" +-#include "isula_libutils/log.h" + #include "utils_file.h" + + void free_layer_t(layer_t *ptr) +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer.h b/src/daemon/modules/image/oci/storage/layer_store/layer.h +index 9387efe0..94831ef4 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer.h ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer.h +@@ -20,9 +20,9 @@ + #include <stdbool.h> + #include <stddef.h> + +-#include "isula_libutils/storage_layer.h" +-#include "isula_libutils/storage_mount_point.h" +-#include "isula_libutils/log.h" ++#include <isula_libutils/storage_layer.h> ++#include <isula_libutils/storage_mount_point.h> ++#include <isula_libutils/log.h> + + #ifdef __cplusplus + extern "C" { +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +index 3ffe0ca7..bb2e7edc 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +@@ -19,6 +19,11 @@ + #include <stdio.h> + #include <limits.h> + #include <dirent.h> ++#include <stdint.h> ++#include <stdlib.h> ++#include <string.h> ++#include <sys/stat.h> ++ + #include <isula_libutils/container_inspect.h> + #include <isula_libutils/storage_layer.h> + #include <isula_libutils/storage_mount_point.h> +@@ -26,10 +31,6 @@ + #include <isula_libutils/log.h> + #include <isula_libutils/storage_entry.h> + #include <isula_libutils/go_crc64.h> +-#include <stdint.h> +-#include <stdlib.h> +-#include <string.h> +-#include <sys/stat.h> + + #include "util_archive.h" + #include "storage.h" +@@ -1709,7 +1710,7 @@ static int load_layers_from_json_files() + } + + ret = 0; +- goto unlock_out; ++ + unlock_out: + layer_store_unlock(); + return ret; +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h +index eba406d4..a1b0857e 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h +@@ -16,11 +16,12 @@ + #define DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_LAYER_STORE_H + + #include <stdint.h> +-#include <isula_libutils/imagetool_fs_info.h> +-#include <isula_libutils/json_common.h> + #include <stdbool.h> + #include <stddef.h> + ++#include <isula_libutils/imagetool_fs_info.h> ++#include <isula_libutils/json_common.h> ++ + #include "storage.h" + #include "io_wrapper.h" + +-- +2.23.0 + diff --git a/0164-image-store-add-UT.patch b/0164-image-store-add-UT.patch new file mode 100644 index 0000000..a390140 --- /dev/null +++ b/0164-image-store-add-UT.patch @@ -0,0 +1,207 @@ +From 2a3078132c461c45cba2932a75517314db06dcbd Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?=E6=AD=A6=E7=A7=AF=E8=B6=85?= <wujichao1@huawei.com> +Date: Fri, 6 Dec 2024 16:32:03 +0800 +Subject: [PATCH 06/19] image store: add UT + +--- + .../oci/storage/images/storage_images_ut.cc | 53 ++++++++++++++++++- + 1 file changed, 51 insertions(+), 2 deletions(-) + +diff --git a/test/image/oci/storage/images/storage_images_ut.cc b/test/image/oci/storage/images/storage_images_ut.cc +index 001090fe..ad0084d6 100644 +--- a/test/image/oci/storage/images/storage_images_ut.cc ++++ b/test/image/oci/storage/images/storage_images_ut.cc +@@ -269,12 +269,21 @@ TEST_F(StorageImagesCompatibilityUnitTest, test_load_v1_image) + EXPECT_CALL(m_storage_mock, FreeLayerList(_)).WillRepeatedly(Invoke(invokeFreeLayerList)); + opts.storage_root = strdup(store_real_path); + opts.driver_name = strdup("overlay"); ++ ++ std::string converted_image_id { "597fa49c3dbc5dd1e84120dd1906b65223afd479a7e094c085b580060c0fccec" }; ++ ASSERT_FALSE(image_store_exists(converted_image_id.c_str())); // before init must false ++ ASSERT_EQ(image_store_delete(converted_image_id.c_str()), -1); // before init must false ++ + ASSERT_EQ(image_store_init(&opts), 0); ++ // init twice will go to errror branch "Image store has already been initialized" ++ ASSERT_EQ(image_store_init(&opts), -1); + free(opts.storage_root); + free(opts.driver_name); +- std::string converted_image_id { "597fa49c3dbc5dd1e84120dd1906b65223afd479a7e094c085b580060c0fccec" }; + ASSERT_TRUE(image_store_exists(converted_image_id.c_str())); ++ const char* null_id = NULL; ++ ASSERT_FALSE(image_store_exists(null_id)); + ASSERT_EQ(image_store_delete(converted_image_id.c_str()), 0); ++ ASSERT_EQ(image_store_delete(null_id), -1); + } + + class StorageImagesUnitTest : public testing::Test { +@@ -320,6 +329,8 @@ protected: + + TEST_F(StorageImagesUnitTest, test_images_load) + { ++ const char* null_id = NULL; ++ ASSERT_EQ(image_store_get_image(null_id), nullptr); + auto image = image_store_get_image(ids.at(0).c_str()); + ASSERT_NE(image, nullptr); + +@@ -345,11 +356,17 @@ TEST_F(StorageImagesUnitTest, test_images_load) + + char **names { nullptr }; + size_t names_len { 0 }; ++ ++ ASSERT_EQ(image_store_big_data_names(null_id, &names, &names_len), -1); + ASSERT_EQ(image_store_big_data_names(ids.at(0).c_str(), &names, &names_len), 0); + ASSERT_EQ(names_len, 2); + ASSERT_STREQ(names[0], "sha256:39891ff67da98ab8540d71320915f33d2eb80ab42908e398472cab3c1ce7ac10"); + ASSERT_STREQ(names[1], "manifest"); + ++ const char* null_name = NULL; ++ ASSERT_EQ(image_store_big_data_size(null_id, names[0]), -1); ++ ASSERT_EQ(image_store_big_data_size(ids.at(0).c_str(), null_name), -1); ++ + ASSERT_EQ(image_store_big_data_size(ids.at(0).c_str(), names[0]), 2235); + ASSERT_EQ(image_store_big_data_size(ids.at(0).c_str(), names[1]), 741); + for (size_t i {}; i < names_len; ++i) { +@@ -418,7 +435,11 @@ TEST_F(StorageImagesUnitTest, test_image_store_create) + std::cout << buffer << std::endl; + + std::string key = "sha256:" + std::string(created_image); ++ const char* null_key = NULL; + ASSERT_EQ(image_store_set_big_data(created_image, key.c_str(), buffer.c_str()), 0); ++ ASSERT_EQ(image_store_set_big_data(created_image, null_key, buffer.c_str()), -1); ++ const char* null_id = NULL; ++ ASSERT_EQ(image_store_set_big_data(null_id, key.c_str(), buffer.c_str()), -1); + + std::string img_store_path = std::string(store_real_path) + "/overlay-images/"; + ASSERT_TRUE(dirExists((img_store_path + id).c_str())); +@@ -440,10 +461,13 @@ TEST_F(StorageImagesUnitTest, test_image_store_create) + std::cout << "manifest :" << std::endl; + std::cout << manifest_content << std::endl; + ++ ASSERT_EQ(image_store_big_data(null_id, "manifest"), nullptr); ++ ASSERT_EQ(image_store_big_data(id.c_str(), null_key), nullptr); + char *data = image_store_big_data(id.c_str(), "manifest"); + ASSERT_STREQ(data, manifest_content.c_str()); + free(data); + ++ ASSERT_EQ(image_store_get_image(null_id), nullptr); + auto image = image_store_get_image(id.c_str()); + ASSERT_NE(image, nullptr); + ASSERT_NE(image->created, nullptr); +@@ -476,16 +500,19 @@ TEST_F(StorageImagesUnitTest, test_image_store_create) + free_imagetool_image(image); + + char *toplayer = nullptr; ++ ASSERT_EQ(image_store_top_layer(null_id), nullptr); + ASSERT_STREQ((toplayer = image_store_top_layer(id.c_str())), + "6194458b07fcf01f1483d96cd6c34302ffff7f382bb151a6d023c4e80ba3050a"); + free(toplayer); + ++ ASSERT_EQ(image_store_set_image_size(null_id, 1000), -1); + ASSERT_EQ(image_store_set_image_size(id.c_str(), 1000), 0); + + image = image_store_get_image(id.c_str()); + ASSERT_EQ(image->size, 1000); + free_imagetool_image(image); + ++ ASSERT_EQ(image_store_add_name(null_id, "isula.org/library/test:latest"), -1); + ASSERT_EQ(image_store_add_name(id.c_str(), "isula.org/library/test:latest"), 0); + image = image_store_get_image(id.c_str()); + ASSERT_EQ(image->repo_tags_len, 2); +@@ -497,6 +524,9 @@ TEST_F(StorageImagesUnitTest, test_image_store_create) + img_names = (char **)util_common_calloc_s(2 * sizeof(char *)); + img_names[0] = util_strdup_s("busybox:latest"); + img_names[1] = util_strdup_s("centos:3.0"); ++ size_t names_len = 0; ++ ASSERT_EQ(image_store_set_names(null_id, (const char **)img_names, 2), -1); ++ ASSERT_EQ(image_store_set_names(id.c_str(), (const char **)img_names, names_len), -1); + ASSERT_EQ(image_store_set_names(id.c_str(), (const char **)img_names, 2), 0); + image = image_store_get_image(id.c_str()); + ASSERT_EQ(image->repo_tags_len, 2); +@@ -505,13 +535,17 @@ TEST_F(StorageImagesUnitTest, test_image_store_create) + util_free_array_by_len(img_names, 2); + free_imagetool_image(image); + ++ ASSERT_EQ(image_store_set_metadata(null_id, "{metadata}"), -1); + ASSERT_EQ(image_store_set_metadata(id.c_str(), "{metadata}"), 0); + char *manifest_val = nullptr; ++ ++ ASSERT_EQ(image_store_metadata(null_id), nullptr); + ASSERT_STREQ((manifest_val = image_store_metadata(id.c_str())), "{metadata}"); + free(manifest_val); + + free(created_image); + ++ ASSERT_EQ(image_store_delete(null_id), -1); + ASSERT_EQ(image_store_delete(id.c_str()), 0); + ASSERT_EQ(image_store_get_image(id.c_str()), nullptr); + ASSERT_FALSE(dirExists((img_store_path + id).c_str())); +@@ -530,6 +564,7 @@ TEST_F(StorageImagesUnitTest, test_image_store_create) + std::cout << cp_command << std::endl; + ASSERT_EQ(system(cp_command.c_str()), 0); + ++ ASSERT_EQ(image_store_big_data_digest(random_id, null_key), nullptr); + char *digest = image_store_big_data_digest(random_id, "manifest"); + ASSERT_STREQ(digest, "sha256:fdb7b1fccaaa535cb8211a194dd6314acc643f3a36d1a7d2b79c299a9173fa7e"); + free(digest); +@@ -547,6 +582,8 @@ TEST_F(StorageImagesUnitTest, test_image_store_lookup) + std::string truncatedId { "e4db68de4ff27" }; + std::string incorrectId { "4db68de4ff27" }; + ++ const char* null_id = NULL; ++ ASSERT_EQ(image_store_lookup(null_id), nullptr); + char *value = nullptr; + ASSERT_STREQ((value = image_store_lookup(name.c_str())), id.c_str()); + free(value); +@@ -563,6 +600,8 @@ TEST_F(StorageImagesUnitTest, test_image_store_exists) + std::string truncatedId { "398" }; + std::string incorrectId { "ff67da98ab8540d713209" }; + ++ const char* null_id = NULL; ++ ASSERT_FALSE(image_store_exists(null_id)); + ASSERT_TRUE(image_store_exists(name.c_str())); + ASSERT_TRUE(image_store_exists(truncatedId.c_str())); + ASSERT_FALSE(image_store_exists(incorrectId.c_str())); +@@ -584,6 +623,7 @@ TEST_F(StorageImagesUnitTest, test_image_store_metadata) + TEST_F(StorageImagesUnitTest, test_image_store_get_all_images) + { + imagetool_images_list *images_list = nullptr; ++ ASSERT_EQ(image_store_get_all_images(images_list), -1); + + images_list = (imagetool_images_list *)util_common_calloc_s(sizeof(imagetool_images_list)); + ASSERT_NE(images_list, nullptr); +@@ -613,7 +653,12 @@ TEST_F(StorageImagesUnitTest, test_image_store_get_something) + + ASSERT_EQ(image_store_get_images_number(), 2); + ASSERT_EQ(image_store_get_fs_info(fs_info), 0); ++ imagetool_fs_info *null_fs_info = nullptr; ++ ASSERT_EQ(image_store_get_fs_info(null_fs_info), -1); ++ + ASSERT_EQ(image_store_get_names(ids.at(0).c_str(), &names, &names_len), 0); ++ const char* null_id = NULL; ++ ASSERT_EQ(image_store_get_names(null_id, &names, &names_len), -1); + ASSERT_EQ(names_len, 1); + ASSERT_STREQ(names[0], "imagehub.isulad.com/official/centos:latest"); + +@@ -632,7 +677,9 @@ TEST_F(StorageImagesUnitTest, test_image_store_delete) + ASSERT_FALSE(image_store_exists(elem.c_str())); + ASSERT_FALSE(dirExists((std::string(store_real_path) + "/overlay-images/" + elem).c_str())); + } +- ++ const char* null_id = NULL; ++ ASSERT_FALSE(image_store_exists(null_id)); ++ ASSERT_EQ(image_store_delete(null_id), -1); + Restore(); + } + +@@ -646,6 +693,8 @@ TEST_F(StorageImagesUnitTest, test_image_store_remove_single_name) + } + + ASSERT_EQ(image_store_add_name(ids.at(0).c_str(), "imagehub.isulad.com/official/busybox:latest"), 0); ++ const char* null_id = NULL; ++ ASSERT_EQ(image_store_add_name(null_id, "imagehub.isulad.com/official/busybox:latest"), -1); + + Restore(); + } +-- +2.23.0 + diff --git a/0165-bugfix-do-purge-container-when-do_start_container-fa.patch b/0165-bugfix-do-purge-container-when-do_start_container-fa.patch new file mode 100644 index 0000000..f216b21 --- /dev/null +++ b/0165-bugfix-do-purge-container-when-do_start_container-fa.patch @@ -0,0 +1,125 @@ +From a6b8a2c04df21d940ce0d22128b776c00d460bba Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Fri, 6 Dec 2024 10:39:22 +0800 +Subject: [PATCH 07/19] bugfix:do purge container when do_start_container + failed + +Signed-off-by: liuxu <liuxu156@huawei.com> +--- + .../modules/container/restore/restore.c | 1 + + .../modules/container/supervisor/supervisor.c | 4 +-- + .../modules/service/service_container.c | 25 ++++++++++++++++--- + 3 files changed, 23 insertions(+), 7 deletions(-) + +diff --git a/src/daemon/modules/container/restore/restore.c b/src/daemon/modules/container/restore/restore.c +index 52f68d21..44ed14df 100644 +--- a/src/daemon/modules/container/restore/restore.c ++++ b/src/daemon/modules/container/restore/restore.c +@@ -95,6 +95,7 @@ static int restore_supervisor(const container_t *cont) + + if (container_supervisor_add_exit_monitor(exit_fifo_fd, exit_fifo, &pid_info, cont)) { + ERROR("Failed to add exit monitor to supervisor"); ++ close(exit_fifo_fd); + ret = -1; + goto out; + } +diff --git a/src/daemon/modules/container/supervisor/supervisor.c b/src/daemon/modules/container/supervisor/supervisor.c +index 294783eb..f77f58d7 100644 +--- a/src/daemon/modules/container/supervisor/supervisor.c ++++ b/src/daemon/modules/container/supervisor/supervisor.c +@@ -328,7 +328,6 @@ int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const p + + if (pid_info == NULL || cont == NULL || cont->common_config == NULL) { + ERROR("Invalid input arguments"); +- close(fd); + return -1; + } + +@@ -336,7 +335,6 @@ int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const p + cgroup_path = merge_container_cgroups_path(cont->common_config->id, cont->hostconfig); + if (cgroup_path == NULL) { + ERROR("Failed to get cgroup path"); +- close(fd); + return -1; + } + #endif +@@ -344,7 +342,6 @@ int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const p + data = util_common_calloc_s(sizeof(struct supervisor_handler_data)); + if (data == NULL) { + ERROR("Memory out"); +- close(fd); + return -1; + } + +@@ -385,6 +382,7 @@ int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const p + goto out; + + err: ++ data->fd = -1; + supervisor_handler_data_free(data); + #ifdef ENABLE_OOM_MONITOR + common_free_cgroup_oom_handler_info(oom_handler_info); +diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c +index 8e80e936..250e8299 100644 +--- a/src/daemon/modules/service/service_container.c ++++ b/src/daemon/modules/service/service_container.c +@@ -283,7 +283,6 @@ static int do_post_start_on_success(container_t *cont, int exit_fifo_fd, + { + int ret = 0; + +- // exit_fifo_fd was closed in container_supervisor_add_exit_monitor + if (container_supervisor_add_exit_monitor(exit_fifo_fd, exit_fifo, pid_info, cont)) { + ERROR("Failed to add exit monitor to supervisor"); + ret = -1; +@@ -936,7 +935,11 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo + + if (runtime_create(id, runtime, &create_params) != 0) { + ret = -1; ++#ifdef ENABLE_CRI_API_V1 ++ goto clean_prepare_container; ++#else + goto close_exit_fd; ++#endif + } + + start_params.rootpath = cont->root_path; +@@ -959,19 +962,33 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo + if (do_post_start_on_success(cont, exit_fifo_fd, exit_fifo, pid_info) != 0) { + ERROR("Failed to do post start on runtime start success"); + ret = -1; +- goto clean_resources; ++#ifdef ENABLE_CRI_API_V1 ++ goto clean_prepare_container; ++#else ++ goto close_exit_fd; ++#endif + } + } else { + // wait monitor cleanup cgroup and processes finished + wait_exit_fifo(id, exit_fifo_fd); ++#ifdef ENABLE_CRI_API_V1 ++ goto clean_prepare_container; ++#else + goto close_exit_fd; ++#endif + } + goto out; + ++#ifdef ENABLE_CRI_API_V1 ++clean_prepare_container: ++ if (cont->common_config->sandbox_info != NULL && ++ sandbox_purge_container(cont->common_config) != 0) { ++ ERROR("Failed to remove container %s from sandbox", id); ++ } ++#endif ++ + close_exit_fd: + close(exit_fifo_fd); +- +-clean_resources: + clean_resources_on_failure(cont, engine_log_path, loglevel); + + out: +-- +2.23.0 + diff --git a/0166-supplementary-registry-design-documentation.patch b/0166-supplementary-registry-design-documentation.patch new file mode 100644 index 0000000..26eac0d --- /dev/null +++ b/0166-supplementary-registry-design-documentation.patch @@ -0,0 +1,101 @@ +From 3e0cafd68cf238646c9a2fad065898d2575501fb Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Tue, 10 Dec 2024 23:53:53 +1400 +Subject: [PATCH 08/19] supplementary registry design documentation + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + .../detailed/Image/registry_degisn_zh.md | 41 +++++++++++++++++++ + docs/images/pull_detail.svg | 16 ++++++++ + 2 files changed, 57 insertions(+) + create mode 100644 docs/images/pull_detail.svg + +diff --git a/docs/design/detailed/Image/registry_degisn_zh.md b/docs/design/detailed/Image/registry_degisn_zh.md +index ac351d9a..df1061a0 100644 +--- a/docs/design/detailed/Image/registry_degisn_zh.md ++++ b/docs/design/detailed/Image/registry_degisn_zh.md +@@ -2,6 +2,7 @@ + | ------ | ---------------------------------------------- | + | Date | 2020-05-28 | + | Email | [wangfengtu@huawei.com](wangfengtu@huawei.com) | ++| Update | 钟涛 2024/12/10 新增约束限制以及流程图 | + + # 1.方案目标 + +@@ -70,6 +71,45 @@ void free_registry_login_options(registry_login_options *options); + + ## **Registry模块** + ++### 流程图 ++ ++ ++pull涉及多个线程（图中蓝色的方框为线程），只有fetch_layer时存在多线程同时拉取的情况，各个线程之间的顺序由condition与complete变量保障，已在上图中标记。 ++ ++g_shared->mutex保护多个下载线程操作g_shared->cached_layers的并发性以及操作condition与complete变量的原子性。 ++ ++### 支持限制 ++拉取容器镜像支持的Media Types类型如下： ++ ++1.支持的manifest Media Types类型。 ++ ++ (1) Docker规范镜像格式。 ++ - Image Manifest Version 2, Schema 1 ++ - application/vnd.docker.distribution.manifest.v1+json ++ - application/vnd.docker.distribution.manifest.v1+prettyjws ++ - Image Manifest Version 2, Schema 2 ++ - application/vnd.docker.distribution.manifest.v2+json ++ - application/vnd.docker.distribution.manifest.list.v2+json ++ ++ (2) OCI规范镜像格式。 ++ application/vnd.oci.image.manifest.v1+json ++ ++2.支持的layer Media Types类型。 ++ ++ (1) Docker规范镜像格式。 ++ - Image Manifest Version 2, Schema 1 ++ - application/vnd.docker.image.rootfs.diff.tar.gzip ++ - Image Manifest Version 2, Schema 2 ++ - application/vnd.docker.image.rootfs.diff.tar.gzip ++ - application/vnd.docker.image.rootfs.foreign.diff.tar.gzip ++ ++ (2) OCI规范镜像格式。 ++ - application/vnd.oci.image.layer.v1.tar+gzip ++ - application/vnd.oci.image.layer.v1.tar ++ - application/vnd.oci.image.layer.nondistributable.v1.tar ++ - application/vnd.oci.image.layer.nondistributable.v1.tar+gzip ++### 详细步骤 ++ + Registry模块调用registry apiv2模块下载镜像相关文件，并进行解压/合法性校验后调store的接口注册成镜像，并对Manager模块提供调用接口。 + + 登录操作：直接调用registry apiv2模块提供的接口实现。 +@@ -388,3 +428,4 @@ libcurl提供了实现请求的原子命令，该模块需要基于libcurl提供 + + 4、自定义消息头信息 + ++**注意**: iSulad目前仅支持从遵循HTTP 1.1协议的镜像仓库拉取镜像，尚不支持从遵循HTTP 2.0协议的镜像仓库拉取。 +diff --git a/docs/images/pull_detail.svg b/docs/images/pull_detail.svg +new file mode 100644 +index 00000000..25f61e39 +--- /dev/null ++++ b/docs/images/pull_detail.svg +@@ -0,0 +1,16 @@ ++<svg version="1.1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1716.7187754377894 755.7398799633647" width="1716.7187754377894" height="755.7398799633647"> ++ <!-- svg-source:excalidraw --> ++ ++ <defs> ++ <style class="style-fonts"> ++ @font-face { ++ font-family: "Virgil"; ++ src: url("https://excalidraw.com/Virgil.woff2"); ++ } ++ @font-face { ++ font-family: "Cascadia"; ++ src: url("https://excalidraw.com/Cascadia.woff2"); ++ } ++ </style> ++ </defs> ++ <rect x="0" y="0" width="1716.7187754377894" height="755.7398799633647" fill="#ffffff"></rect><g stroke-linecap="round" transform="translate(10 10) rotate(0 139 39)"><path d="M19.5 0 M19.5 0 C90.12 -1.19, 159.19 -0.76, 258.5 0 M19.5 0 C78.71 1.62, 137.37 1.57, 258.5 0 M258.5 0 C271.63 1.86, 276.21 6.76, 278 19.5 M258.5 0 C270.28 2, 277.52 6.23, 278 19.5 M278 19.5 C276.6 31.61, 278.78 39.7, 278 58.5 M278 19.5 C278.13 28.06, 278.48 36.44, 278 58.5 M278 58.5 C277.52 69.6, 270.66 77.17, 258.5 78 M278 58.5 C279.02 69.79, 272.97 78.42, 258.5 78 M258.5 78 C180.83 77.89, 102.41 79.07, 19.5 78 M258.5 78 C184.87 77.34, 113.29 77.44, 19.5 78 M19.5 78 C6.17 77.83, -1.54 70.48, 0 58.5 M19.5 78 C8.36 76.58, 1.74 72.37, 0 58.5 M0 58.5 C-0.3 42.14, 1.03 30.84, 0 19.5 M0 58.5 C-0.04 44.58, 0.51 28.51, 0 19.5 M0 19.5 C-0.43 8.25, 7.15 0.19, 19.5 0 M0 19.5 C-2.25 5.35, 4.59 1.87, 19.5 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(87.5 36.5) rotate(0 61.5 12.5)"><text x="61.5" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">registry_pull</text></g><g stroke-linecap="round" transform="translate(18.147989908854015 193.70363023545997) rotate(0 139 39)"><path d="M19.5 0 M19.5 0 C95.75 -0.5, 171.25 -1.08, 258.5 0 M19.5 0 C102.73 1.09, 186.41 2.49, 258.5 0 M258.5 0 C271.61 -0.33, 277.83 4.96, 278 19.5 M258.5 0 C270.32 1.86, 276.58 8.24, 278 19.5 M278 19.5 C277.55 32.36, 276.3 46.87, 278 58.5 M278 19.5 C278.41 33.1, 278.92 47.53, 278 58.5 M278 58.5 C276.26 71.07, 273.25 78.65, 258.5 78 M278 58.5 C278.22 69.25, 270.35 76.09, 258.5 78 M258.5 78 C168.06 77.46, 75.29 78.28, 19.5 78 M258.5 78 C170.87 77.75, 83.6 76.4, 19.5 78 M19.5 78 C7.87 77.85, 0.73 72.3, 0 58.5 M19.5 78 C5.99 78.59, 0.03 71.76, 0 58.5 M0 58.5 C1.82 49.81, 1.54 41.04, 0 19.5 M0 58.5 C0.91 49.46, 0.41 40.44, 0 19.5 M0 19.5 C1.45 5.51, 7.04 -1.15, 19.5 0 M0 19.5 C0.88 7.87, 6.09 0.41, 19.5 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(85.64798990885402 220.20363023545997) rotate(0 71.5 12.5)"><text x="71.5" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">registry_fetch</text></g><g stroke-linecap="round" transform="translate(14.722222222222399 358.2037319607207) rotate(0 136.5 34.5)"><path d="M17.25 0 M17.25 0 C70.45 1.67, 123.66 0.01, 255.75 0 M17.25 0 C88.98 1.15, 160.94 1.24, 255.75 0 M255.75 0 C268.22 -0.39, 274.76 6.3, 273 17.25 M255.75 0 C265.77 -2.3, 275.14 5.87, 273 17.25 M273 17.25 C273.23 25.54, 270.75 34.42, 273 51.75 M273 17.25 C273.47 25.6, 272.53 34.19, 273 51.75 M273 51.75 C272.61 64.21, 268.07 69.57, 255.75 69 M273 51.75 C271.61 64.63, 268.9 69.19, 255.75 69 M255.75 69 C159.96 68.52, 66.6 69.07, 17.25 69 M255.75 69 C181.53 69.33, 106.28 69.64, 17.25 69 M17.25 69 C7.27 70.53, 0.54 64.83, 0 51.75 M17.25 69 C8 67.99, -2.26 64.69, 0 51.75 M0 51.75 C0.26 42.93, 1.11 30.86, 0 17.25 M0 51.75 C-0.03 43.3, 0.6 36.63, 0 17.25 M0 17.25 C1.89 6.31, 5.46 -0.45, 17.25 0 M0 17.25 C-0.65 6.54, 3.56 -0.72, 17.25 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(79.2222222222224 380.2037319607207) rotate(0 72 12.5)"><text x="72" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">register_image</text></g><g stroke-linecap="round"><g transform="translate(152.70368109809033 89.00001695420997) rotate(0 0.5532208919897812 50.27837848758941)"><path d="M-0.43 0.93 C-0.37 17.55, 0.78 83.63, 0.91 100.18 M1.54 0.38 C1.48 16.63, 0.74 82.21, 0.44 98.54" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(152.70368109809033 89.00001695420997) rotate(0 0.5532208919897812 50.27837848758941)"><path d="M-7.86 71.72 C-4.38 80.53, -0.94 91.7, -0.44 96.58 M-8.45 70.32 C-5.5 81.02, -3.14 90.07, -0.01 97.63" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(152.70368109809033 89.00001695420997) rotate(0 0.5532208919897812 50.27837848758941)"><path d="M12.66 72.01 C9.08 80.64, 5.46 91.71, -0.44 96.58 M12.06 70.61 C7.59 81.18, 2.51 90.12, -0.01 97.63" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(527.2188054865057 118.7878957420885) rotate(0 144 38.5)"><path d="M19.25 0 M19.25 0 C116.54 2.16, 213.81 2.04, 268.75 0 M19.25 0 C87.92 0.18, 155.03 0.59, 268.75 0 M268.75 0 C280.01 -1.43, 286.26 6.55, 288 19.25 M268.75 0 C282.03 -0.63, 288.07 7.01, 288 19.25 M288 19.25 C286.86 29.05, 287.73 43.56, 288 57.75 M288 19.25 C287.79 28.07, 287.64 39.3, 288 57.75 M288 57.75 C286.12 68.83, 280.29 75.06, 268.75 77 M288 57.75 C288.83 69.67, 282.34 76.8, 268.75 77 M268.75 77 C214.21 79.91, 162.62 77.43, 19.25 77 M268.75 77 C209.63 76.22, 150.28 75.11, 19.25 77 M19.25 77 C4.97 76.11, -0.04 71.82, 0 57.75 M19.25 77 C7.08 77.08, 0.84 68.68, 0 57.75 M0 57.75 C-0.97 42.66, 0.85 29.54, 0 19.25 M0 57.75 C-1.09 47.23, -1.12 37.91, 0 19.25 M0 19.25 C1.98 7.35, 6.98 -0.74, 19.25 0 M0 19.25 C1.15 4.46, 6.43 2.08, 19.25 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(534.7188054865057 144.7878957420885) rotate(0 136.5 12.5)"><text x="136.5" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">fetch_and_parse_manifest</text></g><g stroke-linecap="round" transform="translate(528.8113076566447 267.04720021257526) rotate(0 140 42.5)"><path d="M21.25 0 M21.25 0 C73.8 -0.38, 127.84 0.44, 258.75 0 M21.25 0 C77.9 0.33, 134.43 0.11, 258.75 0 M258.75 0 C271.09 1.52, 281.11 6.13, 280 21.25 M258.75 0 C274.44 0, 279.62 8.79, 280 21.25 M280 21.25 C279.46 33.98, 280.13 46.12, 280 63.75 M280 21.25 C280.34 38.63, 280.18 55.1, 280 63.75 M280 63.75 C278.34 78.85, 273.43 83.08, 258.75 85 M280 63.75 C279.22 78.27, 273.67 85.9, 258.75 85 M258.75 85 C175.94 84.09, 92.81 85.57, 21.25 85 M258.75 85 C175.64 86.32, 91.93 87.2, 21.25 85 M21.25 85 C8.17 85.09, -1.7 79.42, 0 63.75 M21.25 85 C5.88 86.45, 0.44 79.26, 0 63.75 M0 63.75 C0.4 50.67, 0.39 39.93, 0 21.25 M0 63.75 C0.61 47.12, 0.23 29.48, 0 21.25 M0 21.25 C-1.1 8.45, 7.79 0.31, 21.25 0 M0 21.25 C1.13 5.54, 8.19 1.63, 21.25 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(622.3113076566447 297.04720021257526) rotate(0 46.5 12.5)"><text x="46.5" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">fetch_all</text></g><g stroke-linecap="round"><g transform="translate(302.33326551649327 225.11116530105255) rotate(0 110.47407757283065 -32.68499217101839)"><path d="M0.38 0.69 C36.84 -10.18, 182.91 -53.56, 219.61 -64.5 M-0.88 0 C35.87 -11.23, 185.04 -54.89, 221.83 -66.06" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(302.33326551649327 225.11116530105255) rotate(0 110.47407757283065 -32.68499217101839)"><path d="M198.88 -49.75 C208.38 -54.78, 216.51 -60.37, 221.4 -65.7 M198.3 -47.21 C205.79 -55.33, 215.83 -61.31, 222.59 -65.57" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(302.33326551649327 225.11116530105255) rotate(0 110.47407757283065 -32.68499217101839)"><path d="M193.03 -69.42 C204.81 -67.18, 215.11 -65.47, 221.4 -65.7 M192.45 -66.88 C201.97 -67.76, 214.16 -66.52, 222.59 -65.57" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g transform="translate(108.25910101996578 125.92592027452247) rotate(0 10.5 14.5)"><text x="0" y="21" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="start" style="white-space: pre;" direction="ltr">①</text></g><g transform="translate(377.552030312895 165.7239185680046) rotate(0 10.5 14.5)"><text x="0" y="21" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="start" style="white-space: pre;" direction="ltr">②</text></g><g stroke-linecap="round"><g transform="translate(660.6631142972701 197.41754797733165) rotate(0 0.08163946676995693 38.83720767005889)"><path d="M0.42 0.89 C0.68 13.94, 0.92 64.76, 0.99 77.36 M-0.82 0.32 C-0.58 13.06, 0.34 62.54, 0.56 75.4" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(660.6631142972701 197.41754797733165) rotate(0 0.08163946676995693 38.83720767005889)"><path d="M-8.71 49.04 C-7.17 55.76, -5.95 60.72, -0.73 74.15 M-9.45 46.44 C-8.24 52.83, -5.52 59.72, 0.17 74.71" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(660.6631142972701 197.41754797733165) rotate(0 0.08163946676995693 38.83720767005889)"><path d="M11.8 48.68 C8.17 55.55, 4.23 60.61, -0.73 74.15 M11.07 46.07 C7.58 52.56, 5.61 59.53, 0.17 74.71" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g transform="translate(685.1075587417147 220.26935461795665) rotate(0 10.5 14.5)"><text x="0" y="21" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="start" style="white-space: pre;" direction="ltr">③</text></g><g stroke-linecap="round" transform="translate(535.7743339153258 415.3249779903531) rotate(0 140 37.5)"><path d="M18.75 0 M18.75 0 C96.92 -1.23, 176 1.18, 261.25 0 M18.75 0 C78.27 2.45, 137.41 1.23, 261.25 0 M261.25 0 C274.58 -0.85, 280.13 7.02, 280 18.75 M261.25 0 C273.22 -0.03, 279.29 7.09, 280 18.75 M280 18.75 C278.51 30.46, 278.66 40.33, 280 56.25 M280 18.75 C281.29 28.35, 280.7 35.88, 280 56.25 M280 56.25 C281.89 70.56, 274.41 74.59, 261.25 75 M280 56.25 C279.8 69.31, 273.34 75.7, 261.25 75 M261.25 75 C165.28 75.57, 70.1 74.91, 18.75 75 M261.25 75 C208.15 77.36, 156.48 76.68, 18.75 75 M18.75 75 C5.06 76.57, -1.57 68.95, 0 56.25 M18.75 75 C6.76 77.04, 1.77 70.01, 0 56.25 M0 56.25 C-0.8 45.63, 1.22 38.33, 0 18.75 M0 56.25 C1.27 46.05, -0.2 33.34, 0 18.75 M0 18.75 C-1.64 5.29, 7.61 -0.6, 18.75 0 M0 18.75 C1.51 5.11, 8.26 -0.09, 18.75 0" stroke="#364fc7" stroke-width="1" fill="none"></path></g><g transform="translate(605.2743339153258 440.3249779903531) rotate(0 70.5 12.5)"><text x="70.5" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#364fc7" text-anchor="middle" style="white-space: pre;" direction="ltr">register_layer</text></g><g stroke-linecap="round"><g transform="translate(666.5890854344229 355.9360212846236) rotate(0 -1.0966409883523625 28.326842080847427)"><path d="M0.4 -0.24 C-0.05 9.25, -1.24 48.34, -1.59 58.07 M-0.85 -1.42 C-1.57 8.21, -2.31 46.53, -2.59 56.1" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(666.5890854344229 355.9360212846236) rotate(0 -1.0966409883523625 28.326842080847427)"><path d="M-12.21 28.53 C-7.43 39.13, -3.68 52.14, -2.69 55.68 M-11.6 27.87 C-9.19 36.84, -7.74 43.66, -2.74 55.87" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(666.5890854344229 355.9360212846236) rotate(0 -1.0966409883523625 28.326842080847427)"><path d="M7.55 29.02 C4.53 39.49, 0.49 52.31, -2.69 55.68 M8.17 28.36 C5.39 37.14, 1.67 43.83, -2.74 55.87" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g transform="translate(691.7742254083812 366.9360212846236) rotate(0 10.5 14.5)"><text x="0" y="21" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="start" style="white-space: pre;" direction="ltr">④</text></g><g stroke-linecap="round" transform="translate(1088.4760014697767 287.511886750809) rotate(0 103.5 42)"><path d="M21 0 M21 0 C70.65 -0.49, 116.05 -1.09, 186 0 M21 0 C67.72 -0.18, 115.74 0.29, 186 0 M186 0 C198.64 -0.29, 206.53 7.64, 207 21 M186 0 C198.48 2, 207.15 6.87, 207 21 M207 21 C205.72 36.95, 207.58 51.08, 207 63 M207 21 C206.82 36.48, 207.27 53.67, 207 63 M207 63 C208.76 78.58, 198.48 84.74, 186 84 M207 63 C206.96 74.96, 198.27 82.61, 186 84 M186 84 C133.12 85.85, 80.09 83.99, 21 84 M186 84 C138.35 83.22, 92.24 81.73, 21 84 M21 84 C8.18 84.75, 1.51 77.03, 0 63 M21 84 C8.86 83.24, -1.83 79.18, 0 63 M0 63 C0.6 52.14, 1.23 44.85, 0 21 M0 63 C1.35 54.5, 0.04 44.71, 0 21 M0 21 C1.48 6.9, 7.62 -0.87, 21 0 M0 21 C2.11 9.12, 8.3 -2.02, 21 0" stroke="#364fc7" stroke-width="1" fill="none"></path></g><g stroke-linecap="round" transform="translate(1073.3246897687827 300.23915947808155) rotate(0 103.5 42)"><path d="M21 0 M21 0 C62.94 -0.92, 100.38 -0.04, 186 0 M21 0 C84.93 0.06, 150.37 1.56, 186 0 M186 0 C199.4 1.31, 206 8.74, 207 21 M186 0 C199.91 0.27, 208.66 5.12, 207 21 M207 21 C208.86 36.99, 206.5 52.25, 207 63 M207 21 C207.09 33.02, 206.6 44.15, 207 63 M207 63 C207.37 76.69, 201.79 83.25, 186 84 M207 63 C206.23 75.45, 199.74 83.27, 186 84 M186 84 C136.24 83.97, 87.36 83.27, 21 84 M186 84 C141.21 82.98, 94.62 83.32, 21 84 M21 84 C6.34 82.48, -1.14 78.63, 0 63 M21 84 C7.82 85.66, 2.04 79.01, 0 63 M0 63 C0.03 53.45, 1.99 45.44, 0 21 M0 63 C-1.04 52.84, -0.76 44.44, 0 21 M0 21 C-1.69 7.58, 7.76 1.37, 21 0 M0 21 C1.58 6.44, 6.61 1.85, 21 0" stroke="#364fc7" stroke-width="1" fill="none"></path></g><g transform="translate(1118.3246897687827 329.73915947808155) rotate(0 58.5 12.5)"><text x="58.5" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#364fc7" text-anchor="middle" style="white-space: pre;" direction="ltr">fetch_layer</text></g><g stroke-linecap="round" transform="translate(1106.0519070095497 279.027001274957) rotate(0 103.5 42)"><path d="M21 0 M21 0 C72.42 1.75, 124.2 -0.14, 186 0 M21 0 C69.66 1.11, 116.87 1.64, 186 0 M186 0 C200.59 0.81, 207.21 6.4, 207 21 M186 0 C202.29 0.49, 208.74 8.93, 207 21 M207 21 C205.11 37.24, 207.59 54.06, 207 63 M207 21 C206.38 37.75, 207.18 53.02, 207 63 M207 63 C205.82 75.6, 198.96 85.4, 186 84 M207 63 C208.96 75.87, 201.21 86.25, 186 84 M186 84 C120.65 84.89, 59.28 84.69, 21 84 M186 84 C147.66 83.45, 108.51 81.64, 21 84 M21 84 C7.16 83.98, 0.16 77.04, 0 63 M21 84 C9.04 86.07, -0.21 78.17, 0 63 M0 63 C-0.18 47.35, 1.39 34.69, 0 21 M0 63 C-0.17 49.74, 0.76 35.67, 0 21 M0 21 C-0.21 8.94, 6.1 0.57, 21 0 M0 21 C-1.86 5.21, 8.99 -1, 21 0" stroke="#364fc7" stroke-width="1" fill="none"></path></g><g mask="url(#mask-jN6AkCabO1AkTi3mCkjgH)" stroke-linecap="round"><g transform="translate(813.6225268517932 303.1216240429617) rotate(0 133.56510213254643 19.96642294312528)"><path d="M1.09 -0.47 C45.5 6.45, 222.8 33.74, 266.93 40.58 M0.2 -1.76 C44.47 5.37, 222.23 34.48, 266.35 41.69" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(813.6225268517932 303.1216240429617) rotate(0 133.56510213254643 19.96642294312528)"><path d="M236.09 48.43 C245.58 44.51, 255.66 41.79, 267.41 40.94 M237 47.49 C244.53 45.92, 252.16 43.88, 265.94 42.61" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(813.6225268517932 303.1216240429617) rotate(0 133.56510213254643 19.96642294312528)"><path d="M239.4 28.17 C247.61 31.18, 256.56 35.36, 267.41 40.94 M240.32 27.24 C246.93 30.94, 253.71 34.17, 265.94 42.61" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask id="mask-jN6AkCabO1AkTi3mCkjgH"><rect x="0" y="0" fill="#fff" width="1179.8550852765952" height="444.52551574072413"></rect><rect x="895.7388060641941" y="311.3235698918429" fill="#000" width="102" height="25" opacity="1"></rect></mask><g transform="translate(895.7388060641941 311.3235698918429) rotate(0 51.44882292014546 11.764477094244057)"><text x="51" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">concurrent</text></g><g stroke-linecap="round"><g transform="translate(466.05039777659306 615.30377791617) rotate(0 -88.37331562049872 -110.3714977202194)"><path d="M-0.99 1.14 C-30.41 -35.89, -147.32 -184.99, -176.68 -221.88 M0.69 0.69 C-28.84 -36.21, -147.66 -183.51, -177.44 -220.61" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(466.05039777659306 615.30377791617) rotate(0 -88.37331562049872 -110.3714977202194)"><path d="M-149.87 -204.47 C-160.01 -209.91, -164.57 -212.35, -176.72 -218.65 M-150.81 -204.45 C-156.87 -208.9, -163.92 -211.9, -177.42 -220.41" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(466.05039777659306 615.30377791617) rotate(0 -88.37331562049872 -110.3714977202194)"><path d="M-165.86 -191.61 C-171.45 -200.59, -171.43 -206.72, -176.72 -218.65 M-166.79 -191.58 C-169.18 -199.11, -172.55 -205.07, -177.42 -220.41" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g transform="translate(691.7120406090329 535.7397831702388) rotate(0 10.5 14.5)"><text x="0" y="21" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="start" style="white-space: pre;" direction="ltr">⑤</text></g><g stroke-linecap="round" transform="translate(1059.3669416704286 155.55469251761815) rotate(0 104.5 30)"><path d="M15 0 M15 0 C56.7 -2.19, 96.56 -0.14, 194 0 M15 0 C58.41 0.79, 102.6 2.08, 194 0 M194 0 C202.93 0.59, 209.44 3.05, 209 15 M194 0 C205.25 -0.69, 208.67 6.09, 209 15 M209 15 C210.74 23.15, 207.51 29.51, 209 45 M209 15 C209.39 21.59, 208.81 29.84, 209 45 M209 45 C208.12 54.94, 204.45 61.45, 194 60 M209 45 C209.52 54.31, 206.14 61.52, 194 60 M194 60 C153.57 60.24, 110.33 58.67, 15 60 M194 60 C147.31 60.73, 101.82 60.87, 15 60 M15 60 C4.61 59.05, 0.48 56.06, 0 45 M15 60 C3.38 57.76, -1.12 56.51, 0 45 M0 45 C-1.78 35.86, -1.63 28.44, 0 15 M0 45 C0.4 33.19, -0.6 21.89, 0 15 M0 15 C-0.94 5.72, 5.76 -1.58, 15 0 M0 15 C-2.12 4.79, 3.95 -1.71, 15 0" stroke="#364fc7" stroke-width="1" fill="none"></path></g><g transform="translate(1101.8669416704286 173.05469251761815) rotate(0 62 12.5)"><text x="62" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#364fc7" text-anchor="middle" style="white-space: pre;" direction="ltr">fetch_config</text></g><g mask="url(#mask-mDMVIXFpqvrn6ccF16Kk7)" stroke-linecap="round"><g transform="translate(814.2373979421299 296.96202080537887) rotate(0 121.47907200964778 -55.18683551512231)"><path d="M-0.51 0.35 C40.08 -17.96, 202.62 -90.53, 243.46 -109.11 M1.43 -0.51 C41.87 -19.21, 202.04 -92.41, 242.55 -110.72" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(814.2373979421299 296.96202080537887) rotate(0 121.47907200964778 -55.18683551512231)"><path d="M221.72 -89.61 C228.96 -93.89, 231.92 -102.59, 241.67 -111.91 M220.34 -90.63 C226.15 -94.8, 230.71 -99.86, 242.22 -109.84" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(814.2373979421299 296.96202080537887) rotate(0 121.47907200964778 -55.18683551512231)"><path d="M213.22 -108.29 C223.09 -106.99, 228.58 -110.17, 241.67 -111.91 M211.84 -109.31 C219.64 -108.94, 226.25 -109.51, 242.22 -109.84" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask id="mask-mDMVIXFpqvrn6ccF16Kk7"><rect x="0" y="0" fill="#fff" width="1157.6448279551505" height="507.03602254148973"></rect><rect x="909.9411129486402" y="229.4250199373232" fill="#000" width="52" height="25" opacity="1"></rect></mask><g transform="translate(909.9411129486401 229.4250199373232) rotate(0 25.775357003137458 12.350165352933345)"><text x="26" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">single</text></g><g transform="translate(556.7290219458388 463.07314547985834) rotate(0 26 12.5)"><text x="0" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="start" style="white-space: pre;" direction="ltr">single</text></g><g stroke-linecap="round"><g transform="translate(1196.756061136574 217.99902167343362) rotate(0 194.07942687517067 146.87343723595995)"><path d="M0.29 -1.15 C64.67 17.91, 359.61 64.4, 385.7 113.74 C411.79 163.08, 195.02 264.76, 156.81 294.89" stroke="#2b8a3e" stroke-width="1.5" fill="none" stroke-dasharray="1.5 7"></path></g></g><mask></mask><g transform="translate(1553.7187754377894 319.5175763609341) rotate(0 76.5 14.5)"><text x="0" y="21" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#2b8a3e" text-anchor="start" style="white-space: pre;" direction="ltr">desc->cond；15s</text></g><g transform="translate(1166.8298594221646 247.2953541387119) rotate(0 40.5 14.5)"><text x="0" y="21" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#2b8a3e" text-anchor="start" style="white-space: pre;" direction="ltr">max：5 </text></g><g stroke-linecap="round"><g transform="translate(1295.7187483110533 368.5175763609341) rotate(0 42.74559004776803 -37.8835918722466)"><path d="M0.96 0.9 C14.84 -4.49, 82.43 -18.3, 84.48 -31.23 C86.54 -44.15, 25.33 -69.35, 13.28 -76.67" stroke="#2b8a3e" stroke-width="1.5" fill="none" stroke-dasharray="1.5 7"></path></g></g><mask></mask><g transform="translate(1376.8298594221644 326.1842430276006) rotate(0 76 12.5)"><text x="0" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#2b8a3e" text-anchor="start" style="white-space: pre;" direction="ltr">g_shared->cond</text></g><g transform="translate(1028.459552347512 446.62874172551756) rotate(0 108.5 12.5)"><text x="0" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="start" style="white-space: pre;" direction="ltr">desc->cancel -->false</text></g><g transform="translate(1028.089204582755 473.2954083921845) rotate(0 141.5 12.5)"><text x="0" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="start" style="white-space: pre;" direction="ltr">desc->config.complete -->true</text></g><g transform="translate(1024.0151079030675 502.55464504583006) rotate(0 170 12.5)"><text x="0" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="start" style="white-space: pre;" direction="ltr">!info->use || info->notified -->false</text></g><g stroke-linecap="round" transform="translate(1012.1633012624426 440.81394951631637) rotate(0 182.59256998697907 50.37038167317701)"><path d="M25.19 0 M25.19 0 C90.01 -0.85, 154.36 -3.44, 340 0 M25.19 0 C108.16 -1.25, 190.27 -1.69, 340 0 M340 0 C357.36 -0.98, 366.06 10.24, 365.19 25.19 M340 0 C354.71 0.66, 363.36 6.17, 365.19 25.19 M365.19 25.19 C365.45 43.21, 365.96 65.29, 365.19 75.56 M365.19 25.19 C364.03 39.72, 365.34 55.57, 365.19 75.56 M365.19 75.56 C366.37 92.86, 356.13 99.52, 340 100.74 M365.19 75.56 C366.11 91.51, 355.42 100.3, 340 100.74 M340 100.74 C276.52 101.4, 210.28 101.88, 25.19 100.74 M340 100.74 C274.93 99.48, 209.81 100.38, 25.19 100.74 M25.19 100.74 C8.2 100.67, 1.02 91.76, 0 75.56 M25.19 100.74 C6.92 99.17, 2.15 92.94, 0 75.56 M0 75.56 C-2.26 63.39, 0.58 51.95, 0 25.19 M0 75.56 C0.8 57.78, 0.28 41.82, 0 25.19 M0 25.19 C-0.67 6.46, 6.76 -1.62, 25.19 0 M0 25.19 C0.8 9.86, 10.17 -0.5, 25.19 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1067.718856817998 405.2953405753442) rotate(0 104.5 12.5)"><text x="0" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="start" style="white-space: pre;" direction="ltr">wait_fetch_complete</text></g><g transform="translate(571.4966481591447 568.9620479321147) rotate(0 96 12.5)"><text x="0" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="start" style="white-space: pre;" direction="ltr">all_fetch_complete</text></g><g stroke-linecap="round" transform="translate(477.87548412623755 610.7398799633647) rotate(0 227 67.5)"><path d="M32 0 M32 0 C169.96 -2.27, 306.54 -2.02, 422 0 M32 0 C144.22 1.96, 256.3 2.32, 422 0 M422 0 C441.87 -0.57, 454.74 11.79, 454 32 M422 0 C441.8 -0.92, 455.92 9.38, 454 32 M454 32 C451.5 47.25, 455.09 64.27, 454 103 M454 32 C452.56 49.03, 453.01 67.55, 454 103 M454 103 C455.39 126.19, 443.63 135.96, 422 135 M454 103 C452.71 123.25, 444.41 133.84, 422 135 M422 135 C319.45 137.52, 218.49 136.27, 32 135 M422 135 C337.58 136.98, 252.61 136.71, 32 135 M32 135 C10.75 136.9, -1.05 125.79, 0 103 M32 135 C9.15 136.63, 2.27 125.28, 0 103 M0 103 C-0.92 84.11, 0.67 60.23, 0 32 M0 103 C-0.4 85.78, 0.15 66.33, 0 32 M0 32 C-1.32 12.29, 11.1 -1.62, 32 0 M0 32 C0.33 12.87, 8.67 0.05, 32 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(503.37548412623755 628.2398799633647) rotate(0 201.5 50.5)"><text x="201.5" y="18.25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">!desc->config.complete -->flase</text><text x="201.5" y="43.5" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">!desc->register_layers_complete -->false</text><text x="201.5" y="68.75" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">infos[i].use && !infos[i].notified -->false</text><text x="201.5" y="94" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">else true</text></g><g stroke-linecap="round"><g transform="translate(665.211861498176 491.6287145987817) rotate(0 0.7219837168451022 56.34321362182504)"><path d="M-0.31 -0.08 C0.02 18.89, 1.05 93.85, 1.34 112.82 M1.73 -1.17 C1.96 18.01, 0.62 94.48, 0.46 113.86" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(665.211861498176 491.6287145987817) rotate(0 0.7219837168451022 56.34321362182504)"><path d="M-9.57 85.76 C-8.81 92.99, -4.79 99.6, 1.48 115.53 M-8.44 85.29 C-5.18 96.33, -1.06 106.48, 0.29 113.52" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(665.211861498176 491.6287145987817) rotate(0 0.7219837168451022 56.34321362182504)"><path d="M10.95 86.02 C6.61 93.13, 5.52 99.68, 1.48 115.53 M12.08 85.56 C7.54 96.53, 3.84 106.58, 0.29 113.52" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round"><g transform="translate(823.0523122216447 445.3845391429513) rotate(0 96.55545522021134 20.79955033131023)"><path d="M0.77 1.15 C32.59 7.72, 159.19 32.88, 191.14 39.41 M-0.29 0.71 C31.89 7.48, 161.36 34.62, 193.4 40.89" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(823.0523122216447 445.3845391429513) rotate(0 96.55545522021134 20.79955033131023)"><path d="M165.65 43.89 C172.98 44.54, 183.83 43.16, 192.23 41.62 M163.35 45.89 C174.53 43.9, 187.83 42.6, 192.84 40.46" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(823.0523122216447 445.3845391429513) rotate(0 96.55545522021134 20.79955033131023)"><path d="M169.74 23.78 C175.77 30.84, 185.33 35.83, 192.23 41.62 M167.45 25.78 C176.89 31.74, 188.57 38.42, 192.84 40.46" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1028.7745615706028 559.3359474497524) rotate(0 176.5 35)"><path d="M17.5 0 M17.5 0 C101.71 2.79, 187.38 3.08, 335.5 0 M17.5 0 C94.57 -1, 171.26 -0.35, 335.5 0 M335.5 0 C349.05 1.01, 354.76 6.45, 353 17.5 M335.5 0 C344.93 2.02, 351.83 4, 353 17.5 M353 17.5 C354.88 32.3, 353.21 41.84, 353 52.5 M353 17.5 C354.23 25.91, 354.26 34.5, 353 52.5 M353 52.5 C354.08 65.92, 346.88 68.16, 335.5 70 M353 52.5 C351.13 65.91, 345.91 71.8, 335.5 70 M335.5 70 C248.79 69.5, 160.68 70.38, 17.5 70 M335.5 70 C227.06 69.04, 117.99 68.84, 17.5 70 M17.5 70 C5.5 70.31, -0.9 63.19, 0 52.5 M17.5 70 C6.95 69.73, -0.79 62.43, 0 52.5 M0 52.5 C0.69 45.57, -1.35 37.4, 0 17.5 M0 52.5 C-0.22 40.14, -0.63 26.56, 0 17.5 M0 17.5 C0.74 5.24, 6.54 0.4, 17.5 0 M0 17.5 C-1.73 6.97, 6.87 -0.25, 17.5 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1054.7745615706028 569.3359474497524) rotate(0 150.5 25.5)"><text x="150.5" y="17.5" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">apply_diff to archive_unpack </text><text x="150.5" y="43" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">image layer</text></g><g stroke-linecap="round"><g transform="translate(811.5855155722463 464.34086362692733) rotate(0 106.95316778426485 60.555551640269414)"><path d="M-1.06 -0.2 C34.36 20.15, 176.98 101.12, 212.63 121.44 M0.58 -1.36 C36.38 19.18, 179.77 101.79, 214.97 122.47" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(811.5855155722463 464.34086362692733) rotate(0 106.95316778426485 60.555551640269414)"><path d="M185.1 116.01 C192.85 118.23, 202.42 120.66, 214.78 123.87 M186.26 117.25 C197.58 118.43, 209.09 121.96, 215.29 121.84" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(811.5855155722463 464.34086362692733) rotate(0 106.95316778426485 60.555551640269414)"><path d="M195.41 98.26 C199.89 105.85, 206.29 113.73, 214.78 123.87 M196.56 99.5 C203.99 107.54, 211.52 117.94, 215.29 121.84" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g transform="translate(381.2823927917059 474.12870542380233) rotate(0 10.5 14.5)"><text x="0" y="21" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="start" style="white-space: pre;" direction="ltr">⑥</text></g><g stroke-linecap="round"><g transform="translate(795.3445984490046 490.642460792782) rotate(0 1.1227395565559277 60.09279141314323)"><path d="M0.49 0.41 C0.6 20.55, 1.59 99.82, 1.75 119.78" stroke="#5c940d" stroke-width="1.5" fill="none" stroke-dasharray="1.5 7"></path></g></g><mask></mask><g transform="translate(740.2334330844211 539.864669451636) rotate(0 92.5 12.5)"><text x="0" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#5c940d" text-anchor="start" style="white-space: pre;" direction="ltr">g_shared->cond; 15</text></g></svg> +\ No newline at end of file +-- +2.23.0 + diff --git a/0167-sandbox-del-shim_sandbox-and-change-sandbox-ops.patch b/0167-sandbox-del-shim_sandbox-and-change-sandbox-ops.patch new file mode 100644 index 0000000..ec48baa --- /dev/null +++ b/0167-sandbox-del-shim_sandbox-and-change-sandbox-ops.patch @@ -0,0 +1,1167 @@ +From 16da66344854a75ce7a9d7908e0a75732eb147dc Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Wed, 27 Nov 2024 15:11:57 +0800 +Subject: [PATCH 01/11] sandbox: del shim_sandbox and change sandbox ops + +Signed-off-by: liuxu <liuxu156@huawei.com> +--- + src/daemon/sandbox/sandbox.cc | 28 ++ + src/daemon/sandbox/sandbox.h | 15 +- + src/daemon/sandbox/sandbox_manager.cc | 9 +- + src/daemon/sandbox/sandbox_ops.cc | 363 +--------------- + .../sandbox/sandboxer/sandboxer_sandbox.cc | 401 ++++++++++++++++++ + .../sandbox/sandboxer/sandboxer_sandbox.h | 33 +- + src/daemon/sandbox/shim/shim_sandbox.cc | 65 --- + src/daemon/sandbox/shim/shim_sandbox.h | 49 --- + 8 files changed, 469 insertions(+), 494 deletions(-) + delete mode 100644 src/daemon/sandbox/shim/shim_sandbox.cc + delete mode 100644 src/daemon/sandbox/shim/shim_sandbox.h + +diff --git a/src/daemon/sandbox/sandbox.cc b/src/daemon/sandbox/sandbox.cc +index 3715e5e0..d105d71a 100644 +--- a/src/daemon/sandbox/sandbox.cc ++++ b/src/daemon/sandbox/sandbox.cc +@@ -1118,4 +1118,32 @@ void Sandbox::FillSandboxMetadata(sandbox_metadata* metadata, Errors &error) + + metadata->sandbox_config_json = util_strdup_s(jsonStr.c_str()); + } ++ ++void Sandbox::LoadSandboxTasks() ++{ ++} ++ ++auto Sandbox::PrepareContainer(const char *containerId, const char *baseFs, ++ const oci_runtime_spec *ociSpec, ++ const char *consoleFifos[]) -> int ++{ ++ return 0; ++} ++ ++auto Sandbox::PrepareExec(const char *containerId, const char *execId, ++ defs_process *processSpec, const char *consoleFifos[]) -> int ++{ ++ return 0; ++} ++ ++auto Sandbox::PurgeContainer(const char *containerId) -> int ++{ ++ return 0; ++} ++ ++auto Sandbox::PurgeExec(const char *containerId, const char *execId) -> int ++{ ++ return 0; ++} ++ + } +\ No newline at end of file +diff --git a/src/daemon/sandbox/sandbox.h b/src/daemon/sandbox/sandbox.h +index 415406ff..58d60ecb 100644 +--- a/src/daemon/sandbox/sandbox.h ++++ b/src/daemon/sandbox/sandbox.h +@@ -141,13 +141,14 @@ public: + void Status(runtime::v1::PodSandboxStatus &status); + + // for sandbox api update +- virtual void LoadSandboxTasks() = 0; +- virtual auto SaveSandboxTasks() -> bool = 0; +- virtual auto AddSandboxTasks(sandbox_task *task) -> bool = 0; +- virtual auto GetAnySandboxTasks() -> std::string = 0; +- virtual void DeleteSandboxTasks(const char *containerId) = 0; +- virtual auto AddSandboxTasksProcess(const char *containerId, sandbox_process *processes) -> bool = 0; +- virtual void DeleteSandboxTasksProcess(const char *containerId, const char *execId) = 0; ++ virtual void LoadSandboxTasks(); ++ virtual auto PrepareContainer(const char *containerId, const char *baseFs, ++ const oci_runtime_spec *ociSpec, ++ const char *consoleFifos[]) -> int; ++ virtual auto PrepareExec(const char *containerId, const char *execId, ++ defs_process *processSpec, const char *consoleFifos[]) -> int; ++ virtual auto PurgeContainer(const char *containerId) -> int; ++ virtual auto PurgeExec(const char *containerId, const char *execId) -> int; + + private: + auto SaveState(Errors &error) -> bool; +diff --git a/src/daemon/sandbox/sandbox_manager.cc b/src/daemon/sandbox/sandbox_manager.cc +index ba003d56..a7908a60 100644 +--- a/src/daemon/sandbox/sandbox_manager.cc ++++ b/src/daemon/sandbox/sandbox_manager.cc +@@ -27,7 +27,6 @@ + #ifdef ENABLE_SANDBOXER + #include "sandboxer_sandbox.h" + #endif +-#include "shim_sandbox.h" + #include "isulad_config.h" + #include "utils_verify.h" + #include "utils_file.h" +@@ -116,12 +115,12 @@ auto SandboxManager::CreateSandbox(const std::string &name, RuntimeInfo &info, s + + #ifdef ENABLE_SANDBOXER + if (info.sandboxer == SHIM_CONTROLLER_NAME) { +- sandbox = std::make_shared<ShimSandbox>(id, m_rootdir, m_statedir, name, info, netMode, netNsPath, sandboxConfig, image); ++ sandbox = std::make_shared<Sandbox>(id, m_rootdir, m_statedir, name, info, netMode, netNsPath, sandboxConfig, image); + } else { + sandbox = std::make_shared<SandboxerSandbox>(id, m_rootdir, m_statedir, name, info, netMode, netNsPath, sandboxConfig, image); + } + #else +- sandbox = std::make_shared<ShimSandbox>(id, m_rootdir, m_statedir, name, info, netMode, netNsPath, sandboxConfig, image); ++ sandbox = std::make_shared<Sandbox>(id, m_rootdir, m_statedir, name, info, netMode, netNsPath, sandboxConfig, image); + #endif + if (sandbox == nullptr) { + ERROR("Failed to malloc for sandbox: %s", name.c_str()); +@@ -485,12 +484,12 @@ auto SandboxManager::LoadSandbox(std::string &id) -> std::shared_ptr<Sandbox> + + #ifdef ENABLE_SANDBOXER + if (IsShimSandbox(id, m_rootdir)) { +- sandbox = std::make_shared<ShimSandbox>(id, m_rootdir, m_statedir); ++ sandbox = std::make_shared<Sandbox>(id, m_rootdir, m_statedir); + } else { + sandbox = std::make_shared<SandboxerSandbox>(id, m_rootdir, m_statedir); + } + #else +- sandbox = std::make_shared<ShimSandbox>(id, m_rootdir, m_statedir); ++ sandbox = std::make_shared<Sandbox>(id, m_rootdir, m_statedir); + #endif + if (sandbox == nullptr) { + ERROR("Failed to malloc for sandboxes: %s", id.c_str()); +diff --git a/src/daemon/sandbox/sandbox_ops.cc b/src/daemon/sandbox/sandbox_ops.cc +index f50a1033..ae881933 100644 +--- a/src/daemon/sandbox/sandbox_ops.cc ++++ b/src/daemon/sandbox/sandbox_ops.cc +@@ -24,12 +24,6 @@ + #include "sandbox.h" + #include "namespace.h" + #include "utils.h" +-#include "utils_timestamp.h" +-#include "utils_array.h" +- +-const std::string SANDBOX_EXTENSIONS_TASKS = "extensions.tasks"; +-const std::string SANDBOX_TASKS_KEY = "tasks"; +-const std::string SANDBOX_TASKS_TYPEURL = "github.com/containerd/containerd/Tasks"; + + static inline bool validate_sandbox_info(const container_sandbox_info *sandbox) + { +@@ -37,99 +31,6 @@ static inline bool validate_sandbox_info(const container_sandbox_info *sandbox) + sandbox->id != NULL); + } + +-static int generate_ctrl_rootfs(sandbox_task *task, +- const container_config_v2_common_config *config) +-{ +- size_t len = 1; +- if (nullptr == config->base_fs) { +- ERROR("Container %s has no base fs", config->id); +- return -1; +- } +- +- // TODO: rootfs's options left to be configured +- task->rootfs = (sandbox_mount **)util_smart_calloc_s(sizeof(sandbox_mount *), len); +- if (task->rootfs == nullptr) { +- ERROR("Out of memory."); +- return -1; +- } +- task->rootfs[0] = (sandbox_mount *)util_common_calloc_s(sizeof(sandbox_mount)); +- if (task->rootfs[0] == nullptr) { +- ERROR("Out of memory."); +- return -1; +- } +- task->rootfs_len = len; +- task->rootfs[0]->type = util_strdup_s(MOUNT_TYPE_BIND); +- task->rootfs[0]->source = util_strdup_s(config->base_fs); +- +- return 0; +-} +- +-static int do_sandbox_update(std::shared_ptr<sandbox::Sandbox> &sandbox, sandbox_sandbox *apiSandbox) +-{ +- Errors err; +- size_t fields_len = 1; +- __isula_auto_string_array_t string_array *fields = nullptr; +- +- fields = util_string_array_new(fields_len); +- if (fields == nullptr) { +- ERROR("Out of memory."); +- return -1; +- } +- if (util_append_string_array(fields, SANDBOX_EXTENSIONS_TASKS.c_str())) { +- ERROR("Out of memory."); +- return -1; +- } +- +- auto controller = sandbox::ControllerManager::GetInstance()->GetController(sandbox->GetSandboxer()); +- if (nullptr == controller) { +- ERROR("Invalid sandboxer name: %s", sandbox->GetSandboxer().c_str()); +- return -1; +- } +- +- if (!controller->Update(apiSandbox, fields, err)) { +- ERROR("Failed to update in container controller update: %s", err.GetCMessage()); +- return -1; +- } +- +- return 0; +-} +- +-static oci_runtime_spec *clone_oci_runtime_spec(const oci_runtime_spec *oci_spec) +-{ +- __isula_auto_free char *json_str = nullptr; +- __isula_auto_free parser_error err = nullptr; +- oci_runtime_spec *ret = nullptr; +- +- json_str = oci_runtime_spec_generate_json(oci_spec, nullptr, &err); +- if (json_str == nullptr) { +- ERROR("Failed to generate spec json: %s", err); +- return nullptr; +- } +- ret = oci_runtime_spec_parse_data(json_str, nullptr, &err); +- if (ret == nullptr) { +- ERROR("Failed to generate spec: %s", err); +- } +- return ret; +-} +- +-static defs_process *clone_defs_process(defs_process *process_spec) +-{ +- __isula_auto_free char *json_str = nullptr; +- __isula_auto_free parser_error err = nullptr; +- defs_process *ret = nullptr; +- +- json_str = defs_process_generate_json(process_spec, nullptr, &err); +- if (json_str == nullptr) { +- ERROR("Failed to generate process spec json: %s", err); +- return nullptr; +- } +- ret = defs_process_parse_data(json_str, nullptr, &err); +- if (ret == nullptr) { +- ERROR("Failed to generate process spec: %s", err); +- } +- return ret; +-} +- + static std::shared_ptr<sandbox::Sandbox> get_prepare_sandbox(const container_config_v2_common_config *config) + { + if (nullptr == config || nullptr == config->id) { +@@ -151,126 +52,11 @@ static std::shared_ptr<sandbox::Sandbox> get_prepare_sandbox(const container_con + return sandbox; + } + +-static sandbox_sandbox_runtime *init_sandbox_runtime(std::shared_ptr<sandbox::Sandbox> sandbox) +-{ +- sandbox_sandbox_runtime *runtime = nullptr; +- +- auto runtime_wrapper = makeUniquePtrCStructWrapper<sandbox_sandbox_runtime>(free_sandbox_sandbox_runtime); +- if (runtime_wrapper == nullptr) { +- ERROR("Out of memory"); +- return nullptr; +- } +- runtime = runtime_wrapper->get(); +- runtime->name = util_strdup_s(sandbox->GetRuntime().c_str()); +- // Just ignore options for now +- +- return runtime_wrapper->move(); +-} +- +-static json_map_string_string *init_sandbox_labels(std::shared_ptr<sandbox::Sandbox> sandbox) +-{ +- json_map_string_string *labels = nullptr; +- +- auto labels_wrapper = makeUniquePtrCStructWrapper<json_map_string_string>(free_json_map_string_string); +- if (labels_wrapper == nullptr) { +- ERROR("Out of memory"); +- return nullptr; +- } +- labels = labels_wrapper->get(); +- if (append_json_map_string_string(labels, "name", sandbox->GetName().c_str()) != 0) { +- ERROR("Out of memory"); +- return nullptr; +- } +- +- return labels_wrapper->move(); +-} +- +-static defs_map_string_object_any *init_sandbox_extensions(std::shared_ptr<sandbox::Sandbox> sandbox) +-{ +- defs_map_string_object_any *extensions = nullptr; +- size_t len = 1; +- std::string task_json; +- +- auto extensions_wrapper = makeUniquePtrCStructWrapper<defs_map_string_object_any>(free_defs_map_string_object_any); +- if (extensions_wrapper == nullptr) { +- ERROR("Out of memory"); +- return nullptr; +- } +- extensions = extensions_wrapper->get(); +- extensions->keys = (char **)util_smart_calloc_s(sizeof(char *), len); +- if (extensions->keys == nullptr) { +- ERROR("Out of memory."); +- return nullptr; +- } +- extensions->len = len; +- extensions->values = (defs_map_string_object_any_element **) +- util_smart_calloc_s(sizeof(defs_map_string_object_any_element *), len); +- if (extensions->values == nullptr) { +- ERROR("Out of memory."); +- return nullptr; +- } +- extensions->values[0] = (defs_map_string_object_any_element *) +- util_common_calloc_s(sizeof(defs_map_string_object_any_element)); +- if (extensions->values[0] == nullptr) { +- ERROR("Out of memory."); +- return nullptr; +- } +- extensions->values[0]->element = (defs_any *)util_common_calloc_s(sizeof(defs_any)); +- if (extensions->values[0]->element == nullptr) { +- ERROR("Out of memory."); +- return nullptr; +- } +- +- extensions->keys[0] = util_strdup_s(SANDBOX_TASKS_KEY.c_str()); +- task_json = sandbox->GetAnySandboxTasks(); +- if (task_json.empty()) { +- ERROR("Failed to get any sandbox tasks"); +- return nullptr; +- } +- DEBUG("Get any sandbox tasks %s", task_json.c_str()); +- extensions->values[0]->element->type_url = util_strdup_s(SANDBOX_TASKS_TYPEURL.c_str()); +- extensions->values[0]->element->value = reinterpret_cast<uint8_t *>(util_strdup_s(task_json.c_str())); +- extensions->values[0]->element->value_len = strlen(task_json.c_str()); +- +- return extensions_wrapper->move(); +-} +- +-static int init_api_sandbox(std::shared_ptr<sandbox::Sandbox> sandbox, sandbox_sandbox *apiSandbox) +-{ +- apiSandbox->sandbox_id = util_strdup_s(sandbox->GetId().c_str()); +- apiSandbox->runtime = init_sandbox_runtime(sandbox); +- if (apiSandbox->runtime == nullptr) { +- ERROR("Failed to init sandbox runtime"); +- return -1; +- } +- // Just ignore spec +- apiSandbox->labels = init_sandbox_labels(sandbox); +- if (apiSandbox->labels == nullptr) { +- ERROR("Failed to init sandbox runtime"); +- return -1; +- } +- apiSandbox->created_at = sandbox->GetCreatedAt(); +- apiSandbox->updated_at = util_get_now_time_nanos(); +- apiSandbox->extensions = init_sandbox_extensions(sandbox); +- if (apiSandbox->extensions == nullptr) { +- ERROR("Failed to init sandbox runtime"); +- return -1; +- } +- apiSandbox->sandboxer = util_strdup_s(sandbox->GetSandboxer().c_str()); +- +- return 0; +-} + + int sandbox_prepare_container(const container_config_v2_common_config *config, + const oci_runtime_spec *oci_spec, + const char * console_fifos[], bool tty) + { +- sandbox_task *task = nullptr; +- sandbox_sandbox *apiSandbox = nullptr; +- int ret = -1; +- +- INFO("Prepare container for sandbox"); +- + if (nullptr == console_fifos) { + ERROR("Invlaid parameter: console_fifos"); + return -1; +@@ -282,67 +68,13 @@ int sandbox_prepare_container(const container_config_v2_common_config *config, + return -1; + } + +- auto apiSandbox_wrapper = makeUniquePtrCStructWrapper<sandbox_sandbox>(free_sandbox_sandbox); +- if (apiSandbox_wrapper == nullptr) { +- ERROR("Out of memory"); +- return -1; +- } +- apiSandbox = apiSandbox_wrapper->get(); +- auto task_wrapper = makeUniquePtrCStructWrapper<sandbox_task>(free_sandbox_task); +- if (task_wrapper == nullptr) { +- ERROR("Out of memory"); +- return -1; +- } +- task = task_wrapper->get(); +- +- task->task_id = util_strdup_s(config->id); +- task->spec = clone_oci_runtime_spec(oci_spec); +- if (task->spec == nullptr) { +- ERROR("Out of memory."); +- return -1; +- } +- if (generate_ctrl_rootfs(task, config) != 0) { +- ERROR("Invalid rootfs"); +- return -1; +- } +- task->stdin = util_strdup_s((nullptr == console_fifos[0]) ? "" : console_fifos[0]); +- task->stdout = util_strdup_s((nullptr == console_fifos[1]) ? "" : console_fifos[1]); +- task->stderr = util_strdup_s((nullptr == console_fifos[2]) ? "" : console_fifos[2]); +- +- if (!sandbox->AddSandboxTasks(task)) { +- ERROR("Failed to add sandbox %s task.", config->id); +- return -1; +- } +- task = task_wrapper->move(); +- ret = init_api_sandbox(sandbox, apiSandbox); +- if (ret != 0) { +- ERROR("Failed to init %s api sandbox.", config->id); +- goto del_out; +- } +- ret = do_sandbox_update(sandbox, apiSandbox); +- +-del_out: +- if (ret != 0) { +- sandbox->DeleteSandboxTasks(config->id); +- } +- if (!sandbox->SaveSandboxTasks()) { +- ERROR("Failed to Save %s sandbox tasks.", config->id); +- ret = -1; +- } +- +- return ret; ++ return sandbox->PrepareContainer(config->id, config->base_fs, oci_spec, console_fifos); + } + + int sandbox_prepare_exec(const container_config_v2_common_config *config, + const char *exec_id, defs_process *process_spec, + const char * console_fifos[], bool tty) + { +- sandbox_process *process = nullptr; +- sandbox_sandbox *apiSandbox = nullptr; +- int ret = -1; +- +- INFO("Prepare exec for container in sandbox"); +- + if (nullptr == console_fifos) { + ERROR("Invlaid parameter: console_fifos"); + return -1; +@@ -354,116 +86,29 @@ int sandbox_prepare_exec(const container_config_v2_common_config *config, + return -1; + } + +- auto apiSandbox_wrapper = makeUniquePtrCStructWrapper<sandbox_sandbox>(free_sandbox_sandbox); +- if (apiSandbox_wrapper == nullptr) { +- ERROR("Out of memory"); +- return -1; +- } +- apiSandbox = apiSandbox_wrapper->get(); +- auto process_wrapper = makeUniquePtrCStructWrapper<sandbox_process>(free_sandbox_process); +- if (process_wrapper == nullptr) { +- ERROR("Out of memory"); +- return -1; +- } +- process = process_wrapper->get(); +- +- process->exec_id = util_strdup_s(exec_id); +- process->spec = clone_defs_process(process_spec); +- if (process->spec == nullptr) { +- ERROR("Out of memory."); +- return -1; +- } +- process->stdin = util_strdup_s((nullptr == console_fifos[0]) ? "" : console_fifos[0]); +- process->stdout = util_strdup_s((nullptr == console_fifos[1]) ? "" : console_fifos[1]); +- process->stderr = util_strdup_s((nullptr == console_fifos[2]) ? "" : console_fifos[2]); +- +- if (!sandbox->AddSandboxTasksProcess(config->id, process)) { +- ERROR("Failed to add sandbox %s process.", config->id); +- return -1; +- } +- process = process_wrapper->move(); +- ret = init_api_sandbox(sandbox, apiSandbox); +- if (ret != 0) { +- ERROR("Failed to init %s api sandbox.", config->id); +- goto del_out; +- } +- ret = do_sandbox_update(sandbox, apiSandbox); +- +-del_out: +- if (ret != 0) { +- sandbox->DeleteSandboxTasksProcess(config->id, exec_id); +- } +- if (!sandbox->SaveSandboxTasks()) { +- ERROR("Failed to Save %s sandbox tasks.", config->id); +- ret = -1; +- } +- +- return ret; ++ return sandbox->PrepareExec(config->id, exec_id, process_spec, console_fifos); + } + + int sandbox_purge_container(const container_config_v2_common_config *config) + { +- sandbox_sandbox *apiSandbox = nullptr; +- +- INFO("Purge container for sandbox"); +- + auto sandbox = get_prepare_sandbox(config); + if (sandbox == nullptr) { + ERROR("Sandbox not found"); + return -1; + } + +- auto apiSandbox_wrapper = makeUniquePtrCStructWrapper<sandbox_sandbox>(free_sandbox_sandbox); +- if (apiSandbox_wrapper == nullptr) { +- ERROR("Out of memory"); +- return -1; +- } +- apiSandbox = apiSandbox_wrapper->get(); +- +- sandbox->DeleteSandboxTasks(config->id); +- if (!sandbox->SaveSandboxTasks()) { +- ERROR("Failed to Save %s sandbox tasks.", config->id); +- return -1; +- } +- +- if (init_api_sandbox(sandbox, apiSandbox) != 0) { +- ERROR("Failed to init %s api sandbox.", config->id); +- return -1; +- } +- return do_sandbox_update(sandbox, apiSandbox); ++ return sandbox->PurgeContainer(config->id); + } + + int sandbox_purge_exec(const container_config_v2_common_config *config, const char *exec_id) + { +- sandbox_sandbox *apiSandbox = nullptr; +- +- INFO("Purge exec for container in sandbox"); +- + auto sandbox = get_prepare_sandbox(config); + if (sandbox == nullptr) { + ERROR("Sandbox not found"); + return -1; + } + +- auto apiSandbox_wrapper = makeUniquePtrCStructWrapper<sandbox_sandbox>(free_sandbox_sandbox); +- if (apiSandbox_wrapper == nullptr) { +- ERROR("Out of memory"); +- return -1; +- } +- apiSandbox = apiSandbox_wrapper->get(); +- +- sandbox->DeleteSandboxTasksProcess(config->id, exec_id); +- if (!sandbox->SaveSandboxTasks()) { +- ERROR("Failed to Save %s sandbox tasks.", config->id); +- return -1; +- } +- +- if (init_api_sandbox(sandbox, apiSandbox) != 0) { +- ERROR("Failed to init %s api sandbox.", exec_id); +- return -1; +- } +- +- return do_sandbox_update(sandbox, apiSandbox); ++ return sandbox->PurgeExec(config->id, exec_id); + } + + int sandbox_on_sandbox_exit(const char *sandbox_id, int exit_code) +diff --git a/src/daemon/sandbox/sandboxer/sandboxer_sandbox.cc b/src/daemon/sandbox/sandboxer/sandboxer_sandbox.cc +index 726b7b3a..b2e2fb32 100644 +--- a/src/daemon/sandbox/sandboxer/sandboxer_sandbox.cc ++++ b/src/daemon/sandbox/sandboxer/sandboxer_sandbox.cc +@@ -23,13 +23,21 @@ + + #include <isula_libutils/log.h> + #include <isula_libutils/auto_cleanup.h> ++#include <isula_libutils/sandbox_sandbox.h> ++#include <isula_libutils/defs_process.h> + + #include "utils_file.h" + #include "utils.h" + #include "cxxutils.h" ++#include "utils_timestamp.h" ++#include "utils_array.h" + + namespace sandbox { + ++const std::string SANDBOX_EXTENSIONS_TASKS = "extensions.tasks"; ++const std::string SANDBOX_TASKS_KEY = "tasks"; ++const std::string SANDBOX_TASKS_TYPEURL = "github.com/containerd/containerd/Tasks"; ++ + SandboxerSandbox::SandboxerSandbox(const std::string id, const std::string &rootdir, const std::string &statedir, const std::string name, + const RuntimeInfo info, std::string netMode, std::string netNsPath, const runtime::v1::PodSandboxConfig sandboxConfig, + const std::string image):Sandbox(id, rootdir, statedir, name, info, netMode, +@@ -251,4 +259,397 @@ void SandboxerSandbox::DeleteSandboxTasksProcess(const char *containerId, const + iter->second->DeleteSandboxTasksProcess(execId); + } + ++static oci_runtime_spec *clone_oci_runtime_spec(const oci_runtime_spec *oci_spec) ++{ ++ __isula_auto_free char *json_str = nullptr; ++ __isula_auto_free parser_error err = nullptr; ++ oci_runtime_spec *ret = nullptr; ++ ++ json_str = oci_runtime_spec_generate_json(oci_spec, nullptr, &err); ++ if (json_str == nullptr) { ++ ERROR("Failed to generate spec json: %s", err); ++ return nullptr; ++ } ++ ret = oci_runtime_spec_parse_data(json_str, nullptr, &err); ++ if (ret == nullptr) { ++ ERROR("Failed to generate spec: %s", err); ++ } ++ return ret; ++} ++ ++static defs_process *clone_defs_process(defs_process *process_spec) ++{ ++ __isula_auto_free char *json_str = nullptr; ++ __isula_auto_free parser_error err = nullptr; ++ defs_process *ret = nullptr; ++ ++ json_str = defs_process_generate_json(process_spec, nullptr, &err); ++ if (json_str == nullptr) { ++ ERROR("Failed to generate process spec json: %s", err); ++ return nullptr; ++ } ++ ret = defs_process_parse_data(json_str, nullptr, &err); ++ if (ret == nullptr) { ++ ERROR("Failed to generate process spec: %s", err); ++ } ++ return ret; ++} ++ ++auto SandboxerSandbox::GenerateCtrlRootfs(sandbox_task *task, const char *baseFs) -> int ++{ ++ size_t len = 1; ++ if (nullptr == baseFs) { ++ ERROR("Container %s has no base fs", task->task_id); ++ return -1; ++ } ++ ++ // TODO: rootfs's options left to be configured ++ task->rootfs = (sandbox_mount **)util_smart_calloc_s(sizeof(sandbox_mount *), len); ++ if (task->rootfs == nullptr) { ++ ERROR("Out of memory."); ++ return -1; ++ } ++ task->rootfs[0] = (sandbox_mount *)util_common_calloc_s(sizeof(sandbox_mount)); ++ if (task->rootfs[0] == nullptr) { ++ ERROR("Out of memory."); ++ return -1; ++ } ++ task->rootfs_len = len; ++ task->rootfs[0]->type = util_strdup_s(MOUNT_TYPE_BIND); ++ task->rootfs[0]->source = util_strdup_s(baseFs); ++ ++ return 0; ++} ++ ++auto SandboxerSandbox::InitSandboxRuntime() -> sandbox_sandbox_runtime * ++{ ++ sandbox_sandbox_runtime *runtime = nullptr; ++ ++ auto runtime_wrapper = makeUniquePtrCStructWrapper<sandbox_sandbox_runtime>(free_sandbox_sandbox_runtime); ++ if (runtime_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return nullptr; ++ } ++ runtime = runtime_wrapper->get(); ++ runtime->name = util_strdup_s(GetRuntime().c_str()); ++ // Just ignore options for now ++ ++ return runtime_wrapper->move(); ++} ++ ++auto SandboxerSandbox::InitSandboxLabels() -> json_map_string_string * ++{ ++ json_map_string_string *labels = nullptr; ++ ++ auto labels_wrapper = makeUniquePtrCStructWrapper<json_map_string_string>(free_json_map_string_string); ++ if (labels_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return nullptr; ++ } ++ labels = labels_wrapper->get(); ++ if (append_json_map_string_string(labels, "name", GetName().c_str()) != 0) { ++ ERROR("Out of memory"); ++ return nullptr; ++ } ++ ++ return labels_wrapper->move(); ++} ++ ++auto SandboxerSandbox::InitSandboxExtensions() -> defs_map_string_object_any * ++{ ++ defs_map_string_object_any *extensions = nullptr; ++ size_t len = 1; ++ std::string task_json; ++ ++ auto extensions_wrapper = makeUniquePtrCStructWrapper<defs_map_string_object_any>(free_defs_map_string_object_any); ++ if (extensions_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return nullptr; ++ } ++ extensions = extensions_wrapper->get(); ++ ++ extensions->keys = (char **)util_smart_calloc_s(sizeof(char *), len); ++ if (extensions->keys == nullptr) { ++ ERROR("Out of memory."); ++ return nullptr; ++ } ++ extensions->values = (defs_map_string_object_any_element **) ++ util_smart_calloc_s(sizeof(defs_map_string_object_any_element *), len); ++ if (extensions->values == nullptr) { ++ ERROR("Out of memory."); ++ free(extensions->keys); ++ extensions->keys = nullptr; ++ return nullptr; ++ } ++ extensions->len = len; ++ ++ extensions->values[0] = (defs_map_string_object_any_element *) ++ util_common_calloc_s(sizeof(defs_map_string_object_any_element)); ++ if (extensions->values[0] == nullptr) { ++ ERROR("Out of memory."); ++ return nullptr; ++ } ++ extensions->values[0]->element = (defs_any *)util_common_calloc_s(sizeof(defs_any)); ++ if (extensions->values[0]->element == nullptr) { ++ ERROR("Out of memory."); ++ return nullptr; ++ } ++ ++ extensions->keys[0] = util_strdup_s(SANDBOX_TASKS_KEY.c_str()); ++ task_json = GetAnySandboxTasks(); ++ if (task_json.empty()) { ++ ERROR("Failed to get any sandbox tasks"); ++ return nullptr; ++ } ++ DEBUG("Get any sandbox tasks %s", task_json.c_str()); ++ extensions->values[0]->element->type_url = util_strdup_s(SANDBOX_TASKS_TYPEURL.c_str()); ++ extensions->values[0]->element->value = reinterpret_cast<uint8_t *>(util_strdup_s(task_json.c_str())); ++ extensions->values[0]->element->value_len = strlen(task_json.c_str()); ++ ++ return extensions_wrapper->move(); ++} ++ ++auto SandboxerSandbox::InitApiSandbox(sandbox_sandbox *apiSandbox) -> int ++{ ++ apiSandbox->sandbox_id = util_strdup_s(GetId().c_str()); ++ apiSandbox->runtime = InitSandboxRuntime(); ++ if (apiSandbox->runtime == nullptr) { ++ ERROR("Failed to init sandbox runtime"); ++ return -1; ++ } ++ // Just ignore spec ++ apiSandbox->labels = InitSandboxLabels(); ++ if (apiSandbox->labels == nullptr) { ++ ERROR("Failed to init sandbox runtime"); ++ return -1; ++ } ++ apiSandbox->created_at = GetCreatedAt(); ++ apiSandbox->updated_at = util_get_now_time_nanos(); ++ apiSandbox->extensions = InitSandboxExtensions(); ++ if (apiSandbox->extensions == nullptr) { ++ ERROR("Failed to init sandbox runtime"); ++ return -1; ++ } ++ apiSandbox->sandboxer = util_strdup_s(GetSandboxer().c_str()); ++ ++ return 0; ++} ++ ++auto SandboxerSandbox::DoSandboxUpdate(sandbox_sandbox *apiSandbox) -> int ++{ ++ Errors err; ++ size_t fields_len = 1; ++ __isula_auto_string_array_t string_array *fields = nullptr; ++ ++ fields = util_string_array_new(fields_len); ++ if (fields == nullptr) { ++ ERROR("Out of memory."); ++ return -1; ++ } ++ if (util_append_string_array(fields, SANDBOX_EXTENSIONS_TASKS.c_str())) { ++ ERROR("Out of memory."); ++ return -1; ++ } ++ ++ auto controller = sandbox::ControllerManager::GetInstance()->GetController(GetSandboxer()); ++ if (nullptr == controller) { ++ ERROR("Invalid sandboxer name: %s", GetSandboxer().c_str()); ++ return -1; ++ } ++ ++ if (!controller->Update(apiSandbox, fields, err)) { ++ ERROR("Failed to update in container controller update: %s", err.GetCMessage()); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++auto SandboxerSandbox::PrepareContainer(const char *containerId, const char *baseFs, ++ const oci_runtime_spec *ociSpec, ++ const char *consoleFifos[]) -> int ++{ ++ sandbox_task *task = nullptr; ++ sandbox_sandbox *apiSandbox = nullptr; ++ ++ INFO("Prepare container for sandbox"); ++ ++ if (nullptr == consoleFifos) { ++ ERROR("Invlaid parameter: consoleFifos"); ++ return -1; ++ } ++ ++ auto apiSandbox_wrapper = makeUniquePtrCStructWrapper<sandbox_sandbox>(free_sandbox_sandbox); ++ if (apiSandbox_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ apiSandbox = apiSandbox_wrapper->get(); ++ auto task_wrapper = makeUniquePtrCStructWrapper<sandbox_task>(free_sandbox_task); ++ if (task_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ task = task_wrapper->get(); ++ ++ task->task_id = util_strdup_s(containerId); ++ task->spec = clone_oci_runtime_spec(ociSpec); ++ if (task->spec == nullptr) { ++ ERROR("Out of memory."); ++ return -1; ++ } ++ if (GenerateCtrlRootfs(task, baseFs) != 0) { ++ ERROR("Invalid rootfs"); ++ return -1; ++ } ++ task->stdin = util_strdup_s((nullptr == consoleFifos[0]) ? "" : consoleFifos[0]); ++ task->stdout = util_strdup_s((nullptr == consoleFifos[1]) ? "" : consoleFifos[1]); ++ task->stderr = util_strdup_s((nullptr == consoleFifos[2]) ? "" : consoleFifos[2]); ++ ++ if (!AddSandboxTasks(task)) { ++ ERROR("Failed to add sandbox %s task.", containerId); ++ return -1; ++ } ++ task = task_wrapper->move(); ++ if (InitApiSandbox(apiSandbox) != 0) { ++ ERROR("Failed to init %s api sandbox.", containerId); ++ goto del_out; ++ } ++ if (DoSandboxUpdate(apiSandbox) != 0) { ++ ERROR("Failed to update %s api sandbox.", containerId); ++ goto del_out; ++ } ++ if (!SaveSandboxTasks()) { ++ ERROR("Failed to Save %s sandbox tasks.", containerId); ++ (void)PurgeContainer(containerId); ++ return -1; ++ } ++ return 0; ++ ++del_out: ++ DeleteSandboxTasks(containerId); ++ return -1; ++} ++ ++auto SandboxerSandbox::PrepareExec(const char *containerId, const char *execId, ++ defs_process *processSpec, const char *consoleFifos[]) -> int ++{ ++ sandbox_process *process = nullptr; ++ sandbox_sandbox *apiSandbox = nullptr; ++ ++ INFO("Prepare exec for container in sandbox"); ++ ++ if (nullptr == consoleFifos) { ++ ERROR("Invlaid parameter: consoleFifos"); ++ return -1; ++ } ++ ++ auto apiSandbox_wrapper = makeUniquePtrCStructWrapper<sandbox_sandbox>(free_sandbox_sandbox); ++ if (apiSandbox_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ apiSandbox = apiSandbox_wrapper->get(); ++ auto process_wrapper = makeUniquePtrCStructWrapper<sandbox_process>(free_sandbox_process); ++ if (process_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ process = process_wrapper->get(); ++ ++ process->exec_id = util_strdup_s(execId); ++ process->spec = clone_defs_process(processSpec); ++ if (process->spec == nullptr) { ++ ERROR("Out of memory."); ++ return -1; ++ } ++ process->stdin = util_strdup_s((nullptr == consoleFifos[0]) ? "" : consoleFifos[0]); ++ process->stdout = util_strdup_s((nullptr == consoleFifos[1]) ? "" : consoleFifos[1]); ++ process->stderr = util_strdup_s((nullptr == consoleFifos[2]) ? "" : consoleFifos[2]); ++ ++ if (!AddSandboxTasksProcess(containerId, process)) { ++ ERROR("Failed to add sandbox %s process.", containerId); ++ return -1; ++ } ++ process = process_wrapper->move(); ++ if (InitApiSandbox(apiSandbox) != 0) { ++ ERROR("Failed to init %s api sandbox.", containerId); ++ goto del_out; ++ } ++ if (DoSandboxUpdate(apiSandbox) != 0) { ++ ERROR("Failed to init %s api sandbox.", containerId); ++ goto del_out; ++ } ++ if (!SaveSandboxTasks()) { ++ ERROR("Failed to Save %s sandbox tasks.", containerId); ++ (void)PurgeExec(containerId, execId); ++ return -1; ++ } ++ return 0; ++ ++del_out: ++ DeleteSandboxTasksProcess(containerId, execId); ++ return -1; ++} ++ ++auto SandboxerSandbox::PurgeContainer(const char *containerId) -> int ++{ ++ sandbox_sandbox *apiSandbox = nullptr; ++ ++ INFO("Purge container for sandbox"); ++ ++ auto apiSandbox_wrapper = makeUniquePtrCStructWrapper<sandbox_sandbox>(free_sandbox_sandbox); ++ if (apiSandbox_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ apiSandbox = apiSandbox_wrapper->get(); ++ ++ DeleteSandboxTasks(containerId); ++ ++ if (InitApiSandbox(apiSandbox) != 0) { ++ ERROR("Failed to init %s api sandbox.", containerId); ++ return -1; ++ } ++ if (DoSandboxUpdate(apiSandbox) != 0) { ++ ERROR("Failed to update %s api sandbox.", containerId); ++ return -1; ++ } ++ if (!SaveSandboxTasks()) { ++ ERROR("Failed to Save %s sandbox tasks.", containerId); ++ return -1; ++ } ++ return 0; ++} ++ ++auto SandboxerSandbox::PurgeExec(const char *containerId, const char *execId) -> int ++{ ++ sandbox_sandbox *apiSandbox = nullptr; ++ ++ INFO("Purge exec for container in sandbox"); ++ ++ auto apiSandbox_wrapper = makeUniquePtrCStructWrapper<sandbox_sandbox>(free_sandbox_sandbox); ++ if (apiSandbox_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ apiSandbox = apiSandbox_wrapper->get(); ++ ++ DeleteSandboxTasksProcess(containerId, execId); ++ ++ if (InitApiSandbox(apiSandbox) != 0) { ++ ERROR("Failed to init %s api sandbox.", execId); ++ return -1; ++ } ++ if (DoSandboxUpdate(apiSandbox) != 0) { ++ ERROR("Failed to update %s api sandbox.", execId); ++ return -1; ++ } ++ if (!SaveSandboxTasks()) { ++ ERROR("Failed to Save %s sandbox tasks.", containerId); ++ return -1; ++ } ++ return 0; ++} ++ + } +\ No newline at end of file +diff --git a/src/daemon/sandbox/sandboxer/sandboxer_sandbox.h b/src/daemon/sandbox/sandboxer/sandboxer_sandbox.h +index 552eb328..37a96cd6 100644 +--- a/src/daemon/sandbox/sandboxer/sandboxer_sandbox.h ++++ b/src/daemon/sandbox/sandboxer/sandboxer_sandbox.h +@@ -33,24 +33,39 @@ public: + const runtime::v1::PodSandboxConfig sandboxConfig = runtime::v1::PodSandboxConfig::default_instance(), + const std::string image = ""); + virtual ~SandboxerSandbox() = default; +- +- // for sandbox api update +- auto GetTasksJsonPath() -> std::string; ++ + void LoadSandboxTasks() override; +- auto SaveSandboxTasks() -> bool override; +- auto AddSandboxTasks(sandbox_task *task) -> bool override; +- auto GetAnySandboxTasks() -> std::string override; +- void DeleteSandboxTasks(const char *containerId) override; +- auto AddSandboxTasksProcess(const char *containerId, sandbox_process *processes) -> bool override; +- void DeleteSandboxTasksProcess(const char *containerId, const char *execId) override; ++ ++ auto PrepareContainer(const char *containerId, const char *baseFs, ++ const oci_runtime_spec *ociSpec, ++ const char *consoleFifos[]) -> int override; ++ auto PrepareExec(const char *containerId, const char *execId, ++ defs_process *processSpec, const char *consoleFifos[]) -> int override; ++ auto PurgeContainer(const char *containerId) -> int override; ++ auto PurgeExec(const char *containerId, const char *execId) -> int override; + + private: ++ auto GetTasksJsonPath() -> std::string; ++ auto SaveSandboxTasks() -> bool; ++ auto AddSandboxTasks(sandbox_task *task) -> bool; ++ auto GetAnySandboxTasks() -> std::string; ++ void DeleteSandboxTasks(const char *containerId); ++ auto AddSandboxTasksProcess(const char *containerId, sandbox_process *processes) -> bool; ++ void DeleteSandboxTasksProcess(const char *containerId, const char *execId); ++ + auto AddTaskById(const char *task_id, sandbox_task *task) -> bool; + auto ReadSandboxTasksJson() -> sandbox_tasks *; + auto WriteSandboxTasksJson(std::string &tasks_json) -> bool; + auto DeleteSandboxTasksJson() -> bool; + void AddSandboxTasksByArray(sandbox_tasks *tasksArray); + ++ auto GenerateCtrlRootfs(sandbox_task *task, const char *baseFs) -> int; ++ auto InitSandboxRuntime() -> sandbox_sandbox_runtime *; ++ auto InitSandboxLabels() -> json_map_string_string *; ++ auto InitSandboxExtensions() -> defs_map_string_object_any *; ++ auto InitApiSandbox(sandbox_sandbox *apiSandbox) -> int; ++ auto DoSandboxUpdate(sandbox_sandbox *apiSandbox) -> int; ++ + private: + // use m_tasksMutex to ensure the correctness of the tasks + RWMutex m_tasksMutex; +diff --git a/src/daemon/sandbox/shim/shim_sandbox.cc b/src/daemon/sandbox/shim/shim_sandbox.cc +deleted file mode 100644 +index 2efb8d7c..00000000 +--- a/src/daemon/sandbox/shim/shim_sandbox.cc ++++ /dev/null +@@ -1,65 +0,0 @@ +-/****************************************************************************** +- * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. +- * iSulad licensed under the Mulan PSL v2. +- * You can use this software according to the terms and conditions of the Mulan PSL v2. +- * You may obtain a copy of Mulan PSL v2 at: +- * http://license.coscl.org.cn/MulanPSL2 +- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +- * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +- * PURPOSE. +- * See the Mulan PSL v2 for more details. +- * Author: liuxu +- * Create: 2024-11-20 +- * Description: provide sandboxer sandbox class definition +- *********************************************************************************/ +-#include "shim_sandbox.h" +- +-#include <unistd.h> +-#include <string> +- +-#include <isula_libutils/log.h> +-#include <isula_libutils/auto_cleanup.h> +- +- +-namespace sandbox { +- +-ShimSandbox::ShimSandbox(const std::string id, const std::string &rootdir, const std::string &statedir, const std::string name, +- const RuntimeInfo info, std::string netMode, std::string netNsPath, const runtime::v1::PodSandboxConfig sandboxConfig, +- const std::string image):Sandbox(id, rootdir, statedir, name, info, netMode, +- netNsPath, sandboxConfig, image) +-{ +-} +- +-void ShimSandbox::LoadSandboxTasks() +-{ +-} +- +-auto ShimSandbox::SaveSandboxTasks() -> bool +-{ +- return true; +-} +- +-auto ShimSandbox::AddSandboxTasks(sandbox_task *task) -> bool +-{ +- return true; +-} +- +-auto ShimSandbox::GetAnySandboxTasks() -> std::string +-{ +- return std::string("Nothing for shim."); +-} +- +-void ShimSandbox::DeleteSandboxTasks(const char *containerId) +-{ +-} +- +-auto ShimSandbox::AddSandboxTasksProcess(const char *containerId, sandbox_process *processes) -> bool +-{ +- return true; +-} +- +-void ShimSandbox::DeleteSandboxTasksProcess(const char *containerId, const char *execId) +-{ +-} +- +-} +\ No newline at end of file +diff --git a/src/daemon/sandbox/shim/shim_sandbox.h b/src/daemon/sandbox/shim/shim_sandbox.h +deleted file mode 100644 +index 82da0573..00000000 +--- a/src/daemon/sandbox/shim/shim_sandbox.h ++++ /dev/null +@@ -1,49 +0,0 @@ +-/****************************************************************************** +- * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. +- * iSulad licensed under the Mulan PSL v2. +- * You can use this software according to the terms and conditions of the Mulan PSL v2. +- * You may obtain a copy of Mulan PSL v2 at: +- * http://license.coscl.org.cn/MulanPSL2 +- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +- * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +- * PURPOSE. +- * See the Mulan PSL v2 for more details. +- * Author: liuxu +- * Create: 2024-11-20 +- * Description: provide shim sandbox class definition +- *********************************************************************************/ +- +-#ifndef DAEMON_SANDBOX_SHIM_SANDBOX_H +-#define DAEMON_SANDBOX_SHIM_SANDBOX_H +- +-#include <string> +-#include <mutex> +-#include <google/protobuf/map.h> +- +-#include "read_write_lock.h" +-#include "sandbox_task.h" +-#include "sandbox.h" +- +-namespace sandbox { +- +-class ShimSandbox : public Sandbox { +-public: +- ShimSandbox(const std::string id, const std::string &rootdir, const std::string &statedir, const std::string name = "", +- const RuntimeInfo info = {"", "", ""}, std::string netMode = DEFAULT_NETMODE, std::string netNsPath = "", +- const runtime::v1::PodSandboxConfig sandboxConfig = runtime::v1::PodSandboxConfig::default_instance(), +- const std::string image = ""); +- virtual ~ShimSandbox() = default; +- +- // for sandbox api update +- void LoadSandboxTasks() override; +- auto SaveSandboxTasks() -> bool override; +- auto AddSandboxTasks(sandbox_task *task) -> bool override; +- auto GetAnySandboxTasks() -> std::string override; +- void DeleteSandboxTasks(const char *containerId) override; +- auto AddSandboxTasksProcess(const char *containerId, sandbox_process *processes) -> bool override; +- void DeleteSandboxTasksProcess(const char *containerId, const char *execId) override; +-}; +- +-} // namespace sandbox +- +-#endif // DAEMON_SANDBOX_SHIM_SANDBOX_H +\ No newline at end of file +-- +2.23.0 + diff --git a/0168-UT-del-shim_sandbox-and-change-sandbox-ops.patch b/0168-UT-del-shim_sandbox-and-change-sandbox-ops.patch new file mode 100644 index 0000000..e9bba80 --- /dev/null +++ b/0168-UT-del-shim_sandbox-and-change-sandbox-ops.patch @@ -0,0 +1,348 @@ +From 1e9031cc064f6980250287641e6b3311af755485 Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Sat, 30 Nov 2024 09:50:33 +0800 +Subject: [PATCH 02/11] UT: del shim_sandbox and change sandbox ops + +Signed-off-by: liuxu <liuxu156@huawei.com> +--- + test/mocks/sandbox_mock.cc | 26 ++++++++ + test/mocks/sandbox_mock.h | 9 +++ + test/mocks/sandboxer_sandbox_mock.cc | 35 +++++----- + test/mocks/sandboxer_sandbox_mock.h | 13 ++-- + test/mocks/shim_sandbox_mock.cc | 72 --------------------- + test/mocks/shim_sandbox_mock.h | 43 ------------ + test/sandbox/sandbox/CMakeLists.txt | 1 - + test/sandbox/sandbox/sandbox_ut.cc | 5 +- + test/sandbox/sandbox_manager/CMakeLists.txt | 1 - + 9 files changed, 61 insertions(+), 144 deletions(-) + delete mode 100644 test/mocks/shim_sandbox_mock.cc + delete mode 100644 test/mocks/shim_sandbox_mock.h + +diff --git a/test/mocks/sandbox_mock.cc b/test/mocks/sandbox_mock.cc +index 9db57a93..cce5a1b6 100644 +--- a/test/mocks/sandbox_mock.cc ++++ b/test/mocks/sandbox_mock.cc +@@ -221,4 +221,30 @@ bool Sandbox::Remove(Errors &error) + } + return true; + } ++ ++void Sandbox::LoadSandboxTasks() {} ++ ++auto Sandbox::PrepareContainer(const char *containerId, const char *baseFs, ++ const oci_runtime_spec *ociSpec, ++ const char *consoleFifos[]) -> int ++{ ++ return 0; ++} ++ ++auto Sandbox::PrepareExec(const char *containerId, const char *execId, ++ defs_process *processSpec, const char *consoleFifos[]) -> int ++{ ++ return 0; ++} ++ ++auto Sandbox::PurgeContainer(const char *containerId) -> int ++{ ++ return 0; ++} ++ ++auto Sandbox::PurgeExec(const char *containerId, const char *execId) -> int ++{ ++ return 0; ++} ++ + } +\ No newline at end of file +diff --git a/test/mocks/sandbox_mock.h b/test/mocks/sandbox_mock.h +index 98f40ad2..4908bcd9 100644 +--- a/test/mocks/sandbox_mock.h ++++ b/test/mocks/sandbox_mock.h +@@ -58,6 +58,15 @@ public: + MOCK_METHOD2(Stop, bool(uint32_t timeoutSecs, Errors &error)); + MOCK_METHOD1(Remove, bool(Errors &error)); + MOCK_METHOD1(Status, void(runtime::v1::PodSandboxStatus &status)); ++ ++ MOCK_METHOD0(LoadSandboxTasks, void()); ++ MOCK_METHOD4(PrepareContainer, int(const char *containerId, const char *baseFs, ++ const oci_runtime_spec *ociSpec, ++ const char *consoleFifos[])); ++ MOCK_METHOD4(PrepareExec, int(const char *containerId, const char *execId, ++ defs_process *processSpec, const char *consoleFifos[])); ++ MOCK_METHOD1(PurgeContainer, int(const char *containerId)); ++ MOCK_METHOD2(PurgeExec, int(const char *containerId, const char *execId)); + }; + + void MockSandbox_SetMock(MockSandbox *mock); +diff --git a/test/mocks/sandboxer_sandbox_mock.cc b/test/mocks/sandboxer_sandbox_mock.cc +index cce58842..6ebe2820 100644 +--- a/test/mocks/sandboxer_sandbox_mock.cc ++++ b/test/mocks/sandboxer_sandbox_mock.cc +@@ -33,40 +33,39 @@ void MockSandboxerSandbox_SetMock(MockSandboxerSandbox *mock) + + void SandboxerSandbox::LoadSandboxTasks() {} + +-auto SandboxerSandbox::SaveSandboxTasks() -> bool ++auto SandboxerSandbox::PrepareContainer(const char *containerId, const char *baseFs, ++ const oci_runtime_spec *ociSpec, ++ const char *consoleFifos[]) -> int + { + if (g_sandboxer_sandbox_mock != nullptr) { +- return g_sandboxer_sandbox_mock->SaveSandboxTasks(); ++ return g_sandboxer_sandbox_mock->PrepareContainer(containerId, baseFs, ociSpec, consoleFifos); + } +- return true; +-} ++ return 0; ++} + +-auto SandboxerSandbox::AddSandboxTasks(sandbox_task *task) -> bool ++auto SandboxerSandbox::PrepareExec(const char *containerId, const char *execId, ++ defs_process *processSpec, const char *consoleFifos[]) -> int + { + if (g_sandboxer_sandbox_mock != nullptr) { +- return g_sandboxer_sandbox_mock->AddSandboxTasks(task); ++ return g_sandboxer_sandbox_mock->PrepareExec(containerId, execId, processSpec, consoleFifos); + } +- return true; +-} ++ return 0; ++} + +-auto SandboxerSandbox::GetAnySandboxTasks() -> std::string ++auto SandboxerSandbox::PurgeContainer(const char *containerId) -> int + { + if (g_sandboxer_sandbox_mock != nullptr) { +- return g_sandboxer_sandbox_mock->GetAnySandboxTasks(); ++ return g_sandboxer_sandbox_mock->PurgeContainer(containerId); + } +- return std::string("Nothing for sandboxer."); ++ return 0; + } + +-void SandboxerSandbox::DeleteSandboxTasks(const char *containerId) {} +- +-auto SandboxerSandbox::AddSandboxTasksProcess(const char *containerId, sandbox_process *processes) -> bool ++auto SandboxerSandbox::PurgeExec(const char *containerId, const char *execId) -> int + { + if (g_sandboxer_sandbox_mock != nullptr) { +- return g_sandboxer_sandbox_mock->AddSandboxTasksProcess(containerId, processes); ++ return g_sandboxer_sandbox_mock->PurgeExec(containerId, execId); + } +- return true; ++ return 0; + } + +-void SandboxerSandbox::DeleteSandboxTasksProcess(const char *containerId, const char *execId) {} +- + } +\ No newline at end of file +diff --git a/test/mocks/sandboxer_sandbox_mock.h b/test/mocks/sandboxer_sandbox_mock.h +index 4f76e5fc..020fe4d6 100644 +--- a/test/mocks/sandboxer_sandbox_mock.h ++++ b/test/mocks/sandboxer_sandbox_mock.h +@@ -28,12 +28,13 @@ public: + virtual ~MockSandboxerSandbox() = default; + + MOCK_METHOD0(LoadSandboxTasks, void()); +- MOCK_METHOD0(SaveSandboxTasks, bool()); +- MOCK_METHOD1(AddSandboxTasks, bool(sandbox_task *task)); +- MOCK_METHOD0(GetAnySandboxTasks, std::string()); +- MOCK_METHOD1(DeleteSandboxTasks, void(const char *containerId)); +- MOCK_METHOD2(AddSandboxTasksProcess, bool(const char *containerId, sandbox_process *processes)); +- MOCK_METHOD2(DeleteSandboxTasksProcess, void(const char *containerId, const char *execId)); ++ MOCK_METHOD4(PrepareContainer, int(const char *containerId, const char *baseFs, ++ const oci_runtime_spec *ociSpec, ++ const char *consoleFifos[])); ++ MOCK_METHOD4(PrepareExec, int(const char *containerId, const char *execId, ++ defs_process *processSpec, const char *consoleFifos[])); ++ MOCK_METHOD1(PurgeContainer, int(const char *containerId)); ++ MOCK_METHOD2(PurgeExec, int(const char *containerId, const char *execId)); + }; + + void MockSandboxerSandbox_SetMock(MockSandboxerSandbox *mock); +diff --git a/test/mocks/shim_sandbox_mock.cc b/test/mocks/shim_sandbox_mock.cc +deleted file mode 100644 +index ccefb424..00000000 +--- a/test/mocks/shim_sandbox_mock.cc ++++ /dev/null +@@ -1,72 +0,0 @@ +-/****************************************************************************** +- * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +- * iSulad licensed under the Mulan PSL v2. +- * You can use this software according to the terms and conditions of the Mulan PSL v2. +- * You may obtain a copy of Mulan PSL v2 at: +- * http://license.coscl.org.cn/MulanPSL2 +- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +- * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +- * PURPOSE. +- * See the Mulan PSL v2 for more details. +- * Author: liuxu +- * Create: 2024-11-21 +- * Description: provide sandbox mock +- ******************************************************************************/ +- +-#include <gmock/gmock.h> +-#include "shim_sandbox_mock.h" +- +-namespace sandbox { +-MockShimSandbox *g_shim_sandbox_mock = nullptr; +- +-ShimSandbox::ShimSandbox(const std::string id, const std::string &rootdir, const std::string &statedir, const std::string name, +- const RuntimeInfo info, std::string netMode, std::string netNsPath, const runtime::v1::PodSandboxConfig sandboxConfig, +- std::string image):Sandbox(id, rootdir, statedir, name, info, netMode, +- netNsPath, sandboxConfig, image) +-{ +-} +- +-void MockShimSandbox_SetMock(MockShimSandbox *mock) +-{ +- g_shim_sandbox_mock = mock; +-} +- +-void ShimSandbox::LoadSandboxTasks() {} +- +-auto ShimSandbox::SaveSandboxTasks() -> bool +-{ +- if (g_shim_sandbox_mock != nullptr) { +- return g_shim_sandbox_mock->SaveSandboxTasks(); +- } +- return true; +-} +- +-auto ShimSandbox::AddSandboxTasks(sandbox_task *task) -> bool +-{ +- if (g_shim_sandbox_mock != nullptr) { +- return g_shim_sandbox_mock->AddSandboxTasks(task); +- } +- return true; +-} +- +-auto ShimSandbox::GetAnySandboxTasks() -> std::string +-{ +- if (g_shim_sandbox_mock != nullptr) { +- return g_shim_sandbox_mock->GetAnySandboxTasks(); +- } +- return std::string("Nothing for shim."); +-} +- +-void ShimSandbox::DeleteSandboxTasks(const char *containerId) {} +- +-auto ShimSandbox::AddSandboxTasksProcess(const char *containerId, sandbox_process *processes) -> bool +-{ +- if (g_shim_sandbox_mock != nullptr) { +- return g_shim_sandbox_mock->AddSandboxTasksProcess(containerId, processes); +- } +- return true; +-} +- +-void ShimSandbox::DeleteSandboxTasksProcess(const char *containerId, const char *execId) {} +- +-} +\ No newline at end of file +diff --git a/test/mocks/shim_sandbox_mock.h b/test/mocks/shim_sandbox_mock.h +deleted file mode 100644 +index 1b16a4cc..00000000 +--- a/test/mocks/shim_sandbox_mock.h ++++ /dev/null +@@ -1,43 +0,0 @@ +-/****************************************************************************** +- * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +- * iSulad licensed under the Mulan PSL v2. +- * You can use this software according to the terms and conditions of the Mulan PSL v2. +- * You may obtain a copy of Mulan PSL v2 at: +- * http://license.coscl.org.cn/MulanPSL2 +- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +- * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +- * PURPOSE. +- * See the Mulan PSL v2 for more details. +- * Author: liuxu +- * Create: 2024-11-21 +- * Description: provide sandbox mock +- ******************************************************************************/ +- +-#ifndef _ISULAD_TEST_MOCKS_SHIM_SANDBOX_MOCK_H +-#define _ISULAD_TEST_MOCKS_SHIM_SANDBOX_MOCK_H +- +-#include <gmock/gmock.h> +-#include "sandbox_mock.h" +-#include "shim_sandbox.h" +- +-namespace sandbox { +- +-class MockShimSandbox : public MockSandbox { +-public: +- MockShimSandbox() = default; +- virtual ~MockShimSandbox() = default; +- +- MOCK_METHOD0(LoadSandboxTasks, void()); +- MOCK_METHOD0(SaveSandboxTasks, bool()); +- MOCK_METHOD1(AddSandboxTasks, bool(sandbox_task *task)); +- MOCK_METHOD0(GetAnySandboxTasks, std::string()); +- MOCK_METHOD1(DeleteSandboxTasks, void(const char *containerId)); +- MOCK_METHOD2(AddSandboxTasksProcess, bool(const char *containerId, sandbox_process *processes)); +- MOCK_METHOD2(DeleteSandboxTasksProcess, void(const char *containerId, const char *execId)); +-}; +- +-void MockShimSandbox_SetMock(MockShimSandbox *mock); +- +-} +- +-#endif +diff --git a/test/sandbox/sandbox/CMakeLists.txt b/test/sandbox/sandbox/CMakeLists.txt +index 9ee67033..6dd6c3ee 100644 +--- a/test/sandbox/sandbox/CMakeLists.txt ++++ b/test/sandbox/sandbox/CMakeLists.txt +@@ -16,7 +16,6 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/sandbox_task.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/controller_manager.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/sandboxer/controller/sandboxer_controller.cc +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/shim/shim_sandbox.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/id_name_manager.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/isulad_config.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/sandbox/controller/controller_common.cc +diff --git a/test/sandbox/sandbox/sandbox_ut.cc b/test/sandbox/sandbox/sandbox_ut.cc +index 192d46ef..dd84d8fb 100644 +--- a/test/sandbox/sandbox/sandbox_ut.cc ++++ b/test/sandbox/sandbox/sandbox_ut.cc +@@ -16,7 +16,6 @@ + #include <gtest/gtest.h> + + #include "sandbox.h" +-#include "shim_sandbox.h" + + namespace sandbox { + +@@ -41,7 +40,7 @@ TEST_F(SandboxTest, TestDefaultGetters) + std::string name = "test"; + RuntimeInfo info = {"runc", "shim", "kuasar"}; + +- auto sandbox = new ShimSandbox(id, rootdir, statedir, name, info); ++ auto sandbox = new Sandbox(id, rootdir, statedir, name, info); + ASSERT_NE(sandbox, nullptr); + + ASSERT_EQ(sandbox->IsReady(), false); +@@ -67,7 +66,7 @@ TEST_F(SandboxTest, TestGettersAndSetters) + std::string statedir = "/test2/statedir"; + std::string mode = "host"; + +- auto sandbox = new ShimSandbox(id, rootdir, statedir); ++ auto sandbox = new Sandbox(id, rootdir, statedir); + ASSERT_NE(sandbox, nullptr); + + sandbox->SetNetMode(mode); +diff --git a/test/sandbox/sandbox_manager/CMakeLists.txt b/test/sandbox/sandbox_manager/CMakeLists.txt +index 9254263c..a7dd8c9d 100644 +--- a/test/sandbox/sandbox_manager/CMakeLists.txt ++++ b/test/sandbox/sandbox_manager/CMakeLists.txt +@@ -12,7 +12,6 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cpputils/read_write_lock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cpputils/transform.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sandbox_mock.cc +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/shim_sandbox_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sandboxer_sandbox_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/sandbox_manager.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/id_name_manager.c +-- +2.23.0 + diff --git a/0169-add-image-storage-unit-test.patch b/0169-add-image-storage-unit-test.patch new file mode 100644 index 0000000..e5770ee --- /dev/null +++ b/0169-add-image-storage-unit-test.patch @@ -0,0 +1,135 @@ +From 7dfa69162cd5ef01592808df555626a0688e6f4c Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Wed, 11 Dec 2024 19:09:20 +1400 +Subject: [PATCH 03/11] add image storage unit test + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + .../oci/storage/image_store/image_store.c | 4 +- + test/image/oci/storage/images/CMakeLists.txt | 2 + + .../oci/storage/images/storage_images_ut.cc | 51 +++++++++++++++++++ + 3 files changed, 56 insertions(+), 1 deletion(-) + +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.c b/src/daemon/modules/image/oci/storage/image_store/image_store.c +index afe53764..71bf36e0 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_store.c ++++ b/src/daemon/modules/image/oci/storage/image_store/image_store.c +@@ -27,6 +27,8 @@ + #include <pthread.h> + #include <stdlib.h> + ++#include <isula_libutils/utils_macro.h> ++ + #include "utils.h" + #include "utils_file.h" + #include "utils_images.h" +@@ -3004,7 +3006,7 @@ static int do_append_image(storage_image *im) + return 0; + } + +-static void strip_host_prefix(char **name) ++STATIC void strip_host_prefix(char **name) + { + char *new_image_name = NULL; + +diff --git a/test/image/oci/storage/images/CMakeLists.txt b/test/image/oci/storage/images/CMakeLists.txt +index 28e0b505..04e60a69 100644 +--- a/test/image/oci/storage/images/CMakeLists.txt ++++ b/test/image/oci/storage/images/CMakeLists.txt +@@ -2,6 +2,8 @@ project(iSulad_UT) + + SET(EXE storage_images_ut) + ++add_definitions(-DUNIT_TEST=ON) ++ + add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/utils.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/utils_regex.c +diff --git a/test/image/oci/storage/images/storage_images_ut.cc b/test/image/oci/storage/images/storage_images_ut.cc +index ad0084d6..15da586e 100644 +--- a/test/image/oci/storage/images/storage_images_ut.cc ++++ b/test/image/oci/storage/images/storage_images_ut.cc +@@ -46,6 +46,10 @@ using ::testing::AtLeast; + using ::testing::Invoke; + using ::testing::_; + ++extern "C" { ++ void strip_host_prefix(char **name); ++} ++ + std::string GetDirectory() + { + char abs_path[PATH_MAX] { 0x00 }; +@@ -299,11 +303,13 @@ protected: + ASSERT_EQ(image_store_init(&opts), 0); + free(opts.storage_root); + free(opts.driver_name); ++ MockIsuladConf_SetMock(&m_isulad_conf); + } + + void TearDown() override + { + image_store_free(); ++ MockIsuladConf_SetMock(nullptr); + } + + void BackUp() +@@ -325,6 +331,7 @@ protected: + std::vector<std::string> ids { "39891ff67da98ab8540d71320915f33d2eb80ab42908e398472cab3c1ce7ac10", + "e4db68de4ff27c2adfea0c54bbb73a61a42f5b667c326de4d7d5b19ab71c6a3b" }; + char store_real_path[PATH_MAX] = { 0x00 }; ++ NiceMock<MockIsuladConf> m_isulad_conf; + }; + + TEST_F(StorageImagesUnitTest, test_images_load) +@@ -714,3 +721,47 @@ TEST_F(StorageImagesUnitTest, test_image_store_remove_multi_name) + + Restore(); + } ++ ++static isulad_daemon_constants *g_test_isulad_daemon_constants = NULL; ++ ++isulad_daemon_constants *invoke_get_isulad_daemon_constants(void) ++{ ++ g_test_isulad_daemon_constants = (isulad_daemon_constants *)util_common_calloc_s(sizeof(isulad_daemon_constants)); ++ if (g_test_isulad_daemon_constants == NULL) { ++ return NULL; ++ } ++ g_test_isulad_daemon_constants->default_host = util_strdup_s("docker.io"); ++ ++ return g_test_isulad_daemon_constants; ++} ++ ++TEST_F(StorageImagesUnitTest, test_strip_host_prefix) ++{ ++ char *name = util_strdup_s("docker.io/test_image"); ++ std::string test_name = "test_image"; ++ std::string test_name_origin = "docker.io/test_image"; ++ char *null_name = NULL; ++ ++ strip_host_prefix(&name); ++ ASSERT_STREQ(name, test_name_origin.c_str()); ++ ++ EXPECT_CALL(m_isulad_conf, GetIsuladDaemonConstants()).WillRepeatedly(Invoke(invoke_get_isulad_daemon_constants)); ++ ++ strip_host_prefix(&name); ++ ASSERT_STREQ(name, test_name.c_str()); ++ ++ strip_host_prefix(&null_name); ++ ASSERT_EQ(null_name, nullptr); ++ ++ free(name); ++ free_isulad_daemon_constants(g_test_isulad_daemon_constants); ++} ++ ++#ifdef ENABLE_REMOTE_LAYER_STORE ++TEST_F(StorageImagesUnitTest, test_remote_layer_common) ++{ ++ ASSERT_EQ(remote_append_image_by_directory_with_lock(NULL), -1); ++ ASSERT_EQ(remote_remove_image_from_memory_with_lock(NULL), -1); ++ ASSERT_EQ(remote_image_get_top_layer_from_json(NULL), nullptr); ++} ++#endif +-- +2.23.0 + diff --git a/0170-fix-some-bad-code.patch b/0170-fix-some-bad-code.patch new file mode 100644 index 0000000..9be0493 --- /dev/null +++ b/0170-fix-some-bad-code.patch @@ -0,0 +1,165 @@ +From 0340a8248e8a4fb133ab3638679755d8590dafae Mon Sep 17 00:00:00 2001 +From: xuxuepeng <xuxuepeng1@huawei.com> +Date: Wed, 11 Dec 2024 13:03:21 +0800 +Subject: [PATCH 04/11] fix some bad code + +Signed-off-by: xuxuepeng <xuxuepeng1@huawei.com> +--- + .../storage/layer_store/graphdriver/driver.c | 19 +++++---------- + .../graphdriver/overlay2/driver_overlay2.c | 24 ++++++++++++++----- + 2 files changed, 24 insertions(+), 19 deletions(-) + +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c +index 94235b80..99fd573c 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c +@@ -121,36 +121,31 @@ static inline void driver_unlock() + + int graphdriver_init(const struct storage_module_init_options *opts) + { +- int ret = 0; + size_t i = 0; + char driver_home[PATH_MAX] = { 0 }; + + if (opts == NULL || opts->storage_root == NULL || opts->driver_name == NULL) { +- ret = -1; +- goto out; ++ return -1; + } + + int nret = snprintf(driver_home, PATH_MAX, "%s/%s", opts->storage_root, opts->driver_name); + if (nret < 0 || (size_t)nret >= PATH_MAX) { + ERROR("Sprintf graph driver path failed"); +- ret = -1; +- goto out; ++ return -1; + } + + for (i = 0; i < g_numdrivers; i++) { + if (strcmp(opts->driver_name, g_drivers[i].name) == 0) { + if (pthread_rwlock_init(&(g_drivers[i].rwlock), NULL) != 0) { + ERROR("Failed to init driver rwlock"); +- ret = -1; +- goto out; ++ return -1; + } + #ifdef ENABLE_REMOTE_LAYER_STORE + g_drivers[i].enable_remote_layer = opts->enable_remote_layer; + #endif + if (g_drivers[i].ops->init(&g_drivers[i], driver_home, (const char **)opts->driver_opts, + opts->driver_opts_len) != 0) { +- ret = -1; +- goto out; ++ return -1; + } + g_graphdriver = &g_drivers[i]; + break; +@@ -159,12 +154,10 @@ int graphdriver_init(const struct storage_module_init_options *opts) + + if (i == g_numdrivers) { + ERROR("unsupported driver %s", opts->driver_name); +- ret = -1; +- goto out; ++ return -1; + } + +-out: +- return ret; ++ return 0; + } + + int graphdriver_create_rw(const char *id, const char *parent, struct driver_create_opts *create_opts) +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c +index 6d45f463..cc24909a 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c +@@ -64,6 +64,9 @@ struct io_read_wrapper; + + #define QUOTA_SIZE_OPTION "overlay2.size" + #define QUOTA_BASESIZE_OPTIONS "overlay2.basesize" ++#define OVERRIDE_KERNELCHECK_OPTIONS "overlay2.override_kernel_check" ++#define SKIP_MOUNT_HOME_OPTIONS "overlay2.skip_mount_home" ++#define MOUNT_OPTIONS "overlay2.mountopt" + // MAX_LAYER_ID_LENGTH represents the number of random characters which can be used to create the unique link identifer + // for every layer. If this value is too long then the page size limit for the mount command may be exceeded. + // The idLength should be selected such that following equation is true (512 is a buffer for label metadata). +@@ -150,7 +153,7 @@ static int overlay2_parse_options(struct graphdriver *driver, const char **optio + goto out; + } + overlay_opts->default_quota = converted; +- } else if (strcasecmp(dup, "overlay2.override_kernel_check") == 0) { ++ } else if (strcasecmp(dup, OVERRIDE_KERNELCHECK_OPTIONS) == 0) { + bool converted_bool = 0; + ret = util_str_to_bool(val, &converted_bool); + if (ret != 0) { +@@ -160,7 +163,7 @@ static int overlay2_parse_options(struct graphdriver *driver, const char **optio + goto out; + } + overlay_opts->override_kernelcheck = converted_bool; +- } else if (strcasecmp(dup, "overlay2.skip_mount_home") == 0) { ++ } else if (strcasecmp(dup, SKIP_MOUNT_HOME_OPTIONS) == 0) { + bool converted_bool = 0; + ret = util_str_to_bool(val, &converted_bool); + if (ret != 0) { +@@ -170,7 +173,7 @@ static int overlay2_parse_options(struct graphdriver *driver, const char **optio + goto out; + } + overlay_opts->skip_mount_home = converted_bool; +- } else if (strcasecmp(dup, "overlay2.mountopt") == 0) { ++ } else if (strcasecmp(dup, MOUNT_OPTIONS) == 0) { + overlay_opts->mount_options = util_strdup_s(val); + } else { + ERROR("Overlay2: unknown option: '%s'", dup); +@@ -693,6 +696,10 @@ static char *get_lower(const char *parent, const char *driver_home) + goto out; + } + ++ /* ++ * lower format: "l/5697636c0104156cb2bd94be25", so "/" and "\0" must be ++ * counted in the size for snprintf. ++ */ + lower_len = strlen(OVERLAY_LINK_DIR) + 1 + strlen(parent_link) + 1; + + parent_lower_file = util_path_join(parent_dir, OVERLAY_LAYER_LOWER); +@@ -707,6 +714,11 @@ static char *get_lower(const char *parent, const char *driver_home) + ERROR("parent lower %s too large", parent_link_file); + goto out; + } ++ /* ++ * with parent link, the lower format will be like ++ * "l/5697636c0104156cb2bd94be25:l/df53b618a57bb50a61755b5623", ++ * so ":" must be counted. ++ */ + lower_len = lower_len + strlen(parent_lowers) + 1; + } + +@@ -911,7 +923,7 @@ static int do_create_remote_ro(const char *id, const char *parent, const struct + #ifdef ENABLE_USERNS_REMAP + if (set_file_owner_for_userns_remap(layer_dir, userns_remap) != 0) { + ERROR("Unable to change directory %s owner for user remap.", layer_dir); +- goto out; ++ goto err_out; + } + #endif + +@@ -977,7 +989,7 @@ static int do_create(const char *id, const char *parent, const struct graphdrive + if (set_file_owner_for_userns_remap(layer_dir, userns_remap) != 0) { + ERROR("Unable to change directory %s owner for user remap.", layer_dir); + ret = -1; +- goto out; ++ goto err_out; + } + #endif + +@@ -1790,7 +1802,7 @@ out: + return ret; + } + +-bool is_valid_layer_link(const char *link_id, const struct graphdriver *driver) ++static bool is_valid_layer_link(const char *link_id, const struct graphdriver *driver) + { + bool valid = false; + char *link_dir = NULL; +-- +2.23.0 + diff --git a/0171-registry-module-code-improve.patch b/0171-registry-module-code-improve.patch new file mode 100644 index 0000000..96ca326 --- /dev/null +++ b/0171-registry-module-code-improve.patch @@ -0,0 +1,578 @@ +From 4f030e07e99dfe996897b69c9d950f3226363afe Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Wed, 11 Dec 2024 04:04:45 +1400 +Subject: [PATCH 05/11] registry module code improve + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/modules/image/oci/oci_pull.c | 2 +- + .../modules/image/oci/registry/http_request.c | 30 +++----- + .../modules/image/oci/registry/registry.c | 74 +++++++++---------- + .../image/oci/registry/registry_apiv1.c | 11 +-- + .../image/oci/registry/registry_apiv2.c | 62 +++++----------- + 5 files changed, 72 insertions(+), 107 deletions(-) + +diff --git a/src/daemon/modules/image/oci/oci_pull.c b/src/daemon/modules/image/oci/oci_pull.c +index 1c486974..245d14fd 100644 +--- a/src/daemon/modules/image/oci/oci_pull.c ++++ b/src/daemon/modules/image/oci/oci_pull.c +@@ -105,7 +105,7 @@ static int pull_image(const im_pull_request *request, progress_status_map *progr + options = (registry_pull_options *)util_common_calloc_s(sizeof(registry_pull_options)); + if (options == NULL) { + ERROR("Out of memory"); +- goto out; ++ return ret; + } + + if (request->auth != NULL) { +diff --git a/src/daemon/modules/image/oci/registry/http_request.c b/src/daemon/modules/image/oci/registry/http_request.c +index 80fc2184..b9b29c39 100644 +--- a/src/daemon/modules/image/oci/registry/http_request.c ++++ b/src/daemon/modules/image/oci/registry/http_request.c +@@ -16,9 +16,6 @@ + #define _GNU_SOURCE /* See feature_test_macros(7) */ + #include "http_request.h" + #include <curl/curl.h> +-#include <isula_libutils/json_common.h> +-#include <isula_libutils/log.h> +-#include <isula_libutils/registry_token.h> + #include <pthread.h> + #include <stdbool.h> + #include <stdio.h> +@@ -27,6 +24,10 @@ + #include <strings.h> + #include <time.h> + ++#include <isula_libutils/json_common.h> ++#include <isula_libutils/log.h> ++#include <isula_libutils/registry_token.h> ++ + #include "buffer.h" + #include "certs.h" + #include "err_msg.h" +@@ -128,7 +129,6 @@ static int setup_ssl_config(pull_descriptor *desc, struct http_get_options *opti + options->ssl_verify_host = !desc->skip_tls_verify; + + out: +- + free(host); + host = NULL; + +@@ -437,16 +437,14 @@ static int setup_common_options(pull_descriptor *desc, struct http_get_options * + if (ret != 0) { + ERROR("Failed setup ssl config"); + isulad_try_set_error_message("setup ssl config failed"); +- ret = -1; +- goto out; ++ return -1; + } + + if (custom_headers != NULL) { + options->custom_headers = util_str_array_dup(custom_headers, util_array_len(custom_headers)); + if (options->custom_headers == NULL) { + ERROR("dup headers failed"); +- ret = -1; +- goto out; ++ return -1; + } + } + +@@ -454,14 +452,10 @@ static int setup_common_options(pull_descriptor *desc, struct http_get_options * + if (ret != 0) { + ERROR("setup auth challenges failed"); + isulad_try_set_error_message("setup auth challenges failed"); +- ret = -1; +- goto out; ++ return -1; + } + + options->debug = false; +- +-out: +- + return ret; + } + +@@ -478,21 +472,16 @@ static int setup_get_token_options(pull_descriptor *desc, struct http_get_option + ret = setup_ssl_config(desc, options, url); + if (ret != 0) { + ERROR("Failed setup ssl config"); +- ret = -1; +- goto out; ++ return -1; + } + + ret = setup_auth_basic(desc, &options->custom_headers); + if (ret != 0) { + ERROR("dup headers failed"); +- ret = -1; +- goto out; ++ return -1; + } + + options->debug = false; +- +-out: +- + return ret; + } + +@@ -526,7 +515,6 @@ static int http_request_buf_options(pull_descriptor *desc, struct http_get_optio + + *output = util_strdup_s(output_buffer->contents); + out: +- + buffer_free(output_buffer); + + return ret; +diff --git a/src/daemon/modules/image/oci/registry/registry.c b/src/daemon/modules/image/oci/registry/registry.c +index 2e99255a..9a3b59a1 100644 +--- a/src/daemon/modules/image/oci/registry/registry.c ++++ b/src/daemon/modules/image/oci/registry/registry.c +@@ -20,13 +20,21 @@ + #include <string.h> + #include <limits.h> + #include <errno.h> ++#include <pthread.h> ++#include <stdlib.h> ++ + #include <isula_libutils/docker_image_rootfs.h> + #include <isula_libutils/json_common.h> + #include <isula_libutils/oci_image_content_descriptor.h> + #include <isula_libutils/oci_image_manifest.h> + #include <isula_libutils/oci_image_spec.h> +-#include <pthread.h> +-#include <stdlib.h> ++#include <isula_libutils/registry_manifest_schema2.h> ++#include <isula_libutils/registry_manifest_schema1.h> ++#include <isula_libutils/docker_image_config_v2.h> ++#include <isula_libutils/image_manifest_v1_compatibility.h> ++#ifdef ENABLE_IMAGE_SEARCH ++#include <isula_libutils/image_search_image.h> ++#endif + + #include "mediatype.h" + #include "isula_libutils/log.h" +@@ -35,13 +43,6 @@ + #include "registry_apiv1.h" + #include "certs.h" + #include "auths.h" +-#include "isula_libutils/registry_manifest_schema2.h" +-#include "isula_libutils/registry_manifest_schema1.h" +-#include "isula_libutils/docker_image_config_v2.h" +-#include "isula_libutils/image_manifest_v1_compatibility.h" +-#ifdef ENABLE_IMAGE_SEARCH +-#include "isula_libutils/image_search_image.h" +-#endif + #include "sha256.h" + #include "map.h" + #include "linked_list.h" +@@ -536,7 +537,6 @@ static char *calc_chain_id(char *parent_chain_id, char *diff_id) + full_digest = util_full_digest(digest); + + out: +- + free(digest); + digest = NULL; + +@@ -797,7 +797,6 @@ static int set_config(pull_descriptor *desc, char *image_id) + } + + out: +- + free(config_str); + config_str = NULL; + +@@ -812,17 +811,15 @@ static int set_loaded_time(pull_descriptor *desc, char *image_id) + if (!util_get_now_time_stamp(&now)) { + ret = -1; + ERROR("get now time stamp failed"); +- goto out; ++ return ret; + } + + ret = storage_img_set_loaded_time(image_id, &now); + if (ret != 0) { + ERROR("set loaded time failed"); +- goto out; ++ return ret; + } + +-out: +- + return ret; + } + +@@ -984,7 +981,6 @@ static int parse_docker_config(pull_descriptor *desc) + desc->config.create_time = util_to_timestamp_from_str(config->created); + + out: +- + free_docker_image_config_v2(config); + config = NULL; + free(err); +@@ -1084,17 +1080,15 @@ static int fetch_and_parse_config(pull_descriptor *desc) + ret = fetch_config(desc); + if (ret != 0) { + ERROR("fetch config failed"); +- goto out; ++ return ret; + } + + ret = parse_config(desc); + if (ret != 0) { + ERROR("parse config failed"); +- goto out; ++ return ret; + } + +-out: +- + return ret; + } + +@@ -1110,17 +1104,15 @@ static int fetch_and_parse_manifest(pull_descriptor *desc) + ret = fetch_manifest(desc); + if (ret != 0) { + ERROR("fetch manifest failed"); +- goto out; ++ return ret; + } + + ret = parse_manifest(desc); + if (ret != 0) { + ERROR("parse manifest failed"); +- goto out; ++ return ret; + } + +-out: +- + return ret; + } + +@@ -2116,6 +2108,26 @@ static void cached_layers_kvfree(void *key, void *value) + return; + } + ++static void free_registry_global(registry_global *registry) ++{ ++ if (registry == NULL) { ++ return; ++ } ++ ++ if (registry->cond_inited) { ++ pthread_cond_destroy(®istry->cond); ++ } ++ if (registry->mutex_inited) { ++ pthread_mutex_destroy(®istry->mutex); ++ } ++ if (registry->image_mutex_inited) { ++ pthread_mutex_destroy(®istry->image_mutex); ++ } ++ map_free(registry->cached_layers); ++ registry->cached_layers = NULL; ++ free(registry); ++} ++ + int registry_init(char *auths_dir, char *certs_dir) + { + int ret = 0; +@@ -2160,18 +2172,7 @@ int registry_init(char *auths_dir, char *certs_dir) + out: + + if (ret != 0) { +- if (g_shared->cond_inited) { +- pthread_cond_destroy(&g_shared->cond); +- } +- if (g_shared->mutex_inited) { +- pthread_mutex_destroy(&g_shared->mutex); +- } +- if (g_shared->image_mutex_inited) { +- pthread_mutex_destroy(&g_shared->image_mutex); +- } +- map_free(g_shared->cached_layers); +- g_shared->cached_layers = NULL; +- free(g_shared); ++ free_registry_global(g_shared); + g_shared = NULL; + } + +@@ -2221,7 +2222,6 @@ int registry_login(registry_login_options *options) + } + + out: +- + free_pull_desc(desc); + desc = NULL; + +diff --git a/src/daemon/modules/image/oci/registry/registry_apiv1.c b/src/daemon/modules/image/oci/registry/registry_apiv1.c +index 6da24c1d..d45f3876 100644 +--- a/src/daemon/modules/image/oci/registry/registry_apiv1.c ++++ b/src/daemon/modules/image/oci/registry/registry_apiv1.c +@@ -18,12 +18,16 @@ + #include <stdio.h> + #include <string.h> + #include <limits.h> +-#include <isula_libutils/http_parser.h> +-#include <isula_libutils/json_common.h> + #include <stdbool.h> + #include <stdlib.h> + #include <strings.h> + ++#include <isula_libutils/http_parser.h> ++#include <isula_libutils/json_common.h> ++#include <isula_libutils/oci_image_index.h> ++#include <isula_libutils/registry_manifest_list.h> ++#include <isula_libutils/imagetool_search_result.h> ++ + #include "registry_type.h" + #include "isula_libutils/log.h" + #include "http.h" +@@ -31,9 +35,6 @@ + #include "utils.h" + #include "parser.h" + #include "mediatype.h" +-#include "isula_libutils/oci_image_index.h" +-#include "isula_libutils/registry_manifest_list.h" +-#include "isula_libutils/imagetool_search_result.h" + #include "auths.h" + #include "err_msg.h" + #include "sha256.h" +diff --git a/src/daemon/modules/image/oci/registry/registry_apiv2.c b/src/daemon/modules/image/oci/registry/registry_apiv2.c +index dd49fab7..5d83b425 100644 +--- a/src/daemon/modules/image/oci/registry/registry_apiv2.c ++++ b/src/daemon/modules/image/oci/registry/registry_apiv2.c +@@ -18,12 +18,15 @@ + #include <stdio.h> + #include <string.h> + #include <limits.h> +-#include <isula_libutils/http_parser.h> +-#include <isula_libutils/json_common.h> + #include <stdbool.h> + #include <stdlib.h> + #include <strings.h> + ++#include <isula_libutils/http_parser.h> ++#include <isula_libutils/json_common.h> ++#include <isula_libutils/oci_image_index.h> ++#include <isula_libutils/registry_manifest_list.h> ++ + #include "registry_type.h" + #include "isula_libutils/log.h" + #include "http.h" +@@ -31,8 +34,6 @@ + #include "utils.h" + #include "parser.h" + #include "mediatype.h" +-#include "isula_libutils/oci_image_index.h" +-#include "isula_libutils/registry_manifest_list.h" + #include "auths.h" + #include "err_msg.h" + #include "sha256.h" +@@ -60,7 +61,6 @@ static void set_body_null_if_exist(char *message) + static int parse_http_header(char *resp_buf, size_t buf_size, struct parsed_http_message *message) + { + char *real_message = NULL; +- int ret = 0; + + if (resp_buf == NULL || message == NULL) { + ERROR("Invalid NULL param"); +@@ -70,8 +70,7 @@ static int parse_http_header(char *resp_buf, size_t buf_size, struct parsed_http + real_message = strstr(resp_buf, "HTTP/1.1"); + if (real_message == NULL) { + ERROR("Failed to parse response, the response do not have HTTP/1.1"); +- ret = -1; +- goto out; ++ return -1; + } + + set_body_null_if_exist(real_message); +@@ -79,13 +78,10 @@ static int parse_http_header(char *resp_buf, size_t buf_size, struct parsed_http + ret = parse_http(real_message, strlen(real_message), message, HTTP_RESPONSE); + if (ret != 0) { + ERROR("Failed to parse response: %s", real_message); +- ret = -1; +- goto out; ++ return -1; + } + +-out: +- +- return ret; ++ return 0; + } + + static int parse_challenges(pull_descriptor *desc, char *schema, char *params) +@@ -225,7 +221,6 @@ static void free_parsed_http_message(struct parsed_http_message **message) + (*message)->body = NULL; + free(*message); + *message = NULL; +- return; + } + + static struct parsed_http_message *get_parsed_message(char *http_head) +@@ -386,7 +381,7 @@ static int registry_ping(pull_descriptor *desc) + ret = registry_pingv2(desc, "https"); + if (ret == 0) { + desc->protocol = util_strdup_s("https"); +- goto out; ++ return ret; + } + + if (desc->insecure_registry) { +@@ -396,15 +391,13 @@ static int registry_ping(pull_descriptor *desc) + ret = registry_pingv2(desc, "http"); + if (ret != 0) { + ERROR("ping %s with http failed", desc->host); +- goto out; ++ return ret; + } + desc->protocol = util_strdup_s("http"); + } else { + ERROR("ping %s with https failed", desc->host); + } + +-out: +- + return ret; + } + +@@ -552,7 +545,6 @@ static int parse_manifest_head(char *http_head, char **content_type, char **dige + } + + out: +- + if (ret != 0) { + free(*content_type); + *content_type = NULL; +@@ -584,19 +576,16 @@ static int append_manifests_accepts(char ***custom_headers) + sret = snprintf(accept, MAX_ACCEPT_LEN, "Accept: %s", mediatypes[i]); + if (sret < 0 || (size_t)sret >= MAX_ACCEPT_LEN) { + ERROR("Failed to sprintf accept media type %s", mediatypes[i]); +- ret = -1; +- goto out; ++ return -1; + } + + ret = util_array_append(custom_headers, accept); + if (ret != 0) { + ERROR("append accepts failed"); +- goto out; ++ return ret; + } + } + +-out: +- + return ret; + } + +@@ -703,7 +692,6 @@ static int fetch_manifest_list(pull_descriptor *desc, char *file, char **content + } + + out: +- + free(http_head); + http_head = NULL; + util_free_array(custom_headers); +@@ -727,7 +715,6 @@ static void try_log_resp_body(char *path, char *file) + ERROR("Get %s response message body: %s", path, body); + } + free(body); +- return; + } + + static int fetch_data(pull_descriptor *desc, char *path, char *file, char *content_type, char *digest) +@@ -1009,25 +996,22 @@ static int fetch_manifest_data(pull_descriptor *desc, char *file, char **content + ERROR("select manifest failed, manifests:%s", manifest_text); + free(manifest_text); + manifest_text = NULL; +- goto out; ++ return ret; + } + + sret = snprintf(path, sizeof(path), "/v2/%s/manifests/%s", desc->name, *digest); + if (sret < 0 || (size_t)sret >= sizeof(path)) { + ERROR("Failed to sprintf path for manifest"); +- ret = -1; +- goto out; ++ return -1; + } + + ret = fetch_data(desc, path, file, *content_type, *digest); + if (ret != 0) { + ERROR("registry: Get %s failed", path); +- goto out; ++ return ret; + } + } + +-out: +- + return ret; + } + +@@ -1096,20 +1080,17 @@ int fetch_config(pull_descriptor *desc) + sret = snprintf(path, sizeof(path), "/v2/%s/blobs/%s", desc->name, desc->config.digest); + if (sret < 0 || (size_t)sret >= sizeof(path)) { + ERROR("Failed to sprintf path for config"); +- ret = -1; +- goto out; ++ return -1; + } + + ret = fetch_data(desc, path, file, desc->config.media_type, desc->config.digest); + if (ret != 0) { + ERROR("registry: Get %s failed", path); +- goto out; ++ return ret; + } + + desc->config.file = util_strdup_s(file); + +-out: +- + return ret; + } + +@@ -1141,18 +1122,15 @@ int fetch_layer(pull_descriptor *desc, size_t index) + sret = snprintf(path, sizeof(path), "/v2/%s/blobs/%s", desc->name, layer->digest); + if (sret < 0 || (size_t)sret >= sizeof(path)) { + ERROR("Failed to sprintf path for layer %zu, name %s, digest %s", index, desc->name, layer->digest); +- ret = -1; +- goto out; ++ return -1; + } + + ret = fetch_data(desc, path, file, layer->media_type, layer->digest); + if (ret != 0) { + ERROR("registry: Get %s failed", path); +- goto out; ++ return ret; + } + +-out: +- + return ret; + } + +@@ -1185,7 +1163,6 @@ int parse_login(char *http_head, char *host) + } + + out: +- + free_parsed_http_message(&message); + + return ret; +@@ -1230,7 +1207,6 @@ int login_to_registry(pull_descriptor *desc) + goto out; + } + out: +- + free(resp_buffer); + resp_buffer = NULL; + +-- +2.23.0 + diff --git a/0172-image-store-fix-code-style.patch b/0172-image-store-fix-code-style.patch new file mode 100644 index 0000000..3194f14 --- /dev/null +++ b/0172-image-store-fix-code-style.patch @@ -0,0 +1,147 @@ +From 02a8be62cc7c1a492be5c9bc1fdf816b7d223b96 Mon Sep 17 00:00:00 2001 +From: zhongjiawei <zhongjiawei1@huawei.com> +Date: Wed, 11 Dec 2024 15:48:55 +0800 +Subject: [PATCH 06/11] image store:fix code style + +--- + .../oci/storage/image_store/image_store.c | 29 +++++++++---------- + .../oci/storage/image_store/image_store.h | 9 +++--- + .../oci/storage/image_store/image_type.c | 6 ++-- + .../oci/storage/image_store/image_type.h | 6 ++-- + 4 files changed, 25 insertions(+), 25 deletions(-) + +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.c b/src/daemon/modules/image/oci/storage/image_store/image_store.c +index 71bf36e0..1909e7f7 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_store.c ++++ b/src/daemon/modules/image/oci/storage/image_store/image_store.c +@@ -19,33 +19,34 @@ + #include <stdio.h> + #include <string.h> + #include <sha256.h> ++#include <limits.h> ++#include <pthread.h> ++#include <stdlib.h> ++ ++#include <isula_libutils/defs.h> ++#include <isula_libutils/log.h> + #include <isula_libutils/storage_image.h> + #include <isula_libutils/imagetool_images_list.h> + #include <isula_libutils/json_common.h> + #include <isula_libutils/auto_cleanup.h> +-#include <limits.h> +-#include <pthread.h> +-#include <stdlib.h> ++#include <isula_libutils/imagetool_image.h> ++#include <isula_libutils/imagetool_image_summary.h> ++#include <isula_libutils/registry_manifest_schema1.h> ++#include <isula_libutils/registry_manifest_schema2.h> ++#include <isula_libutils/oci_image_manifest.h> ++#include <isula_libutils/image_manifest_v1_compatibility.h> + + #include <isula_libutils/utils_macro.h> + + #include "utils.h" + #include "utils_file.h" + #include "utils_images.h" +-#include "isula_libutils/log.h" + #include "constants.h" + #include "utils_array.h" + #include "utils_string.h" + #include "utils_regex.h" +-#include "isula_libutils/defs.h" + #include "map.h" + #include "utils_convert.h" +-#include "isula_libutils/imagetool_image.h" +-#include "isula_libutils/imagetool_image_summary.h" +-#include "isula_libutils/registry_manifest_schema1.h" +-#include "isula_libutils/registry_manifest_schema2.h" +-#include "isula_libutils/oci_image_manifest.h" +-#include "isula_libutils/image_manifest_v1_compatibility.h" + #include "registry_type.h" + #include "mediatype.h" + #include "storage.h" +@@ -1361,8 +1362,7 @@ int image_store_set_big_data(const char *id, const char *key, const char *data) + + if (!image_store_lock(EXCLUSIVE)) { + ERROR("Failed to lock image store with exclusive lock, not allowed to change image big data assignments"); +- ret = -1; +- goto out; ++ return -1; + } + + img = lookup(id); +@@ -1648,8 +1648,7 @@ int image_store_get_names(const char *id, char ***names, size_t *names_len) + + if (!image_store_lock(SHARED)) { + ERROR("Failed to lock image store with shared lock, not allowed to get image names assignments"); +- ret = -1; +- goto out; ++ return -1; + } + + img = lookup(id); +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.h b/src/daemon/modules/image/oci/storage/image_store/image_store.h +index 4544f84b..82bc1696 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_store.h ++++ b/src/daemon/modules/image/oci/storage/image_store/image_store.h +@@ -18,16 +18,17 @@ + #include <stdbool.h> + #include <string.h> + #include <pthread.h> +-#include <isula_libutils/imagetool_fs_info.h> + #include <stdint.h> + ++#include <isula_libutils/imagetool_fs_info.h> ++#include <isula_libutils/imagetool_image.h> ++#include <isula_libutils/imagetool_images_list.h> ++#include <isula_libutils/imagetool_image_summary.h> ++ + #include "storage.h" + #include "utils_timestamp.h" + #include "map.h" + #include "linked_list.h" +-#include "isula_libutils/imagetool_image.h" +-#include "isula_libutils/imagetool_images_list.h" +-#include "isula_libutils/imagetool_image_summary.h" + + #ifdef __cplusplus + extern "C" { +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_type.c b/src/daemon/modules/image/oci/storage/image_store/image_type.c +index 50a81db2..67421cd6 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_type.c ++++ b/src/daemon/modules/image/oci/storage/image_store/image_type.c +@@ -19,11 +19,11 @@ + #include <stdlib.h> + #include <stdio.h> + +-#include "isula_libutils/storage_image.h" ++#include <isula_libutils/storage_image.h> ++#include <isula_libutils/log.h> ++ + #include "util_atomic.h" + #include "utils.h" +-#include "isula_libutils/log.h" +- + #include "utils_images.h" + + static image_t *create_empty_image() +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_type.h b/src/daemon/modules/image/oci/storage/image_store/image_type.h +index d8376644..bbf7a7dc 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_type.h ++++ b/src/daemon/modules/image/oci/storage/image_store/image_type.h +@@ -19,9 +19,9 @@ + #include <stdint.h> + #include <pthread.h> + +-#include "isula_libutils/storage_image.h" +-#include "isula_libutils/log.h" +-#include "isula_libutils/oci_image_spec.h" ++#include <isula_libutils/storage_image.h> ++#include <isula_libutils/log.h> ++#include <isula_libutils/oci_image_spec.h> + + #ifdef __cplusplus + extern "C" { +-- +2.23.0 + diff --git a/0173-bugfix-mem-leak.patch b/0173-bugfix-mem-leak.patch new file mode 100644 index 0000000..a477201 --- /dev/null +++ b/0173-bugfix-mem-leak.patch @@ -0,0 +1,64 @@ +From 3144357f7c735e24af180b9352378618ce8b2368 Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Wed, 11 Dec 2024 11:32:06 +0800 +Subject: [PATCH 07/11] bugfix: mem leak + +Signed-off-by: liuxu <liuxu156@huawei.com> +--- + src/daemon/executor/container_cb/execution_network.c | 2 ++ + src/daemon/modules/service/inspect_container.c | 2 ++ + src/utils/cutils/utils.c | 9 ++++++++- + 3 files changed, 12 insertions(+), 1 deletion(-) + +diff --git a/src/daemon/executor/container_cb/execution_network.c b/src/daemon/executor/container_cb/execution_network.c +index a145e33a..8e34998c 100644 +--- a/src/daemon/executor/container_cb/execution_network.c ++++ b/src/daemon/executor/container_cb/execution_network.c +@@ -1213,6 +1213,8 @@ static int generate_network_element(const char **bridges, const size_t len, defs + defs_map_string_object_networks_element *), len); + if (networks->values == NULL) { + ERROR("Out of memory "); ++ free(networks->keys); ++ networks->keys = NULL; + return -1; + } + +diff --git a/src/daemon/modules/service/inspect_container.c b/src/daemon/modules/service/inspect_container.c +index 40cf7aa1..ca3955c6 100644 +--- a/src/daemon/modules/service/inspect_container.c ++++ b/src/daemon/modules/service/inspect_container.c +@@ -629,6 +629,8 @@ static int do_transform_cni_to_map(container_network_settings *settings) + util_smart_calloc_s(sizeof(defs_map_string_object_port_bindings_element *), settings->cni_ports_len); + if (result->values == NULL) { + ERROR("Out of memory"); ++ free(result->keys); ++ result->keys = NULL; + ret = -1; + goto out; + } +diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c +index 69f6dbf0..cf207acc 100644 +--- a/src/utils/cutils/utils.c ++++ b/src/utils/cutils/utils.c +@@ -1609,10 +1609,17 @@ defs_map_string_object *dup_map_string_empty_object(defs_map_string_object *src) + } + + dst->keys = util_smart_calloc_s(sizeof(char *), src->len); ++ if (dst->keys == NULL) { ++ ERROR("Out of memory"); ++ ret = -1; ++ goto out; ++ } + dst->values = util_smart_calloc_s(sizeof(defs_map_string_object_element *), src->len); +- if (dst->keys == NULL || dst->values == NULL) { ++ if (dst->values == NULL) { + ERROR("Out of memory"); + ret = -1; ++ free(dst->keys); ++ dst->keys = NULL; + goto out; + } + +-- +2.23.0 + diff --git a/0174-bugfix-for-parse_http_header.patch b/0174-bugfix-for-parse_http_header.patch new file mode 100644 index 0000000..c30030b --- /dev/null +++ b/0174-bugfix-for-parse_http_header.patch @@ -0,0 +1,25 @@ +From 7a3d70dba97facedf1394e65a80f7cc12be8273c Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Wed, 18 Dec 2024 16:37:33 +1400 +Subject: [PATCH 08/11] bugfix for parse_http_header + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/modules/image/oci/registry/registry_apiv2.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/src/daemon/modules/image/oci/registry/registry_apiv2.c b/src/daemon/modules/image/oci/registry/registry_apiv2.c +index 5d83b425..7f33646d 100644 +--- a/src/daemon/modules/image/oci/registry/registry_apiv2.c ++++ b/src/daemon/modules/image/oci/registry/registry_apiv2.c +@@ -61,6 +61,7 @@ static void set_body_null_if_exist(char *message) + static int parse_http_header(char *resp_buf, size_t buf_size, struct parsed_http_message *message) + { + char *real_message = NULL; ++ int ret = 0; + + if (resp_buf == NULL || message == NULL) { + ERROR("Invalid NULL param"); +-- +2.23.0 + diff --git a/0175-add-layer-storage-ut-test.patch b/0175-add-layer-storage-ut-test.patch new file mode 100644 index 0000000..264fb09 --- /dev/null +++ b/0175-add-layer-storage-ut-test.patch @@ -0,0 +1,244 @@ +From 96ce67b474de6d6cff1a87cd652ff00dafda7d6e Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?=E6=AD=A6=E7=A7=AF=E8=B6=85?= <wujichao1@huawei.com> +Date: Tue, 24 Dec 2024 19:39:26 +0800 +Subject: [PATCH 11/11] add layer storage ut test + +--- + test/image/oci/storage/layers/CMakeLists.txt | 1 + + .../oci/storage/layers/storage_layers_ut.cc | 166 +++++++++++++++++- + 2 files changed, 165 insertions(+), 2 deletions(-) + +diff --git a/test/image/oci/storage/layers/CMakeLists.txt b/test/image/oci/storage/layers/CMakeLists.txt +index e1c76453..c4384e8f 100644 +--- a/test/image/oci/storage/layers/CMakeLists.txt ++++ b/test/image/oci/storage/layers/CMakeLists.txt +@@ -148,5 +148,6 @@ target_link_libraries(${LAYER_EXE} + ${LIBTAR_LIBRARY} + -lwebsockets -lcrypto -lyajl -larchive ${SELINUX_LIBRARY} -ldevmapper -lz -lcap) + ++set_target_properties(${LAYER_EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,map_new -Wl,--wrap,map_insert -Wl,--wrap,map_search -Wl,--wrap,util_common_calloc_s -Wl,--wrap,util_smart_calloc_s") + add_test(NAME ${LAYER_EXE} COMMAND ${LAYER_EXE} --gtest_output=xml:${LAYER_EXE}-Results.xml) + set_tests_properties(${LAYER_EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/image/oci/storage/layers/storage_layers_ut.cc b/test/image/oci/storage/layers/storage_layers_ut.cc +index 73611fdc..a03f4ce8 100644 +--- a/test/image/oci/storage/layers/storage_layers_ut.cc ++++ b/test/image/oci/storage/layers/storage_layers_ut.cc +@@ -29,6 +29,8 @@ + #include "storage.h" + #include "layer.h" + #include "driver_quota_mock.h" ++#include "map.h" ++#include "mock.h" + + using ::testing::Args; + using ::testing::ByRef; +@@ -41,6 +43,95 @@ using ::testing::AtLeast; + using ::testing::Invoke; + using ::testing::_; + ++static int g_map_search_count = 0; ++static int g_map_search_match = 1; ++static int g_map_new_count = 0; ++static int g_map_new_match = 1; ++static int g_map_insert_count = 0; ++static int g_map_insert_match = 1; ++ ++extern "C" { ++ DECLARE_WRAPPER_V(map_new, map_t *, (map_type_t kvtype, map_cmp_func comparator, map_kvfree_func kvfree)); ++ DEFINE_WRAPPER_V(map_new, map_t *, (map_type_t kvtype, map_cmp_func comparator, map_kvfree_func kvfree), (kvtype, comparator, kvfree)); ++ DECLARE_WRAPPER_V(map_insert, bool, (map_t *map, void *key, void *value)); ++ DEFINE_WRAPPER_V(map_insert, bool, (map_t *map, void *key, void *value), (map, key, value)); ++ DECLARE_WRAPPER_V(map_search, void *, (const map_t *map, void *key)); ++ DEFINE_WRAPPER_V(map_search, void *, (const map_t *map, void *key), (map, key)); ++ ++ DECLARE_WRAPPER_V(util_smart_calloc_s, void *, (size_t size, size_t len)); ++ DEFINE_WRAPPER_V(util_smart_calloc_s, void *, (size_t size, size_t len), (size, len)); ++ DECLARE_WRAPPER_V(util_common_calloc_s, void *, (size_t size)); ++ DEFINE_WRAPPER_V(util_common_calloc_s, void *, (size_t size), (size)); ++} ++ ++/* ++* Repeatedly calling the function executes the wrapper function and original function in the following order: ++* wrapper function; original function, wrapper function; original function, original function, wrapper function;... ++* Similar to regular queues (1 means wrapper, 0 means original): 1; 0 1; 0 0 1; 0 0 0 1; ... ++* It's used to MOCK a function that repeat permutation. ++* If you want a regular queue, the variables needs to be assigned back to the initial value. ++*/ ++static map_t *map_new_return_null(map_type_t kvtype, map_cmp_func comparator, map_kvfree_func kvfree) ++{ ++ g_map_new_count++; ++ if (g_map_new_count == g_map_new_match) { ++ g_map_new_match++; ++ g_map_new_count = 0; ++ return nullptr; ++ } else { ++ return __real_map_new(kvtype, comparator, kvfree); ++ } ++} ++ ++/* ++* Repeatedly calling the function executes the wrapper function and original function in the following order: ++* wrapper function; original function, wrapper function; original function, original function, wrapper function;... ++* Similar to regular queues (1 means wrapper, 0 means original): 1; 0 1; 0 0 1; 0 0 0 1; ... ++* It's used to MOCK a function that repeat permutation. ++* If you want a regular queue, the variables needs to be assigned back to the initial value. ++*/ ++static bool map_insert_return_false(map_t *map, void *key, void *value) ++{ ++ g_map_insert_count++; ++ if (g_map_insert_count == g_map_insert_match) { ++ g_map_insert_match++; ++ g_map_insert_count = 0; ++ return false; ++ } else { ++ return __real_map_insert(map, key, value); ++ } ++} ++ ++/* ++* Repeatedly calling the function executes the wrapper function and original function in the following order: ++* wrapper function; original function, wrapper function; original function, original function, wrapper function;... ++* Similar to regular queues (1 means wrapper, 0 means original): 1; 0 1; 0 0 1; 0 0 0 1; ... ++* It's used to MOCK a function that repeat permutation. ++* If you want a regular queue, the variables needs to be assigned back to the initial value. ++*/ ++void *map_search_fail(const map_t *map, void *key) ++{ ++ g_map_search_count++; ++ if (g_map_search_count == g_map_search_match) { ++ g_map_search_match++; ++ g_map_search_count = 0; ++ return nullptr; ++ } else { ++ return __real_map_search(map, key); ++ } ++ ++} ++ ++void *util_common_calloc_s_fail(size_t size) ++{ ++ return nullptr; ++} ++ ++void *util_smart_calloc_s_fail(size_t size, size_t len) ++{ ++ return nullptr; ++} ++ + std::string GetDirectory() + { + char abs_path[PATH_MAX] { 0x00 }; +@@ -178,6 +269,7 @@ protected: + std::string isulad_dir = "/tmp/isulad/"; + mkdir(isulad_dir.c_str(), 0755); + std::string root_dir = isulad_dir + "data"; ++ mkdir(root_dir.c_str(), 0755); + std::string run_dir = isulad_dir + "data/run"; + std::string data_dir = GetDirectory() + "/data"; + +@@ -194,12 +286,40 @@ protected: + opts.storage_root = strdup(real_path); + ASSERT_STRNE(util_clean_path(run_dir.c_str(), real_run_path, sizeof(real_run_path)), nullptr); + opts.storage_run_root = strdup(real_run_path); +- opts.driver_name = strdup("overlay"); + opts.driver_opts = static_cast<char **>(util_smart_calloc_s(sizeof(char *), 1)); + opts.driver_opts[0] = strdup("overlay2.skip_mount_home=true"); + opts.driver_opts_len = 1; +- ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ opts.enable_remote_layer = true; ++#endif + EXPECT_CALL(m_driver_quota_mock, QuotaCtl(_, _, _, _)).WillRepeatedly(Invoke(invokeQuotaCtl)); ++ ++ opts.driver_name = NULL; ++ ASSERT_EQ(layer_store_init(&opts), -1); ++ ++ char over_path_max_driver_name[5000] { 0x00 }; // PATH_MAX = 4096 ++ std::memset(over_path_max_driver_name, 'a', 4999); ++ over_path_max_driver_name[4999]= '\0'; ++ opts.driver_name = over_path_max_driver_name; ++ ASSERT_EQ(layer_store_init(&opts), -1); ++ ++ opts.driver_name = strdup("overlay"); ++ MOCK_SET_V(map_new, map_new_return_null); ++ g_map_new_count = 0; ++ g_map_new_match = 1; ++ ASSERT_EQ(layer_store_init(&opts), -1); ++ ASSERT_EQ(layer_store_init(&opts), -1); ++ ASSERT_EQ(layer_store_init(&opts), -1); ++ ASSERT_EQ(layer_store_init(&opts), -1); ++ MOCK_CLEAR(map_new); ++ ++ MOCK_SET_V(map_insert, map_insert_return_false); ++ g_map_insert_count = 0; ++ g_map_insert_match = 1; ++ ASSERT_EQ(layer_store_init(&opts), -1); ++ ASSERT_EQ(layer_store_init(&opts), -1); ++ MOCK_CLEAR(map_insert); ++ + ASSERT_EQ(layer_store_init(&opts), 0); + + free(opts.storage_root); +@@ -238,6 +358,13 @@ TEST_F(StorageLayersUnitTest, test_layers_load) + struct layer_list *layer_list = (struct layer_list *)util_common_calloc_s(sizeof(struct layer_list)); + ASSERT_NE(layer_list, nullptr); + ++ ASSERT_EQ(layer_store_list(NULL), -1); ++ MOCK_SET_V(util_smart_calloc_s, util_smart_calloc_s_fail); ++ ASSERT_EQ(layer_store_list(layer_list), -1); ++ MOCK_CLEAR(util_smart_calloc_s); ++ MOCK_SET_V(util_common_calloc_s, util_common_calloc_s_fail); ++ ASSERT_EQ(layer_store_list(layer_list), -1); ++ MOCK_CLEAR(util_common_calloc_s); + ASSERT_EQ(layer_store_list(layer_list), 0); + ASSERT_EQ(layer_list->layers_len, 2); + +@@ -315,6 +442,18 @@ TEST_F(StorageLayersUnitTest, test_layer_store_by_compress_digest) + std::string id { "9c27e219663c25e0f28493790cc0b88bc973ba3b1686355f221c38a36978ac63" }; + struct layer_list *layer_list = (struct layer_list *)util_common_calloc_s(sizeof(struct layer_list)); + ++ MOCK_SET_V(util_smart_calloc_s, util_smart_calloc_s_fail); ++ ASSERT_EQ(layer_store_by_compress_digest(compress.c_str(), layer_list), -1); ++ MOCK_CLEAR(util_smart_calloc_s); ++ MOCK_SET_V(util_common_calloc_s, util_common_calloc_s_fail); ++ ASSERT_EQ(layer_store_by_compress_digest(compress.c_str(), layer_list), -1); ++ MOCK_CLEAR(util_common_calloc_s); ++ MOCK_SET_V(map_search, map_search_fail); ++ g_map_search_count = 0; ++ g_map_search_match = 1; ++ ASSERT_EQ(layer_store_by_compress_digest(compress.c_str(), layer_list), -1); ++ MOCK_CLEAR(map_search); ++ + ASSERT_EQ(layer_store_by_compress_digest(compress.c_str(), layer_list), 0); + ASSERT_EQ(layer_list->layers_len, 1); + +@@ -324,3 +463,26 @@ TEST_F(StorageLayersUnitTest, test_layer_store_by_compress_digest) + + free_layer_list(layer_list); + } ++ ++#ifdef ENABLE_REMOTE_LAYER_STORE ++TEST_F(StorageLayersUnitTest, test_remote_layer_common) ++{ ++ ASSERT_EQ(remote_layer_remove_memory_stores_with_lock(NULL), -1); ++ char arr[] = "random_id"; ++ const char *random_id = arr; ++ MOCK_SET_V(map_search, map_search_fail); ++ g_map_search_count = 0; ++ g_map_search_match = 1; ++ ASSERT_EQ(remote_layer_remove_memory_stores_with_lock(random_id), 0); ++ MOCK_CLEAR(map_search); ++ ++ ASSERT_EQ(remote_load_one_layer(NULL), -1); ++ MOCK_SET_V(map_search, map_search_fail); ++ g_map_search_count = 0; ++ g_map_search_match = 1; ++ ASSERT_EQ(remote_load_one_layer(random_id), -1); ++ MOCK_CLEAR(map_search); ++ ++ ASSERT_EQ(remote_load_one_layer(random_id), -1); ++} ++#endif +-- +2.23.0 + diff --git a/iSulad.spec b/iSulad.spec index eafa82a..d9e894f 100644 --- a/iSulad.spec +++ b/iSulad.spec @@ -1,11 +1,17 @@ %global _version 2.1.5 -%global _release 1 +%global _release 18 %global is_systemd 1 %global enable_criv1 1 +%global enable_cdi 1 %global enable_shimv2 1 %global is_embedded 1 %global cpp_std 17 +%ifarch x86_64 aarch64 +%global enable_nri 1 +%global enable_sandboxer 1 +%endif + Name: iSulad Version: %{_version} Release: %{_release} @@ -15,6 +21,182 @@ URL: https://gitee.com/openeuler/iSulad Source: https://gitee.com/openeuler/iSulad/repository/archive/v%{version}.tar.gz BuildRoot: {_tmppath}/iSulad-%{version} +Patch0001: 0001-code-improve-for-sandbox.cc.patch +Patch0002: 0002-fix-compile-error-with-protobuf-25.1-and-grpc-1.60.x.patch +Patch0003: 0003-bugfix-for-mount-point-remains-under-special-circums.patch +Patch0004: 0004-do-not-cleanup-if-the-directory-does-not-exist.patch +Patch0005: 0005-module-only-deletes-the-temporary-files-it-creates.patch +Patch0006: 0006-skip-devmapper-ut.patch +Patch0007: 0007-update-annotations-and-add-ci-cases.patch +Patch0008: 0008-bug-fix-for-device-cgroup-ulimt-oci-update.patch +Patch0009: 0009-improve-dt-for-oci-spec-update.patch +Patch0010: 0010-open-run-container-with-dev-volume-testcase.patch +Patch0011: 0011-add-cpu-usage-nano-cores-for-sandbox.patch +Patch0012: 0012-sleep-some-time-in-ServiceWorkThread-to-prevent-the-.patch +Patch0013: 0013-restore-name-for-rename-failed.patch +Patch0014: 0014-2371-Allow-iSulad-to-pull-load-image-with-symlink.patch +Patch0015: 0015-Replace-http-parser-dependency-with-lcr.patch +Patch0016: 0016-add-more-detailed-log-information-for-load-sandbox.patch +Patch0017: 0017-bugfix-for-the-concurrency-competition-between-the-r.patch +Patch0018: 0018-add-concurrent-load-test.patch +Patch0019: 0019-get-the-realpath-of-the-host-path-for-archive-when-c.patch +Patch0020: 0020-bugfix-for-wrong-goto-branch.patch +Patch0021: 0021-bugfix-for-wrong-dynamic-allocation-object-type.patch +Patch0022: 0022-add-swap-usage-in-cri.patch +Patch0023: 0023-add-benchmark-result-of-perf-test-in-cri.patch +Patch0024: 0024-add-support-for-systemd-cgroup-driver.patch +Patch0025: 0025-add-ci-cases-for-systemd-cgroup-driver.patch +Patch0026: 0026-move-systemd_cgroup-CI-test-to-manual-cases.patch +Patch0027: 0027-feature-add-support-for-cgroup-v2-metrics.patch +Patch0028: 0028-use-supervisor-to-notify-sandbox-exit-event.patch +Patch0029: 0029-refactor-cgroup-module.patch +Patch0030: 0030-adaptor-unit-test-for-cgroup-module.patch +Patch0031: 0031-cgroup-v2-does-not-support-isulad-setting-cpu_rt-opt.patch +Patch0032: 0032-add-test-that-isulad-cannot-set-cpu_rt-parameters-wh.patch +Patch0033: 0033-fix-sandbox-container-bool-value-uninitialized.patch +Patch0034: 0034-bugfix-for-cpurt.sh.patch +Patch0035: 0035-monitor-cgroup-oom-killed-event-and-update-to-cri-of.patch +Patch0036: 0036-add-ci-cases-for-oomkilled-monitor.patch +Patch0037: 0037-add-cgroup-v2-doc.patch +Patch0038: 0038-add-modify-for-cgroup-v2-ci-test.patch +Patch0039: 0039-fix-run-ubuntu-container-bug-in-inspect.sh.patch +Patch0040: 0040-add-support-for-GetContainerEvents.patch +Patch0041: 0041-fix-cpurt-init-bug-for-systemd-cgroup.patch +Patch0042: 0042-fix-message-queue-concurrent-bug.patch +Patch0043: 0043-specify-runtime-as-runc-for-oom-test-CI.patch +Patch0044: 0044-set-oomkilled-in-cri.patch +Patch0045: 0045-add-cri-1.29-update-design-doc.patch +Patch0046: 0046-oom-monitor-in-manual-cases.patch +Patch0047: 0047-add-usage-restrictions-for-CRI-1.29-update.patch +Patch0048: 0048-CDI-interface-definition.patch +Patch0049: 0049-distinguish-between-runtime-and-runtime_cmd-in-isula.patch +Patch0050: 0050-Use-user-defined-shm-for-CRI-request.patch +Patch0051: 0051-Fix-memory-leak-in-set_connected_container_shm_path.patch +Patch0052: 0052-init-enable_pod_events-as-false.patch +Patch0053: 0053-remove-container-root-path-in-rt_lcr_rm-if-lcr-runti.patch +Patch0054: 0054-ensure-sandbox-can-be-removed-if-sandbox-container-r.patch +Patch0055: 0055-bugfix-for-shim-timeout-exit-error-log-changes.patch +Patch0056: 0056-bugfix-for-the-pre-created-pipe-was-not-closed-when-.patch +Patch0057: 0057-add-debug-msg-info-in-image_load.sh.patch +Patch0058: 0058-empty-pointer-check-in-lcr_rt_ops.patch +Patch0059: 0059-modify-some-grpc-status-codes-of-cri-in-case-of-erro.patch +Patch0060: 0060-cdi-return-int-instead-of-error-string.patch +Patch0061: 0061-cdi-support-modules-operate-registry-annotations.patch +Patch0062: 0062-do-not-umount-shmpath-for-sandbox-container.patch +Patch0063: 0063-remove-default-systemd-cgroup-and-enable-cri-v1-valu.patch +Patch0064: 0064-cdi-support-module-cache.patch +Patch0065: 0065-change-default-subscribe-timeout-to-5min.patch +Patch0066: 0066-cdi-support-modules-version-spec-spec_dirs-device.patch +Patch0067: 0067-cdi-support-modules-container_edits-parser.patch +Patch0068: 0068-cdi-invoke-cdi-operate-when-init-isulad-and-create-c.patch +Patch0069: 0069-bugfix-fix-cni_operate_ut-ut.patch +Patch0070: 0070-isolate-sandboxer-code-by-using-macro.patch +Patch0071: 0071-Remove-sandboxer-ut-if-sandboxer-is-not-enabled.patch +Patch0072: 0072-cdi-design-doc.patch +Patch0073: 0073-bugfix-cdi-version-check.patch +Patch0074: 0074-bugfix-of-background-execution-exec-error-command.patch +Patch0075: 0075-bugfix-for-setting-cpu-rt-to-a-negative-value-when-e.patch +Patch0076: 0076-cdi-add-UT.patch +Patch0077: 0077-remove-extra-s-in-CreateContainerLogSymlink.patch +Patch0078: 0078-allow-env-variable-has-an-empty-value.patch +Patch0079: 0079-Fix-Failed-to-execute-image-pull-on-name-tag-digest-.patch +Patch0080: 0080-bugfix-for-hostname-env-set-only-once.patch +Patch0081: 0081-set-the-sandbox-status-to-not-ready-under-abnormal-c.patch +Patch0082: 0082-fix-shim-controller-set-incorrect-sandbox-status-sta.patch +Patch0083: 0083-fix-bug-for-invalid-env-write.patch +Patch0084: 0084-trim-key-value-for-env.patch +Patch0085: 0085-cdi-allow-env-variable-has-an-empty-value.patch +Patch0086: 0086-cdi-test-case-and-gateway.patch +Patch0087: 0087-code-improve.patch +Patch0088: 0088-testcase-close-cdi-testcase.patch +Patch0089: 0089-docs-update-cni-doc.patch +Patch0090: 0090-modify-the-user-error-log-to-be-the-same-as-before.patch +Patch0091: 0091-add-enable-cri-v1-in-k8s-integration.patch +Patch0092: 0092-isolate-oom-monitor-codes.patch +Patch0093: 0093-change-fork-process-exit-mode.patch +Patch0094: 0094-fix-error-log-for-verify_cpu_realtime.patch +Patch0095: 0095-bugfix-change-max-network-name-len.patch +Patch0096: 0096-del-useless-info.patch +Patch0097: 0097-code-improve.patch +Patch0098: 0098-cdi-add-debug-info.patch +Patch0099: 0099-bugfix-cni-network-name-UT.patch +Patch0100: 0100-bugfix-malloc-right-type-size.patch +Patch0101: 0101-use-isula_clean_path-rather-than-realpath.patch +Patch0102: 0102-fix-false-engine-rootpath-reference.patch +Patch0103: 0103-bugfix-add-note.patch +Patch0104: 0104-bugfix-adapt-network-name-max-len.patch +Patch0105: 0105-start-sandbox-before-setup-network-by-default.patch +Patch0106: 0106-Revert-use-isula_clean_path-rather-than-realpath.patch +Patch0107: 0107-bugfix-for-start-sandbox-before-setup-network-by-def.patch +Patch0108: 0108-skip-test-rely-on-docker.io.patch +Patch0109: 0109-modify-default-registry-mirrors-in-ci-test.patch +Patch0110: 0110-add-timestamp-in-PodSandboxStatu-response.patch +Patch0111: 0111-bugfix-for-file-param-verify.patch +Patch0112: 0112-bugfix-change-cni-log-info.patch +Patch0113: 0113-move-shutdown-handle-after-init-module.patch +Patch0114: 0114-bugfix-for-null-pointer-reference.patch +Patch0115: 0115-bugfix-for-m_criService-shutdown.patch +Patch0116: 0116-fix-bug-in-ci-test.patch +Patch0117: 0117-add-nri-design-doc.patch +Patch0118: 0118-NRI-add-nri-head-file-and-common-func.patch +Patch0119: 0119-skip-calling-cni-plugin-cleanup-when-network-namespa.patch +Patch0120: 0120-nri-add-convert-and-utils-impl-for-nri.patch +Patch0121: 0121-get-realpath-before-ns-mountpoint-verification.patch +Patch0122: 0122-nri-impl-for-nri-plugin-and-adaption.patch +Patch0123: 0123-code-improve-for-codecheck.patch +Patch0124: 0124-change-pull-registry-to-hub.oepkgs.net.patch +Patch0125: 0125-fix-clang-build-error.patch +Patch0126: 0126-add-a-new-registry-to-prevent-missing-mirrors.patch +Patch0127: 0127-change-image-digest-ci-test-for-registry-change.patch +Patch0128: 0128-bugfix-for-ci-make-and-install-shell.patch +Patch0129: 0129-do-not-use-1000-as-the-test-gid-to-prevent-conflicts.patch +Patch0130: 0130-only-use-the-openeuler-mirror-registry-in-ci.patch +Patch0131: 0131-modify-alpine-image-source-to-isulad-alpine.patch +Patch0132: 0132-update-docs-design-README_zh.md.patch +Patch0133: 0133-modify-the-image-name-isulad-ubuntu-to-ubuntu.patch +Patch0134: 0134-ignore-chdir-failed-errmsg-when-kill-and-delete.patch +Patch0135: 0135-followlocation-only-not-with-head.patch +Patch0136: 0136-update-docs-design-detailed-Image-image_storage_driv.patch +Patch0137: 0137-upgrade-isulad-compilation-script-install_iSulad_on_.patch +Patch0138: 0138-bugfix-for-log-in-make_safedir_is_noexec.patch +Patch0139: 0139-containers-in-paused-state-are-not-allowed-to-start.patch +Patch0140: 0140-remove-meaningless-code.patch +Patch0141: 0141-fix-unqualified-call-to-std-move.patch +Patch0142: 0142-pull-failure-shows-error-reason.patch +Patch0143: 0143-move-CGROUP2_SUPER_MAGIC-define-to-cgroup.c.patch +Patch0144: 0144-update-centos-build-script.patch +Patch0145: 0145-cni-change-error-info.patch +Patch0146: 0146-bugfix-for-sem_wait-call-when-errno-is-EINTR.patch +Patch0147: 0147-add-no-pivot-root-support.patch +Patch0148: 0148-fix-issues-Isula-ps-cannot-display-port-mapping.patch +Patch0149: 0149-move-nri-call-in-stop-and-remove-con.patch +Patch0150: 0150-add-missing-con-linux-info-for-nri-module.patch +Patch0151: 0151-sandbox-sandbox-api-update.patch +Patch0152: 0152-add-omitted-macro-definition.patch +Patch0153: 0153-sandbox-sandbox-api-adapt-rust-interface.patch +Patch0154: 0154-add-linux-capability.h-head-file.patch +Patch0155: 0155-sandbox-fix-unused-variables.patch +Patch0156: 0156-sandbox-sandbox-api-adapt-rust-interface-UT.patch +Patch0157: 0157-bugfix-for-nri-init.patch +Patch0158: 0158-Revert-move-nri-call-in-stop-and-remove-con.patch +Patch0159: 0159-bugfix-overwriting-when-i-is-len-1.patch +Patch0160: 0160-bug-fix-Isula-ps-not-display-N-A-when-ports-empty.patch +Patch0161: 0161-bugfix-for-workdir-len-verify.patch +Patch0162: 0162-bugfix-fix-exec-detach-for-shim-v2.patch +Patch0163: 0163-image-layer-fix-code-style.patch +Patch0164: 0164-image-store-add-UT.patch +Patch0165: 0165-bugfix-do-purge-container-when-do_start_container-fa.patch +Patch0166: 0166-supplementary-registry-design-documentation.patch +Patch0167: 0167-sandbox-del-shim_sandbox-and-change-sandbox-ops.patch +Patch0168: 0168-UT-del-shim_sandbox-and-change-sandbox-ops.patch +Patch0169: 0169-add-image-storage-unit-test.patch +Patch0170: 0170-fix-some-bad-code.patch +Patch0171: 0171-registry-module-code-improve.patch +Patch0172: 0172-image-store-fix-code-style.patch +Patch0173: 0173-bugfix-mem-leak.patch +Patch0174: 0174-bugfix-for-parse_http_header.patch +Patch0175: 0175-add-layer-storage-ut-test.patch + %ifarch x86_64 aarch64 Provides: libhttpclient.so()(64bit) Provides: libisula_client.so()(64bit) @@ -46,7 +228,7 @@ BuildRequires: gtest-devel gmock-devel BuildRequires: libisula-devel > %{lcrver_lower} libisula-devel < %{lcrver_upper} BuildRequires: cmake gcc-c++ yajl-devel -BuildRequires: grpc grpc-plugins grpc-devel protobuf-devel +BuildRequires: grpc grpc-plugins grpc-devel protobuf-devel ncurses-devel BuildRequires: libcurl libcurl-devel libarchive-devel device-mapper-devel BuildRequires: libseccomp-devel libcap-devel libselinux-devel libwebsockets libwebsockets-devel BuildRequires: systemd-devel git @@ -54,7 +236,9 @@ BuildRequires: libevhtp-devel libevent-devel %if 0%{?enable_shimv2} BuildRequires: lib-shim-v2 lib-shim-v2-devel %endif - +%if 0%{?enable_nri} || 0%{?enable_sandboxer} +BuildRequires: isula-rust-extensions-devel +%endif Requires: libisula > %{lcrver_lower} libisula < %{lcrver_upper} Requires: grpc protobuf @@ -67,6 +251,9 @@ BuildRequires: libevhtp libevent %if 0%{?enable_shimv2} Requires: lib-shim-v2 %endif +%if 0%{?enable_nri} || 0%{?enable_sandboxer} +Requires: isula-rust-extensions +%endif %description This is a umbrella project for gRPC-services based Lightweight Container @@ -85,8 +272,16 @@ cd build -DCMAKE_INSTALL_PREFIX=/usr \ %if 0%{?enable_criv1} -DENABLE_CRI_API_V1=ON \ +%if 0%{?enable_cdi} + -DENABLE_CDI=ON \ +%endif +%if 0%{?enable_nri} + -DENABLE_NRI=ON \ +%endif +%if 0%{?enable_sandboxer} -DENABLE_SANDBOXER=ON \ %endif +%endif %if 0%{?enable_shimv2} -DENABLE_SHIM_V2=ON \ %endif @@ -195,11 +390,11 @@ if [ -e %{_unitdir}/lcrd.service.rpmsave ]; then mv %{_unitdir}/lcrd.service.rpmsave %{_unitdir}/isulad.service sed -i 's/lcrd/isulad/g' %{_unitdir}/isulad.service fi -# During the isulad upgrade process, the isulad service may still be running, but the service may be unavailable +# During the isulad upgrade process, the isulad service may still be running, but the service may be unavailable # due to configuration updates and other reasons. -# it may fail if the X package is upgraded synchronously with isulad and depends on the isulad command, +# it may fail if the X package is upgraded synchronously with isulad and depends on the isulad command, # For example syscontianer-tools and lxcfs-tools. -# Therefore, after upgrading isulad, if the original status of isulad is running, +# Therefore, after upgrading isulad, if the original status of isulad is running, # we need to restart isulad to ensure that the service is available during the upgrade process. systemctl status isulad | grep 'Active:' | grep 'running' if [ $? -eq 0 ]; then @@ -268,3 +463,768 @@ fi %endif %changelog +* Mon Dec 30 2024 jingxiaolu<lujingxiao@huawei.com> - 2.1.5-18 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: sync patches from upstream for refactoring sandbox and bugfixing + +* Mon Dec 30 2024 jingxiaolu<lujingxiao@huawei.com> - 2.1.5-17 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: sync patches from upstream + +* Wed Dec 18 2024 zhongtao <zhongtao17@huawei.com> - 2.1.5-16 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: bugfix for nri + +* Thu Nov 28 2024 liuxu <liuxu156@huawei.com> - 2.1.5-15 +- Type: update +- ID: NA +- SUG: NA +- DESC: sandboxer require isula-rust-extensions + +* Mon Nov 25 2024 liuxu <liuxu156@huawei.com> - 2.1.5-14 +- Type: update +- ID: NA +- SUG: NA +- DESC: upgrade from upstream + +* Thu Nov 21 2024 zhongtao <zhongtao17@huawei.com> - 2.1.5-13 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: enable nri support only on specified architecture + +* Mon Oct 21 2024 wujichao <wujichao1@huawei.com> - 2.1.5-12 +- Type: update +- ID: NA +- SUG: NA +- DESC: combine ten similar submissions into one(0125-fix-clang-build-error.patch) and Upgrade from upstream + +* Mon Aug 19 2024 zhongtao <zhongtao17@huawei.com> - 2.1.5-11 +- Type: update +- ID: NA +- SUG: NA +- DESC: add impl for nri and bugfix + +* Tue Jun 11 2024 zhongtao <zhongtao17@huawei.com> - 2.1.5-10 +- Type: update +- ID: NA +- SUG: NA +- DESC: code improve and bugfix + +* Sat May 11 2024 liuxu <liuxu156@huawei.com> - 2.1.5-9 +- Type: update +- ID: NA +- SUG: NA +- DESC: upgrade from upstream + +* Mon Apr 29 2024 zhongtao <zhongtao17@huawei.com> - 2.1.5-8 +- Type: update +- ID: NA +- SUG: NA +- DESC: remove extra %s in CreateContainerLogSymlink and allow env variable has an empty value + +* Thu Apr 25 2024 zhongtao <zhongtao17@huawei.com> - 2.1.5-7 +- Type: update +- ID: NA +- SUG: NA +- DESC: bugfix of background execution exec error command and setting negative cpu-rt issue + +* Sat Apr 20 2024 liuxu <liuxu156@huawei.com> - 2.1.5-6 +- Type: update +- ID: NA +- SUG: NA +- DESC: upgrade from upstream + +* Sat Apr 20 2024 liuxu <liuxu156@huawei.com> - 2.1.5-5 +- Type: update +- ID: NA +- SUG: NA +- DESC: upgrade from upstream + +* Tue Apr 02 2024 jikai <jikai11@huawei.com> - 2.1.5-4 +- Type: update +- ID: NA +- SUG: NA +- DESC: upgrade from upstream + +* Sat Mar 30 2024 zhongtao <zhongtao17@huawei.com> - 2.1.5-3 +- Type: update +- ID: NA +- SUG: NA +- DESC: add support for cgroup v2 + +* Tue Mar 19 2024 zhongtao <zhongtao17@huawei.com> - 2.1.5-2 +- Type: update +- ID: NA +- SUG: NA +- DESC: upgrade from upstream + +* Tue Jan 30 2024 zhongtao <zhongtao17@huawei.com> - 2.1.5-1 +- Type: update +- ID: NA +- SUG: NA +- DESC: update to v2.1.5 + +* Fri Dec 29 2023 yangjiaqi <yangjiaqi16@huawei.com> - 2.1.4-5 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: keep the service status unchanged after iSulad service upgrade + +* Thu Dec 28 2023 leizhongkai <leizhongkai@huawei.com> - 2.1.4-4 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: upgrade from upstream + +* Wed Dec 20 2023 zhongtao <zhongtao17@huawei.com> - 2.1.4-3 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: upgrade from upstream + +* Wed Nov 15 2023 zhongtao <zhongtao17@huawei.com> - 2.1.4-2 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: delete libisulad_img.so and compilation dependency: lxc, and add running dependency:runc + +* Tue Nov 14 2023 zhongtao <zhongtao17@huawei.com> - 2.1.4-1 +- Type: update +- ID: NA +- SUG: NA +- DESC: update to v2.1.4 + +* Wed Sep 13 2023 xuxuepeng <xuxuepeng1@huawei.com> - 2.1.3-2 +- Type: update +- ID: NA +- SUG: NA +- DESC: Add vsock support for exec + +* Tue Aug 29 2023 xuxuepeng <xuxuepeng1@huawei.com> - 2.1.3-1 +- Type: update +- ID: NA +- SUG: NA +- DESC: update to v2.1.3 + +* Mon Aug 28 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.2-8 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: modify lxc dependence to docker-runc dependence + +* Mon Aug 21 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.2-7 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: enable grpc and fix compile failed + +* Wed Aug 09 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.2-6 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: modify daemon json default runtime to runc + +* Thu Jul 20 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.2-5 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: compile using c++17 + +* Mon May 29 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.2-4 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: bugfix for memleak and malloc + +* Thu May 25 2023 zhongtao <zhongtao17@huawei.com> - 2.1.2-3 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: upgrade from upstream + +* Fri May 12 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.2-2 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix remote grpc macro + +* Thu May 11 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.2-1 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update from upstream to update cri + +* Fri Mar 24 2023 wangrunze <wangrunze13@huawei.com> - 2.1.1-6 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update from upstream to include remote feature + +* Thu Mar 16 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.1-5 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: use CURLOPT_XFERINFOFUNCTION instead of deprecated CURLOPT_PROGRESSFUNCTION since curl 7.32.0 + +* Wed Feb 22 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.1-4 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: upgrade from upstream + +* Thu Feb 16 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.1-3 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: add check + +* Mon Feb 06 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.1-2 +- Type: update +- ID: NA +- SUG: NA +- DESC: modify dependence from lcr to libisula + +* Mon Feb 06 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.1-1 +- Type: update +- ID: NA +- SUG: NA +- DESC: update to v2.1.1 + +* Tue Jan 03 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.18-1 +- Type: update +- ID: NA +- SUG: NA +- DESC: update to v2.0.18 + +* Thu Dec 22 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-14 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: remove clean_module_fill_ctx for libisulad_img.so + +* Mon Dec 19 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-13 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update lcr dependence version + +* Fri Dec 16 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-12 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update from upstream + +* Tue Dec 06 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-11 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update from upstream + +* Mon Nov 28 2022 yangjiaqi <yangjiaqi16@huawei.com> - 2.0.17-10 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: remove chmod 751 permission for dirs by engine when user-remap enabled + +* Fri Nov 25 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-9 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: add primary group to additional groups + +* Mon Nov 21 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-8 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix cannot install isulad and unknown option + +* Wed Nov 16 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-7 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update tar package from source + +* Wed Nov 02 2022 wangrunze <wangrunze13@huawei.com> - 2.0.17-6 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix cleanup module memory leak + +* Tue Nov 01 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: add ut and bugfix for device mapper and websocket + +* Mon Oct 31 2022 wujing <wujing50@huawei.com> - 2.0.17-4 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from openEuler + +* Wed Oct 19 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: add required package lcr clibcni lower and upper version + +* Mon Oct 10 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-2 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: use epoll instead of select for wait_exit_fifo + +* Sun Oct 09 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-1 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: update iSulad version to 2.0.17-1 + +* Thu Sep 29 2022 haozi007 <liuhao27@huawei.com> - 2.0.16-8 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from openEuler + +* Tue Sep 20 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.16-7 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: change libisulad_tools.so mode + +* Thu Sep 15 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.16-6 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: shield upgrade error if lcrd not exist + +* Tue Sep 13 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.16-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: calculate timezone by tm_gmtoff + +* Thu Sep 08 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.16-4 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: invalid free default-runtime and cri-runtime after free json-confs + +* Wed Sep 07 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.16-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sycn patches from openeuler/iSulad + +* Tue Aug 30 2022 leizhongkai <leizhongkai@huawei.com> - 2.0.16-2 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream openeuler/iSulad + +* Tue Aug 23 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.16-1 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: update iSulad version to 2.0.16-1 + +* Mon Aug 22 2022 zhongtao <zhongtao17@huawei.com> - 2.0.15-6 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: remove rpath by cmake + +* Wed Aug 17 2022 haozi007 <liuhao27@huawei.com> - 2.0.15-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sycn patches from openeuler + +* Mon Aug 15 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.15-4 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: change default umask to 0022 + +* Tue Aug 9 2022 haozi007 <liuhao27@huawei.com> - 2.0.15-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sycn patches from openeuler + +* Mon Aug 1 2022 chengzeruizhi <chengzeruizhi@huawei.com> - 2.0.15-2 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sycn patches from openeuler branch + +* Fri Jul 8 2022 haozi007 <liuhao27@huawei.com> - 2.0.15-1 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: update version to v2.0.15 + +* Fri Jul 8 2022 haozi007 <liuhao27@huawei.com> - 2.0.14-11 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: add limit to lcr version + +* Wed Jun 22 2022 yangjiaqi <yangjiaqi16@huawei.com> - 2.0.14-10 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream openeuler/iSulad + +* Tue Jun 21 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.14-9 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream openeuler/iSulad + +* Wed Jun 15 2022 chengzeruizhi <chengzeruizhi@huawei.com> - 2.0.14-8 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream openeuler/iSulad + +* Tue May 31 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.14-7 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: fix type convert, add null pointer check, remove unuse macro + +* Tue May 31 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.14-6 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: fix different type convert and add check to arguments + +* Mon May 30 2022 chengzrz <chengzeruizhi@huawei.com> - 2.0.14-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: seccomp optimization + +* Fri May 27 2022 haozi007 <liuhao27@huawei.com> - 2.0.14-4 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: improve fuzz test for pw and gr parser + +* Tue May 24 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.14-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: fix install error when android + +* Tue May 24 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.14-2 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: do not mkdir of isulad if no controller found + +* Mon May 23 2022 haozi007 <liuhao27@huawei.com> - 2.0.14-1 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: update version to v2.0.14 + +* Mon May 16 2022 haozi007<liuhao27@huawei.com> - 2.0.13-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream openEuler/iSulad + +* Tue May 10 2022 hejunjie<hejunjie10@huawei.com> - 2.0.13-4 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: bionic adaptation, increase lcov coverage + +* Thu May 5 2022 hejunjie<hejunjie10@huawei.com> - 2.0.13-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: bionic adaptation for pwgr obj parser + +* Mon Apr 25 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.13-2 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: sync from upstream + +* Mon Apr 18 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.13-1 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update version to v2.0.13 + +* Fri Mar 25 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.12-1 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update version to v2.0.12 + +* Thu Mar 17 2022 haozi007 <liuhao27@huawei.com> - 2.0.11-6 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: remove unnecessary error message + +* Thu Mar 17 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.11-5 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix uid/gid error when load image + +* Wed Mar 09 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.11-4 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: add provides of libisulad_tools.so + +* Thu Mar 03 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.11-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: Add the function of isolating the user namespaces + +* Thu Mar 03 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.11-2 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: add libisulad_tools.so + +* Thu Feb 24 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.11-1 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: update version to v2.0.11 + +* Wed Jan 12 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.10-15 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix compile error of isula-transform + +* Wed Jan 12 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.10-14 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix compile error with grpc 1.41.x + +* Tue Jan 4 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.10-13 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix compile error when building embedded image + +* Mon Dec 27 2021 wangfengtu <wangfengtu@huawei.com> - 2.0.10-12 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: sync patches from upstream + +* Thu Dec 09 2021 chengzeruizhi <chengzeruizhi@huawei.com> - 2.0.10-11 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fixed a bug that occurs when starting a container in host mode + +* Thu Dec 09 2021 wangfengtu <wagnfengtu@huawei.com> - 2.0.10-10 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: remove dependance of sqlite + +* Mon Dec 06 2021 gaohuatao <gaohuatao@huawei.com> - 2.0.10-9 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: specify version + +* Fri Dec 03 2021 wangfengtu <wangfengtu@huawei.com> - 2.0.10-8 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix undefined reference to `service_arguments_free' in libisulad_img.so + +* Thu Dec 02 2021 wangfengtu <wangfengtu@huawei.com> - 2.0.10-7 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: sync patches from upstream + +* Tue Nov 23 2021 chengzeruizhi <chengzeruizhi@huawei.com> - 2.0.10-6 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: modify the procedure of running a pod + +* Fri Nov 19 2021 gaohuatao <gaohuatao@huawei.com> - 2.0.10-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream + +* Fri Nov 19 2021 wangfengtu <wangfengtu@huawei.com> - 2.0.10-4 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix memleak when use multiple --volumes-from + +* Tue Nov 16 2021 wujing <wujing50@huawei.com> - 2.0.10-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: add shimv2 build switch + +* Tue Nov 16 2021 wujing <wujing50@huawei.com> - 2.0.10-2 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: remove build platform restrictions + +* Tue Nov 09 2021 gaohuatao <gaohuatao@huawei.com> - 2.0.10-1 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update from openeuler + +* Tue Oct 19 2021 wangfengtu <wangfengtu@huawei.com> - 2.0.9-20211019.121837.gitf067b3ce +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: strip sha256 prefix when decrease hold references + +* Fri Jun 25 2021 wujing <wujing50@huawei.com> - 2.0.9-20210625.165022.git5a088d9c +- Type: update to v2.0.9 +- ID: NA +- SUG: NA +- DESC: update from master + +* Tue May 18 2021 wangfengtu <wangfengtu@huawei.com> - 2.0.8-20210518.144540.git5288ed92 +- Type: sync from upstream +- ID: NA +- SUG: NA +- DESC: update from master + +* Fri Mar 26 2021 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.8-20210326.094027.gitac974aa6 +- Type: sync from upstream +- ID: NA +- SUG: NA +- DESC: update from master + +* Tue Mar 23 2021 haozi007 <liuhao27@huawei.com> - 20210323.094917.git7e6aa593 +- Type: sync from upstream +- ID: NA +- SUG: NA +- DESC: update from master + +* Tue Feb 2 2021 lifeng <lifeng68@huawei.com> - 2.0.8-20210202.153251.gite082dcf3 +- Type: sync from upstream +- ID: NA +- SUG: NA +- DESC: update from master + +* Mon Jan 18 2021 lifeng <lifeng68@huawei.com> - 2.0.8-20210118.195254.git077e10f2 +- Type: sync from upstream +- ID: NA +- SUG: NA +- DESC: update from master + +* Wed Dec 30 2020 lifeng <lifeng68@huawei.com> - 2.0.8-20201230.155843.git6557a6eb +- Type: update to v2.0.8 +- ID: NA +- SUG: NA +- DESC: update from master + +* Mon Dec 7 2020 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.7-20201207.151847.gita1fce123 +- Type: update +- ID: NA +- SUG: NA +- DESC: update from master + +* Sat Dec 5 2020 lifeng <lifeng68@huawei.com> - 2.0.7-20201205.145752.gita461cc51 +- Type: bugfix +- ID:NA +- SUG:NA +- DESC: ignore list containers errors + +* Thu Dec 3 2020 haozi007 <liuhao27@huawei.com> - 2.0.7-20201203.190902.git48f598fd +- Type:update from master +- ID:NA +- SUG:NA +- DESC: update from master + +* Sat Nov 28 2020 lifeng<lifeng68@huawei.com> - 2.0.7-20201128.095506.git1e1623a5 +- Type: bugfix +- ID:NA +- SUG:NA +- DESC: Mounts: only qsort the configed mounts and make possible to bind mount /proc and /sys/fs. +- related lxc PR fixed: +- 1.add check whether have /proc mounts entry, if has, skip the auto +- 2.mount cgroup before do mount entrys +- 3.pass if the mount on top of /proc and the source of the mount is a proc filesystem + +* Wed Nov 25 2020 wangfengtu<wangfengtu@huawei.com> - 2.0.7-20201125.165149.git7d150c3c +- Type: bugfix +- ID:NA +- SUG:NA +- DESC: update from openeuler + +* Wed Nov 25 2020 wangfengtu<wangfengtu@huawei.com> - 2.0.6-20201125.160534.git9fb5e75d +- Type: bugfix +- ID:NA +- SUG:NA +- DESC: fix rpath not work + +* Thu Nov 12 2020 gaohuatao<gaohuatao@huawei.com> - 2.0.6-20201112.193005.git8a6b73c8 +- Type: update from openeuler +- ID:NA +- SUG:NA +- DESC: update from openeuler + +* Wed Oct 14 2020 lifeng68<lifeng68@huawei.com> - 2.0.6-20201014.152749.gitc8a43925 +- Type: upgrade to v2.0.6 +- ID:NA +- SUG:NA +- DESC: upgrade to v2.0.6 + +* Fri Sep 18 2020 <lifeng68@huawei.com> - 2.0.5-20200918.112827.git9aea9b75 +- Type:bugfix +- ID:NA +- SUG:NA +- DESC: modify log level to warn + +* Mon Sep 14 2020 <lifeng68@huawei.com> - 2.0.5-20200914.172527.gitae86920a +- Type:bugfix +- ID:NA +- SUG:NA +- DESC: remove unused config + +* Thu Sep 10 2020 <yangjiaqi11@huawei.com> - 2.0.5-20200910.144345.git71b1055b +- Type:enhancement +- ID:NA +- SUG:NA +- DESC: add chrpath + +* Fri Sep 04 2020 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.5-20200904.114315.gitff1761c3 +- Type:enhancement +- ID:NA +- SUG:NA +- DESC: upgrade from v2.0.3 to v2.0.5 + +* Wed Sep 02 2020 YoungJQ <yangjiaqi11@huawei.com> - 2.0.3-20200902.114727.git6d945f26 +- Type:enhancement +- ID:NA +- SUG:NA +- DESC: modify source0 address |