From e45819fcb4a96649a4030db7684f140d5ca46735 Mon Sep 17 00:00:00 2001 From: CoprDistGit Date: Tue, 3 Sep 2024 03:24:28 +0000 Subject: automatic import of iSulad --- .gitignore | 1 + 0001-code-improve-for-sandbox.cc.patch | 27 + ...-error-with-protobuf-25.1-and-grpc-1.60.x.patch | 255 ++ ...mount-point-remains-under-special-circums.patch | 123 + ...t-cleanup-if-the-directory-does-not-exist.patch | 71 + ...ly-deletes-the-temporary-files-it-creates.patch | 151 + 0006-skip-devmapper-ut.patch | 26 + 0007-update-annotations-and-add-ci-cases.patch | 174 + ...ug-fix-for-device-cgroup-ulimt-oci-update.patch | 184 + 0009-improve-dt-for-oci-spec-update.patch | 281 ++ ...en-run-container-with-dev-volume-testcase.patch | 30 + 0011-add-cpu-usage-nano-cores-for-sandbox.patch | 81 + ...time-in-ServiceWorkThread-to-prevent-the-.patch | 27 + 0013-restore-name-for-rename-failed.patch | 29 + ...ow-iSulad-to-pull-load-image-with-symlink.patch | 45 + 0015-Replace-http-parser-dependency-with-lcr.patch | 427 ++ ...detailed-log-information-for-load-sandbox.patch | 26 + ...the-concurrency-competition-between-the-r.patch | 83 + 0018-add-concurrent-load-test.patch | 73 + ...lpath-of-the-host-path-for-archive-when-c.patch | 63 + 0020-bugfix-for-wrong-goto-branch.patch | 35 + ...-for-wrong-dynamic-allocation-object-type.patch | 29 + 0022-add-swap-usage-in-cri.patch | 80 + ...-add-benchmark-result-of-perf-test-in-cri.patch | 256 ++ 0024-add-support-for-systemd-cgroup-driver.patch | 523 +++ 0025-add-ci-cases-for-systemd-cgroup-driver.patch | 219 + ...ve-systemd_cgroup-CI-test-to-manual-cases.patch | 18 + ...feature-add-support-for-cgroup-v2-metrics.patch | 1084 +++++ ...e-supervisor-to-notify-sandbox-exit-event.patch | 254 ++ 0029-refactor-cgroup-module.patch | 2538 +++++++++++ 0030-adaptor-unit-test-for-cgroup-module.patch | 351 ++ ...oes-not-support-isulad-setting-cpu_rt-opt.patch | 71 + ...at-isulad-cannot-set-cpu_rt-parameters-wh.patch | 30 + ...andbox-container-bool-value-uninitialized.patch | 26 + 0034-bugfix-for-cpurt.sh.patch | 47 + ...oup-oom-killed-event-and-update-to-cri-of.patch | 868 ++++ 0036-add-ci-cases-for-oomkilled-monitor.patch | 279 ++ 0037-add-cgroup-v2-doc.patch | 252 ++ 0038-add-modify-for-cgroup-v2-ci-test.patch | 608 +++ ...ix-run-ubuntu-container-bug-in-inspect.sh.patch | 27 + 0040-add-support-for-GetContainerEvents.patch | 2601 +++++++++++ 0041-fix-cpurt-init-bug-for-systemd-cgroup.patch | 74 + 0042-fix-message-queue-concurrent-bug.patch | 41 + 0043-specify-runtime-as-runc-for-oom-test-CI.patch | 26 + 0044-set-oomkilled-in-cri.patch | 27 + 0045-add-cri-1.29-update-design-doc.patch | 312 ++ 0046-oom-monitor-in-manual-cases.patch | 120 + ...dd-usage-restrictions-for-CRI-1.29-update.patch | 27 + 0048-CDI-interface-definition.patch | 1192 +++++ ...-between-runtime-and-runtime_cmd-in-isula.patch | 145 + 0050-Use-user-defined-shm-for-CRI-request.patch | 171 + ...-leak-in-set_connected_container_shm_path.patch | 26 + 0052-init-enable_pod_events-as-false.patch | 62 + ...ainer-root-path-in-rt_lcr_rm-if-lcr-runti.patch | 33 + ...box-can-be-removed-if-sandbox-container-r.patch | 58 + ...x-for-shim-timeout-exit-error-log-changes.patch | 175 + ...the-pre-created-pipe-was-not-closed-when-.patch | 36 + 0057-add-debug-msg-info-in-image_load.sh.patch | 26 + 0058-empty-pointer-check-in-lcr_rt_ops.patch | 214 + ...-grpc-status-codes-of-cri-in-case-of-erro.patch | 315 ++ 0060-cdi-return-int-instead-of-error-string.patch | 363 ++ ...port-modules-operate-registry-annotations.patch | 516 +++ ...-not-umount-shmpath-for-sandbox-container.patch | 30 + ...ult-systemd-cgroup-and-enable-cri-v1-valu.patch | 27 + 0064-cdi-support-module-cache.patch | 862 ++++ ...-change-default-subscribe-timeout-to-5min.patch | 76 + ...ort-modules-version-spec-spec_dirs-device.patch | 1254 ++++++ ...di-support-modules-container_edits-parser.patch | 1210 +++++ ...cdi-operate-when-init-isulad-and-create-c.patch | 371 ++ 0069-bugfix-fix-cni_operate_ut-ut.patch | 24 + 0070-isolate-sandboxer-code-by-using-macro.patch | 143 + ...-sandboxer-ut-if-sandboxer-is-not-enabled.patch | 28 + 0072-cdi-design-doc.patch | 391 ++ 0073-bugfix-cdi-version-check.patch | 52 + ...f-background-execution-exec-error-command.patch | 33 + ...setting-cpu-rt-to-a-negative-value-when-e.patch | 35 + 0076-cdi-add-UT.patch | 741 +++ ...move-extra-s-in-CreateContainerLogSymlink.patch | 26 + 0078-allow-env-variable-has-an-empty-value.patch | 69 + ...to-execute-image-pull-on-name-tag-digest-.patch | 72 + 0080-bugfix-for-hostname-env-set-only-once.patch | 148 + ...dbox-status-to-not-ready-under-abnormal-c.patch | 86 + ...ntroller-set-incorrect-sandbox-status-sta.patch | 60 + 0083-fix-bug-for-invalid-env-write.patch | 158 + 0084-trim-key-value-for-env.patch | 26 + ...cdi-allow-env-variable-has-an-empty-value.patch | 135 + 0086-cdi-test-case-and-gateway.patch | 359 ++ 0087-code-improve.patch | 1690 +++++++ 0088-testcase-close-cdi-testcase.patch | 29 + 0089-docs-update-cni-doc.patch | 68 + ...e-user-error-log-to-be-the-same-as-before.patch | 100 + 0091-add-enable-cri-v1-in-k8s-integration.patch | 54 + 0092-isolate-oom-monitor-codes.patch | 317 ++ 0093-change-fork-process-exit-mode.patch | 64 + 0094-fix-error-log-for-verify_cpu_realtime.patch | 26 + 0095-bugfix-change-max-network-name-len.patch | 54 + 0096-del-useless-info.patch | 26 + 0097-code-improve.patch | 83 + 0098-cdi-add-debug-info.patch | 33 + 0099-bugfix-cni-network-name-UT.patch | 28 + 0100-bugfix-malloc-right-type-size.patch | 40 + ...use-isula_clean_path-rather-than-realpath.patch | 26 + 0102-fix-false-engine-rootpath-reference.patch | 32 + 0103-bugfix-add-note.patch | 25 + 0104-bugfix-adapt-network-name-max-len.patch | 64 + ...t-sandbox-before-setup-network-by-default.patch | 140 + ...use-isula_clean_path-rather-than-realpath.patch | 28 + ...start-sandbox-before-setup-network-by-def.patch | 28 + 0108-skip-test-rely-on-docker.io.patch | 59 + ...odify-default-registry-mirrors-in-ci-test.patch | 26 + ...add-timestamp-in-PodSandboxStatu-response.patch | 25 + 0111-bugfix-for-file-param-verify.patch | 75 + 0112-bugfix-change-cni-log-info.patch | 26 + 0113-move-shutdown-handle-after-init-module.patch | 43 + 0114-bugfix-for-null-pointer-reference.patch | 56 + 0115-bugfix-for-m_criService-shutdown.patch | 33 + 0116-fix-bug-in-ci-test.patch | 54 + 0117-NRI-add-nri-head-file-and-common-func.patch | 1051 +++++ ...g-cni-plugin-cleanup-when-network-namespa.patch | 51 + 0119-nri-add-convert-and-utils-impl-for-nri.patch | 2254 ++++++++++ ...ealpath-before-ns-mountpoint-verification.patch | 72 + 0121-nri-impl-for-nri-plugin-and-adaption.patch | 4724 ++++++++++++++++++++ iSulad.spec | 1118 +++++ sources | 1 + 124 files changed, 35371 insertions(+) create mode 100644 0001-code-improve-for-sandbox.cc.patch create mode 100644 0002-fix-compile-error-with-protobuf-25.1-and-grpc-1.60.x.patch create mode 100644 0003-bugfix-for-mount-point-remains-under-special-circums.patch create mode 100644 0004-do-not-cleanup-if-the-directory-does-not-exist.patch create mode 100644 0005-module-only-deletes-the-temporary-files-it-creates.patch create mode 100644 0006-skip-devmapper-ut.patch create mode 100644 0007-update-annotations-and-add-ci-cases.patch create mode 100644 0008-bug-fix-for-device-cgroup-ulimt-oci-update.patch create mode 100644 0009-improve-dt-for-oci-spec-update.patch create mode 100644 0010-open-run-container-with-dev-volume-testcase.patch create mode 100644 0011-add-cpu-usage-nano-cores-for-sandbox.patch create mode 100644 0012-sleep-some-time-in-ServiceWorkThread-to-prevent-the-.patch create mode 100644 0013-restore-name-for-rename-failed.patch create mode 100644 0014-2371-Allow-iSulad-to-pull-load-image-with-symlink.patch create mode 100644 0015-Replace-http-parser-dependency-with-lcr.patch create mode 100644 0016-add-more-detailed-log-information-for-load-sandbox.patch create mode 100644 0017-bugfix-for-the-concurrency-competition-between-the-r.patch create mode 100644 0018-add-concurrent-load-test.patch create mode 100644 0019-get-the-realpath-of-the-host-path-for-archive-when-c.patch create mode 100644 0020-bugfix-for-wrong-goto-branch.patch create mode 100644 0021-bugfix-for-wrong-dynamic-allocation-object-type.patch create mode 100644 0022-add-swap-usage-in-cri.patch create mode 100644 0023-add-benchmark-result-of-perf-test-in-cri.patch create mode 100644 0024-add-support-for-systemd-cgroup-driver.patch create mode 100644 0025-add-ci-cases-for-systemd-cgroup-driver.patch create mode 100644 0026-move-systemd_cgroup-CI-test-to-manual-cases.patch create mode 100644 0027-feature-add-support-for-cgroup-v2-metrics.patch create mode 100644 0028-use-supervisor-to-notify-sandbox-exit-event.patch create mode 100644 0029-refactor-cgroup-module.patch create mode 100644 0030-adaptor-unit-test-for-cgroup-module.patch create mode 100644 0031-cgroup-v2-does-not-support-isulad-setting-cpu_rt-opt.patch create mode 100644 0032-add-test-that-isulad-cannot-set-cpu_rt-parameters-wh.patch create mode 100644 0033-fix-sandbox-container-bool-value-uninitialized.patch create mode 100644 0034-bugfix-for-cpurt.sh.patch create mode 100644 0035-monitor-cgroup-oom-killed-event-and-update-to-cri-of.patch create mode 100644 0036-add-ci-cases-for-oomkilled-monitor.patch create mode 100644 0037-add-cgroup-v2-doc.patch create mode 100644 0038-add-modify-for-cgroup-v2-ci-test.patch create mode 100644 0039-fix-run-ubuntu-container-bug-in-inspect.sh.patch create mode 100644 0040-add-support-for-GetContainerEvents.patch create mode 100644 0041-fix-cpurt-init-bug-for-systemd-cgroup.patch create mode 100644 0042-fix-message-queue-concurrent-bug.patch create mode 100644 0043-specify-runtime-as-runc-for-oom-test-CI.patch create mode 100644 0044-set-oomkilled-in-cri.patch create mode 100644 0045-add-cri-1.29-update-design-doc.patch create mode 100644 0046-oom-monitor-in-manual-cases.patch create mode 100644 0047-add-usage-restrictions-for-CRI-1.29-update.patch create mode 100644 0048-CDI-interface-definition.patch create mode 100644 0049-distinguish-between-runtime-and-runtime_cmd-in-isula.patch create mode 100644 0050-Use-user-defined-shm-for-CRI-request.patch create mode 100644 0051-Fix-memory-leak-in-set_connected_container_shm_path.patch create mode 100644 0052-init-enable_pod_events-as-false.patch create mode 100644 0053-remove-container-root-path-in-rt_lcr_rm-if-lcr-runti.patch create mode 100644 0054-ensure-sandbox-can-be-removed-if-sandbox-container-r.patch create mode 100644 0055-bugfix-for-shim-timeout-exit-error-log-changes.patch create mode 100644 0056-bugfix-for-the-pre-created-pipe-was-not-closed-when-.patch create mode 100644 0057-add-debug-msg-info-in-image_load.sh.patch create mode 100644 0058-empty-pointer-check-in-lcr_rt_ops.patch create mode 100644 0059-modify-some-grpc-status-codes-of-cri-in-case-of-erro.patch create mode 100644 0060-cdi-return-int-instead-of-error-string.patch create mode 100644 0061-cdi-support-modules-operate-registry-annotations.patch create mode 100644 0062-do-not-umount-shmpath-for-sandbox-container.patch create mode 100644 0063-remove-default-systemd-cgroup-and-enable-cri-v1-valu.patch create mode 100644 0064-cdi-support-module-cache.patch create mode 100644 0065-change-default-subscribe-timeout-to-5min.patch create mode 100644 0066-cdi-support-modules-version-spec-spec_dirs-device.patch create mode 100644 0067-cdi-support-modules-container_edits-parser.patch create mode 100644 0068-cdi-invoke-cdi-operate-when-init-isulad-and-create-c.patch create mode 100644 0069-bugfix-fix-cni_operate_ut-ut.patch create mode 100644 0070-isolate-sandboxer-code-by-using-macro.patch create mode 100644 0071-Remove-sandboxer-ut-if-sandboxer-is-not-enabled.patch create mode 100644 0072-cdi-design-doc.patch create mode 100644 0073-bugfix-cdi-version-check.patch create mode 100644 0074-bugfix-of-background-execution-exec-error-command.patch create mode 100644 0075-bugfix-for-setting-cpu-rt-to-a-negative-value-when-e.patch create mode 100644 0076-cdi-add-UT.patch create mode 100644 0077-remove-extra-s-in-CreateContainerLogSymlink.patch create mode 100644 0078-allow-env-variable-has-an-empty-value.patch create mode 100644 0079-Fix-Failed-to-execute-image-pull-on-name-tag-digest-.patch create mode 100644 0080-bugfix-for-hostname-env-set-only-once.patch create mode 100644 0081-set-the-sandbox-status-to-not-ready-under-abnormal-c.patch create mode 100644 0082-fix-shim-controller-set-incorrect-sandbox-status-sta.patch create mode 100644 0083-fix-bug-for-invalid-env-write.patch create mode 100644 0084-trim-key-value-for-env.patch create mode 100644 0085-cdi-allow-env-variable-has-an-empty-value.patch create mode 100644 0086-cdi-test-case-and-gateway.patch create mode 100644 0087-code-improve.patch create mode 100644 0088-testcase-close-cdi-testcase.patch create mode 100644 0089-docs-update-cni-doc.patch create mode 100644 0090-modify-the-user-error-log-to-be-the-same-as-before.patch create mode 100644 0091-add-enable-cri-v1-in-k8s-integration.patch create mode 100644 0092-isolate-oom-monitor-codes.patch create mode 100644 0093-change-fork-process-exit-mode.patch create mode 100644 0094-fix-error-log-for-verify_cpu_realtime.patch create mode 100644 0095-bugfix-change-max-network-name-len.patch create mode 100644 0096-del-useless-info.patch create mode 100644 0097-code-improve.patch create mode 100644 0098-cdi-add-debug-info.patch create mode 100644 0099-bugfix-cni-network-name-UT.patch create mode 100644 0100-bugfix-malloc-right-type-size.patch create mode 100644 0101-use-isula_clean_path-rather-than-realpath.patch create mode 100644 0102-fix-false-engine-rootpath-reference.patch create mode 100644 0103-bugfix-add-note.patch create mode 100644 0104-bugfix-adapt-network-name-max-len.patch create mode 100644 0105-start-sandbox-before-setup-network-by-default.patch create mode 100644 0106-Revert-use-isula_clean_path-rather-than-realpath.patch create mode 100644 0107-bugfix-for-start-sandbox-before-setup-network-by-def.patch create mode 100644 0108-skip-test-rely-on-docker.io.patch create mode 100644 0109-modify-default-registry-mirrors-in-ci-test.patch create mode 100644 0110-add-timestamp-in-PodSandboxStatu-response.patch create mode 100644 0111-bugfix-for-file-param-verify.patch create mode 100644 0112-bugfix-change-cni-log-info.patch create mode 100644 0113-move-shutdown-handle-after-init-module.patch create mode 100644 0114-bugfix-for-null-pointer-reference.patch create mode 100644 0115-bugfix-for-m_criService-shutdown.patch create mode 100644 0116-fix-bug-in-ci-test.patch create mode 100644 0117-NRI-add-nri-head-file-and-common-func.patch create mode 100644 0118-skip-calling-cni-plugin-cleanup-when-network-namespa.patch create mode 100644 0119-nri-add-convert-and-utils-impl-for-nri.patch create mode 100644 0120-get-realpath-before-ns-mountpoint-verification.patch create mode 100644 0121-nri-impl-for-nri-plugin-and-adaption.patch create mode 100644 iSulad.spec create mode 100644 sources diff --git a/.gitignore b/.gitignore index e69de29..f2b049a 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1 @@ +/v2.1.5.tar.gz diff --git a/0001-code-improve-for-sandbox.cc.patch b/0001-code-improve-for-sandbox.cc.patch new file mode 100644 index 0000000..f804203 --- /dev/null +++ b/0001-code-improve-for-sandbox.cc.patch @@ -0,0 +1,27 @@ +From 9497e03709a035805effd96eaa21f6c221a79e94 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Fri, 19 Jan 2024 17:12:30 +0800 +Subject: [PATCH 01/43] code improve for sandbox.cc + +Signed-off-by: zhongtao +--- + src/daemon/sandbox/sandbox.cc | 3 +-- + 1 file changed, 1 insertion(+), 2 deletions(-) + +diff --git a/src/daemon/sandbox/sandbox.cc b/src/daemon/sandbox/sandbox.cc +index 359cfbad..7b6496ed 100644 +--- a/src/daemon/sandbox/sandbox.cc ++++ b/src/daemon/sandbox/sandbox.cc +@@ -1086,8 +1086,7 @@ void Sandbox::FillSandboxMetadata(sandbox_metadata* metadata, Errors &error) + metadata->task_address = util_strdup_s(m_taskAddress.c_str()); + metadata->net_ns_path = util_strdup_s(m_netNsPath.c_str()); + +- google::protobuf::util::MessageToJsonString(*m_sandboxConfig.get(), &jsonStr); +- if (jsonStr.length() == 0) { ++ if (!google::protobuf::util::MessageToJsonString(*m_sandboxConfig.get(), &jsonStr).ok()) { + error.Errorf("Failed to get sandbox config json for sandbox: '%s'", m_id.c_str()); + ERROR("Failed to get sandbox config json for sandbox: '%s'", m_id.c_str()); + } +-- +2.34.1 + diff --git a/0002-fix-compile-error-with-protobuf-25.1-and-grpc-1.60.x.patch b/0002-fix-compile-error-with-protobuf-25.1-and-grpc-1.60.x.patch new file mode 100644 index 0000000..f43ce86 --- /dev/null +++ b/0002-fix-compile-error-with-protobuf-25.1-and-grpc-1.60.x.patch @@ -0,0 +1,255 @@ +From 71f8d4accbec5153b362281bbaf9a516ccd083f5 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Mon, 22 Jan 2024 15:55:16 +0800 +Subject: [PATCH 02/43] fix compile error with protobuf 25.1 and grpc 1.60.x + +Signed-off-by: zhongtao +--- + cmake/checker.cmake | 76 +++++++++++++++++++ + cmake/protoc.cmake | 3 - + src/CMakeLists.txt | 12 +-- + .../sandbox/controller/manager/CMakeLists.txt | 4 +- + .../sandboxer/async_wait_call/CMakeLists.txt | 4 +- + .../sandboxer/sandboxer_client/CMakeLists.txt | 4 +- + .../sandboxer_controller/CMakeLists.txt | 4 +- + test/sandbox/controller/shim/CMakeLists.txt | 4 +- + test/sandbox/sandbox/CMakeLists.txt | 4 +- + test/sandbox/sandbox_manager/CMakeLists.txt | 4 +- + 10 files changed, 87 insertions(+), 32 deletions(-) + +diff --git a/cmake/checker.cmake b/cmake/checker.cmake +index e19618e4..b0c395ef 100644 +--- a/cmake/checker.cmake ++++ b/cmake/checker.cmake +@@ -154,7 +154,83 @@ if (GRPC_CONNECTOR) + find_library(GPR_LIBRARY gpr) + _CHECK(GPR_LIBRARY "GPR_LIBRARY-NOTFOUND" "libgpr.so") + # no check ++ ++ # The use of absl libraries depends on the version of protobuf and grpc. ++ # Versions of protobuf before v22.0 do not require absl libraries at all. ++ # However, versions after v22.0 require the support of absl libraries. ++ # As a result, we skip the check for absl libraries in order to accommodate different protobuf and grpc versions. ++ set(ISULAD_ABSL_USED_TARGETS) + find_library(ABSL_SYNC_LIB absl_synchronization) ++ if (ABSL_SYNC_LIB) ++ set(ISULAD_ABSL_USED_TARGETS ++ ${ISULAD_ABSL_USED_TARGETS} ++ ${ABSL_SYNC_LIB} ++ ) ++ endif() ++ ++ find_library(ABSL_CORD_LIB absl_cord) ++ if (ABSL_CORD_LIB) ++ set(ISULAD_ABSL_USED_TARGETS ++ ${ISULAD_ABSL_USED_TARGETS} ++ ${ABSL_CORD_LIB} ++ ) ++ endif() ++ ++ find_library(ABSL_CORDZ_FUNCTIONS_LIB absl_cordz_functions) ++ if (ABSL_CORDZ_FUNCTIONS_LIB) ++ set(ISULAD_ABSL_USED_TARGETS ++ ${ISULAD_ABSL_USED_TARGETS} ++ ${ABSL_CORDZ_FUNCTIONS_LIB} ++ ) ++ endif() ++ ++ find_library(ABSL_CORDZ_INFO_LIB absl_cordz_info) ++ if (ABSL_CORDZ_INFO_LIB) ++ set(ISULAD_ABSL_USED_TARGETS ++ ${ISULAD_ABSL_USED_TARGETS} ++ ${ABSL_CORDZ_INFO_LIB} ++ ) ++ endif() ++ ++ find_library(ABSL_HASH_LIB absl_hash) ++ if (ABSL_HASH_LIB) ++ set(ISULAD_ABSL_USED_TARGETS ++ ${ISULAD_ABSL_USED_TARGETS} ++ ${ABSL_HASH_LIB} ++ ) ++ endif() ++ ++ find_library(ABSL_LOG_INTERNAL_CHECK_OP_LIB absl_log_internal_check_op) ++ if (ABSL_LOG_INTERNAL_CHECK_OP_LIB) ++ set(ISULAD_ABSL_USED_TARGETS ++ ${ISULAD_ABSL_USED_TARGETS} ++ ${ABSL_LOG_INTERNAL_CHECK_OP_LIB} ++ ) ++ endif() ++ ++ find_library(ABSL_LOG_INTERNAL_MESSAGE_LIB absl_log_internal_message) ++ if (ABSL_LOG_INTERNAL_MESSAGE_LIB) ++ set(ISULAD_ABSL_USED_TARGETS ++ ${ISULAD_ABSL_USED_TARGETS} ++ ${ABSL_LOG_INTERNAL_MESSAGE_LIB} ++ ) ++ endif() ++ ++ find_library(ABSL_LOG_INTERNAL_NULLGUARD_LIB absl_log_internal_nullguard) ++ if (ABSL_LOG_INTERNAL_NULLGUARD_LIB) ++ set(ISULAD_ABSL_USED_TARGETS ++ ${ISULAD_ABSL_USED_TARGETS} ++ ${ABSL_LOG_INTERNAL_NULLGUARD_LIB} ++ ) ++ endif() ++ ++ find_library(ABSL_STATUS_LIB absl_status) ++ if (ABSL_STATUS_LIB) ++ set(ISULAD_ABSL_USED_TARGETS ++ ${ISULAD_ABSL_USED_TARGETS} ++ ${ABSL_STATUS_LIB} ++ ) ++ endif() + + # check websocket + find_path(WEBSOCKET_INCLUDE_DIR libwebsockets.h) +diff --git a/cmake/protoc.cmake b/cmake/protoc.cmake +index 80c08687..6e2d1b84 100644 +--- a/cmake/protoc.cmake ++++ b/cmake/protoc.cmake +@@ -69,9 +69,6 @@ endif() + + if (ENABLE_CRI_API_V1 AND ENABLE_SANDBOXER) + execute_process(COMMAND mkdir -p ${SANDBOX_PROTOS_OUT_PATH}) +- PROTOC_CPP_GEN(sandbox ${SANDBOX_PROTOS_OUT_PATH} ${PROTOS_PATH}/sandbox/google/protobuf/any.proto) +- PROTOC_CPP_GEN(sandbox ${SANDBOX_PROTOS_OUT_PATH} ${PROTOS_PATH}/sandbox/google/protobuf/empty.proto) +- PROTOC_CPP_GEN(sandbox ${SANDBOX_PROTOS_OUT_PATH} ${PROTOS_PATH}/sandbox/google/protobuf/timestamp.proto) + PROTOC_CPP_GEN(sandbox ${SANDBOX_PROTOS_OUT_PATH} ${PROTOS_PATH}/sandbox/sandbox/types/sandbox.proto) + PROTOC_CPP_GEN(sandbox ${SANDBOX_PROTOS_OUT_PATH} ${PROTOS_PATH}/sandbox/sandbox/types/mount.proto) + PROTOC_CPP_GEN(sandbox ${SANDBOX_PROTOS_OUT_PATH} ${PROTOS_PATH}/sandbox/sandbox/types/platform.proto) +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index 860447de..d1bc65f9 100644 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -85,10 +85,8 @@ target_link_libraries(libisula_client libisulad_tools) + if (GRPC_CONNECTOR) + target_link_libraries(libisula_client -Wl,--as-needed -lstdc++) + target_link_libraries(libisula_client -Wl,--as-needed ${PROTOBUF_LIBRARY}) +- target_link_libraries(libisula_client -Wl,--no-as-needed ${GRPC_PP_REFLECTION_LIBRARY} ${GRPC_PP_LIBRARY} ${GRPC_LIBRARY} ${GPR_LIBRARY}) +- if(ABSL_SYNC_LIB) +- target_link_libraries(libisula_client -Wl,--no-as-needed ${ABSL_SYNC_LIB}) +- endif() ++ target_link_libraries(libisula_client -Wl,--as-needed ${GRPC_PP_REFLECTION_LIBRARY} ${GRPC_PP_LIBRARY} ${GRPC_LIBRARY} ${GPR_LIBRARY}) ++ target_link_libraries(libisula_client -Wl,--as-needed ${ISULAD_ABSL_USED_TARGETS}) + else() + target_link_libraries(libisula_client -ldl libhttpclient) + set_target_properties(libisula_client PROPERTIES LINKER_LANGUAGE "C") +@@ -182,13 +180,11 @@ endif() + if (GRPC_CONNECTOR) + target_link_libraries(isulad -Wl,--as-needed -lstdc++) + target_link_libraries(isulad -Wl,--as-needed ${WEBSOCKET_LIBRARY} ${PROTOBUF_LIBRARY}) +- target_link_libraries(isulad -Wl,--no-as-needed ${GRPC_PP_REFLECTION_LIBRARY} ${GRPC_PP_LIBRARY} ${GRPC_LIBRARY} ${GPR_LIBRARY}) ++ target_link_libraries(isulad -Wl,--as-needed ${GRPC_PP_REFLECTION_LIBRARY} ${GRPC_PP_LIBRARY} ${GRPC_LIBRARY} ${GPR_LIBRARY}) + if (ENABLE_METRICS) + target_link_libraries(isulad ${EVHTP_LIBRARY} ${EVENT_LIBRARY}) + endif() +- if(ABSL_SYNC_LIB) +- target_link_libraries(isulad -Wl,--no-as-needed ${ABSL_SYNC_LIB}) +- endif() ++ target_link_libraries(isulad -Wl,--as-needed ${ISULAD_ABSL_USED_TARGETS}) + else() + message("Restful iSulad") + target_link_libraries(isulad ${EVHTP_LIBRARY} ${EVENT_LIBRARY}) +diff --git a/test/sandbox/controller/manager/CMakeLists.txt b/test/sandbox/controller/manager/CMakeLists.txt +index 3724538e..6e8c9052 100644 +--- a/test/sandbox/controller/manager/CMakeLists.txt ++++ b/test/sandbox/controller/manager/CMakeLists.txt +@@ -40,8 +40,6 @@ target_include_directories(${EXE} PUBLIC + ) + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lgrpc -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) +-if(ABSL_SYNC_LIB) +- target_link_libraries(${EXE} -Wl,--no-as-needed ${ABSL_SYNC_LIB}) +-endif() ++target_link_libraries(${EXE} -Wl,--as-needed ${ISULAD_ABSL_USED_TARGETS}) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) + set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/sandbox/controller/sandboxer/async_wait_call/CMakeLists.txt b/test/sandbox/controller/sandboxer/async_wait_call/CMakeLists.txt +index c8eb803e..0631988a 100644 +--- a/test/sandbox/controller/sandboxer/async_wait_call/CMakeLists.txt ++++ b/test/sandbox/controller/sandboxer/async_wait_call/CMakeLists.txt +@@ -32,8 +32,6 @@ target_include_directories(${EXE} PUBLIC + ) + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lgrpc -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) +-if(ABSL_SYNC_LIB) +- target_link_libraries(${EXE} -Wl,--no-as-needed ${ABSL_SYNC_LIB}) +-endif() ++target_link_libraries(${EXE} -Wl,--as-needed ${ISULAD_ABSL_USED_TARGETS}) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) + set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/sandbox/controller/sandboxer/sandboxer_client/CMakeLists.txt b/test/sandbox/controller/sandboxer/sandboxer_client/CMakeLists.txt +index 91f26883..881797c6 100644 +--- a/test/sandbox/controller/sandboxer/sandboxer_client/CMakeLists.txt ++++ b/test/sandbox/controller/sandboxer/sandboxer_client/CMakeLists.txt +@@ -36,8 +36,6 @@ target_include_directories(${EXE} PUBLIC + ) + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lgrpc -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) +-if(ABSL_SYNC_LIB) +- target_link_libraries(${EXE} -Wl,--no-as-needed ${ABSL_SYNC_LIB}) +-endif() ++target_link_libraries(${EXE} -Wl,--as-needed ${ISULAD_ABSL_USED_TARGETS}) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) + set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/sandbox/controller/sandboxer/sandboxer_controller/CMakeLists.txt b/test/sandbox/controller/sandboxer/sandboxer_controller/CMakeLists.txt +index d38392e0..963ce9a5 100644 +--- a/test/sandbox/controller/sandboxer/sandboxer_controller/CMakeLists.txt ++++ b/test/sandbox/controller/sandboxer/sandboxer_controller/CMakeLists.txt +@@ -33,8 +33,6 @@ target_include_directories(${EXE} PUBLIC + ) + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lgrpc -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) +-if(ABSL_SYNC_LIB) +- target_link_libraries(${EXE} -Wl,--no-as-needed ${ABSL_SYNC_LIB}) +-endif() ++target_link_libraries(${EXE} -Wl,--as-needed ${ISULAD_ABSL_USED_TARGETS}) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) + set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/sandbox/controller/shim/CMakeLists.txt b/test/sandbox/controller/shim/CMakeLists.txt +index af066546..6423bb80 100644 +--- a/test/sandbox/controller/shim/CMakeLists.txt ++++ b/test/sandbox/controller/shim/CMakeLists.txt +@@ -50,8 +50,6 @@ target_include_directories(${EXE} PUBLIC + ) + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) +-if(ABSL_SYNC_LIB) +- target_link_libraries(${EXE} -Wl,--no-as-needed ${ABSL_SYNC_LIB}) +-endif() ++target_link_libraries(${EXE} -Wl,--as-needed ${ISULAD_ABSL_USED_TARGETS}) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) + set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/sandbox/sandbox/CMakeLists.txt b/test/sandbox/sandbox/CMakeLists.txt +index efcc2bdc..138d4d8d 100644 +--- a/test/sandbox/sandbox/CMakeLists.txt ++++ b/test/sandbox/sandbox/CMakeLists.txt +@@ -48,8 +48,6 @@ target_include_directories(${EXE} PUBLIC + ) + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lgrpc -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) +-if(ABSL_SYNC_LIB) +- target_link_libraries(${EXE} -Wl,--no-as-needed ${ABSL_SYNC_LIB}) +-endif() ++target_link_libraries(${EXE} -Wl,--as-needed ${ISULAD_ABSL_USED_TARGETS}) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) + set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/sandbox/sandbox_manager/CMakeLists.txt b/test/sandbox/sandbox_manager/CMakeLists.txt +index f43b0f97..5a7cb2ea 100644 +--- a/test/sandbox/sandbox_manager/CMakeLists.txt ++++ b/test/sandbox/sandbox_manager/CMakeLists.txt +@@ -48,8 +48,6 @@ target_include_directories(${EXE} PUBLIC + ) + set_target_properties(${EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,util_list_all_subdir") + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lgrpc -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) +-if(ABSL_SYNC_LIB) +- target_link_libraries(${EXE} -Wl,--no-as-needed ${ABSL_SYNC_LIB}) +-endif() ++target_link_libraries(${EXE} -Wl,--as-needed ${ISULAD_ABSL_USED_TARGETS}) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) + set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +-- +2.34.1 + diff --git a/0003-bugfix-for-mount-point-remains-under-special-circums.patch b/0003-bugfix-for-mount-point-remains-under-special-circums.patch new file mode 100644 index 0000000..4465d41 --- /dev/null +++ b/0003-bugfix-for-mount-point-remains-under-special-circums.patch @@ -0,0 +1,123 @@ +From cd018d3c1ebff2a328912d99fc43c9a7e4f60704 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Thu, 25 Jan 2024 11:24:59 +0800 +Subject: [PATCH 03/43] bugfix for mount point remains under special + circumstances + +Signed-off-by: zhongtao +--- + src/cmd/isulad/main.c | 14 +++++++------- + .../modules/container/leftover_cleanup/cleanup.c | 14 +++++++------- + src/utils/tar/util_archive.c | 14 +++++++------- + 3 files changed, 21 insertions(+), 21 deletions(-) + +diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c +index d9d8afa0..deca72be 100644 +--- a/src/cmd/isulad/main.c ++++ b/src/cmd/isulad/main.c +@@ -1271,23 +1271,23 @@ static int do_ensure_isulad_tmpdir_security(const char *isulad_tmp_dir) + char tmp_dir[PATH_MAX] = { 0 }; + char cleanpath[PATH_MAX] = { 0 }; + +- nret = snprintf(tmp_dir, PATH_MAX, "%s/isulad_tmpdir", isulad_tmp_dir); +- if (nret < 0 || (size_t)nret >= PATH_MAX) { +- ERROR("Failed to snprintf"); ++ if (realpath(isulad_tmp_dir, cleanpath) == NULL) { ++ ERROR("Failed to get real path for %s", tmp_dir); + return -1; + } + +- if (util_clean_path(tmp_dir, cleanpath, sizeof(cleanpath)) == NULL) { +- ERROR("Failed to clean path for %s", tmp_dir); ++ nret = snprintf(tmp_dir, PATH_MAX, "%s/isulad_tmpdir", cleanpath); ++ if (nret < 0 || (size_t)nret >= PATH_MAX) { ++ ERROR("Failed to snprintf"); + return -1; + } + +- if (isulad_tmpdir_security_check(cleanpath) == 0) { ++ if (isulad_tmpdir_security_check(tmp_dir) == 0) { + return 0; + } + + INFO("iSulad tmpdir: %s does not meet security requirements, recreate it", isulad_tmp_dir); +- return recreate_tmpdir(cleanpath); ++ return recreate_tmpdir(tmp_dir); + } + + static int ensure_isulad_tmpdir_security() +diff --git a/src/daemon/modules/container/leftover_cleanup/cleanup.c b/src/daemon/modules/container/leftover_cleanup/cleanup.c +index b78a4d15..08151f42 100644 +--- a/src/daemon/modules/container/leftover_cleanup/cleanup.c ++++ b/src/daemon/modules/container/leftover_cleanup/cleanup.c +@@ -175,22 +175,22 @@ static void cleanup_path(char *dir) + char tmp_dir[PATH_MAX] = { 0 }; + char cleanpath[PATH_MAX] = { 0 }; + +- nret = snprintf(tmp_dir, PATH_MAX, "%s/isulad_tmpdir", dir); +- if (nret < 0 || (size_t)nret >= PATH_MAX) { +- ERROR("Failed to snprintf"); ++ if (realpath(dir, cleanpath) == NULL) { ++ ERROR("get real path for %s failed", tmp_dir); + return; + } + +- if (util_clean_path(tmp_dir, cleanpath, sizeof(cleanpath)) == NULL) { +- ERROR("clean path for %s failed", tmp_dir); ++ nret = snprintf(tmp_dir, PATH_MAX, "%s/isulad_tmpdir", cleanpath); ++ if (nret < 0 || (size_t)nret >= PATH_MAX) { ++ ERROR("Failed to snprintf"); + return; + } + +- if (!util_dir_exists(cleanpath)) { ++ if (!util_dir_exists(tmp_dir)) { + return; + } + +- nret = util_scan_subdirs(cleanpath, walk_isulad_tmpdir_cb, NULL); ++ nret = util_scan_subdirs(tmp_dir, walk_isulad_tmpdir_cb, NULL); + if (nret != 0) { + ERROR("failed to scan isulad tmp subdirs"); + } +diff --git a/src/utils/tar/util_archive.c b/src/utils/tar/util_archive.c +index 0a7309c9..e4c302bc 100644 +--- a/src/utils/tar/util_archive.c ++++ b/src/utils/tar/util_archive.c +@@ -218,18 +218,18 @@ static int make_safedir_is_noexec(const char *flock_path, const char *dstdir, ch + isulad_tmpdir_env = DEFAULT_ISULAD_TMPDIR; + } + +- nret = snprintf(isula_tmpdir, PATH_MAX, "%s/isulad_tmpdir", isulad_tmpdir_env); +- if (nret < 0 || (size_t)nret >= PATH_MAX) { +- ERROR("Failed to snprintf"); ++ if (realpath(isulad_tmpdir_env, cleanpath) == NULL) { ++ ERROR("Failed to get real path for %s", isula_tmpdir); + return -1; + } + +- if (util_clean_path(isula_tmpdir, cleanpath, sizeof(cleanpath)) == NULL) { +- ERROR("clean path for %s failed", isula_tmpdir); ++ nret = snprintf(isula_tmpdir, PATH_MAX, "%s/isulad_tmpdir", cleanpath); ++ if (nret < 0 || (size_t)nret >= PATH_MAX) { ++ ERROR("Failed to snprintf"); + return -1; + } + +- nret = snprintf(tmp_dir, PATH_MAX, "%s/tar-chroot-XXXXXX", cleanpath); ++ nret = snprintf(tmp_dir, PATH_MAX, "%s/tar-chroot-XXXXXX", isula_tmpdir); + if (nret < 0 || (size_t)nret >= PATH_MAX) { + ERROR("Failed to snprintf string"); + return -1; +@@ -247,7 +247,7 @@ static int make_safedir_is_noexec(const char *flock_path, const char *dstdir, ch + } + + // ensure parent dir is exist +- if (util_mkdir_p(cleanpath, ISULAD_TEMP_DIRECTORY_MODE) != 0) { ++ if (util_mkdir_p(isula_tmpdir, ISULAD_TEMP_DIRECTORY_MODE) != 0) { + return -1; + } + +-- +2.34.1 + diff --git a/0004-do-not-cleanup-if-the-directory-does-not-exist.patch b/0004-do-not-cleanup-if-the-directory-does-not-exist.patch new file mode 100644 index 0000000..d643d8e --- /dev/null +++ b/0004-do-not-cleanup-if-the-directory-does-not-exist.patch @@ -0,0 +1,71 @@ +From 7f13d95572040d30b70edbfac3c4b7350ee8855c Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Fri, 26 Jan 2024 12:59:45 +0800 +Subject: [PATCH 04/43] do not cleanup if the directory does not exist + +Signed-off-by: zhongtao +--- + src/cmd/isulad/main.c | 20 ++++++++++++++++++- + .../container/leftover_cleanup/cleanup.c | 13 +++++++++++- + 2 files changed, 31 insertions(+), 2 deletions(-) + +diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c +index deca72be..fd0b6e89 100644 +--- a/src/cmd/isulad/main.c ++++ b/src/cmd/isulad/main.c +@@ -1270,8 +1270,26 @@ static int do_ensure_isulad_tmpdir_security(const char *isulad_tmp_dir) + int nret; + char tmp_dir[PATH_MAX] = { 0 }; + char cleanpath[PATH_MAX] = { 0 }; ++ char isulad_tmp_cleanpath[PATH_MAX] = { 0 }; + +- if (realpath(isulad_tmp_dir, cleanpath) == NULL) { ++ if (util_clean_path(isulad_tmp_dir, isulad_tmp_cleanpath, sizeof(isulad_tmp_cleanpath)) == NULL) { ++ ERROR("Failed to clean path for %s", isulad_tmp_dir); ++ return -1; ++ } ++ ++ // Determine whether isulad_tmp_dir exists. If it does not exist, create it ++ // to prevent realpath from reporting errors because the folder does not exist. ++ if (!util_dir_exists(isulad_tmp_cleanpath)) { ++ nret = snprintf(tmp_dir, PATH_MAX, "%s/isulad_tmpdir", isulad_tmp_cleanpath); ++ if (nret < 0 || (size_t)nret >= PATH_MAX) { ++ ERROR("Failed to snprintf"); ++ return -1; ++ } ++ INFO("iSulad tmpdir: %s does not exist, create it", isulad_tmp_dir); ++ return recreate_tmpdir(tmp_dir); ++ } ++ ++ if (realpath(isulad_tmp_cleanpath, cleanpath) == NULL) { + ERROR("Failed to get real path for %s", tmp_dir); + return -1; + } +diff --git a/src/daemon/modules/container/leftover_cleanup/cleanup.c b/src/daemon/modules/container/leftover_cleanup/cleanup.c +index 08151f42..16dba630 100644 +--- a/src/daemon/modules/container/leftover_cleanup/cleanup.c ++++ b/src/daemon/modules/container/leftover_cleanup/cleanup.c +@@ -174,8 +174,19 @@ static void cleanup_path(char *dir) + int nret; + char tmp_dir[PATH_MAX] = { 0 }; + char cleanpath[PATH_MAX] = { 0 }; ++ char dir_cleanpath[PATH_MAX] = { 0 }; + +- if (realpath(dir, cleanpath) == NULL) { ++ if (util_clean_path(dir, dir_cleanpath, sizeof(dir_cleanpath)) == NULL) { ++ ERROR("clean path for %s failed", dir); ++ return; ++ } ++ ++ // If dir does not exist, skip cleanup ++ if (!util_dir_exists(dir_cleanpath)) { ++ return; ++ } ++ ++ if (realpath(dir_cleanpath, cleanpath) == NULL) { + ERROR("get real path for %s failed", tmp_dir); + return; + } +-- +2.34.1 + diff --git a/0005-module-only-deletes-the-temporary-files-it-creates.patch b/0005-module-only-deletes-the-temporary-files-it-creates.patch new file mode 100644 index 0000000..50e6515 --- /dev/null +++ b/0005-module-only-deletes-the-temporary-files-it-creates.patch @@ -0,0 +1,151 @@ +From 69dcd191afbdea5a178fb96a21e28537c2fc6a75 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Sat, 27 Jan 2024 11:16:37 +0800 +Subject: [PATCH 05/43] module only deletes the temporary files it creates + +Signed-off-by: zhongtao +--- + src/cmd/isulad/main.c | 2 +- + src/common/constants.h | 2 +- + src/daemon/modules/image/oci/oci_image.c | 44 +++++++++++++++++-- + src/daemon/modules/image/oci/oci_image.h | 4 ++ + src/daemon/modules/image/oci/oci_load.c | 2 +- + .../modules/image/oci/registry/registry.c | 2 +- + 6 files changed, 48 insertions(+), 8 deletions(-) + +diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c +index fd0b6e89..7c0c072e 100644 +--- a/src/cmd/isulad/main.c ++++ b/src/cmd/isulad/main.c +@@ -1252,7 +1252,7 @@ static int isulad_tmpdir_security_check(const char *tmp_dir) + + static int recreate_tmpdir(const char *tmp_dir) + { +- if (util_recursive_rmdir(tmp_dir, 0) != 0) { ++ if (util_path_remove(tmp_dir) != 0) { + ERROR("Failed to remove directory %s", tmp_dir); + return -1; + } +diff --git a/src/common/constants.h b/src/common/constants.h +index 27d4956e..8a6f86d8 100644 +--- a/src/common/constants.h ++++ b/src/common/constants.h +@@ -50,7 +50,7 @@ extern "C" { + + #define TEMP_DIRECTORY_MODE 0700 + +-#define ISULAD_TEMP_DIRECTORY_MODE 0600 ++#define ISULAD_TEMP_DIRECTORY_MODE 0700 + + #define CONSOLE_FIFO_DIRECTORY_MODE 0770 + +diff --git a/src/daemon/modules/image/oci/oci_image.c b/src/daemon/modules/image/oci/oci_image.c +index 9cf2cd4f..ce1c8a6b 100644 +--- a/src/daemon/modules/image/oci/oci_image.c ++++ b/src/daemon/modules/image/oci/oci_image.c +@@ -283,10 +283,42 @@ out: + return ret; + } + ++// remove dir that image module created ++// return false when failed to rmdir ++// eg: oci-image-load-XXXXXX && registry-XXXXXX ++static bool remove_image_tmpdir_cb(const char *path_name, const struct dirent *sub_dir, void *context) ++{ ++ int nret = 0; ++ char tmpdir[PATH_MAX] = { 0 }; ++ ++ if (sub_dir == NULL) { ++ return true; ++ } ++ ++ if (!util_has_prefix(sub_dir->d_name, LOAD_TMPDIR_PREFIX) && !util_has_prefix(sub_dir->d_name, REGISTRY_TMPDIR_PREFIX)) { ++ // only remove directory that image module created ++ return true; ++ } ++ ++ nret = snprintf(tmpdir, PATH_MAX, "%s/%s", path_name, sub_dir->d_name); ++ if (nret < 0 || (size_t)nret >= PATH_MAX) { ++ ERROR("Failed to snprintf for %s", sub_dir->d_name); ++ return false; ++ } ++ ++ if (util_recursive_rmdir(tmpdir, 0) != 0) { ++ ERROR("Failed to remove path %s", tmpdir); ++ return false; ++ } ++ ++ return true; ++} ++ + static int recreate_image_tmpdir() + { + char *image_tmp_path = NULL; + int ret = 0; ++ int nret = 0; + + image_tmp_path = oci_get_isulad_tmpdir(g_oci_image_module_data.root_dir); + if (image_tmp_path == NULL) { +@@ -295,10 +327,14 @@ static int recreate_image_tmpdir() + goto out; + } + +- if (util_recursive_rmdir(image_tmp_path, 0)) { +- ERROR("failed to remove directory %s", image_tmp_path); +- ret = -1; +- goto out; ++ // If image_tmp_path exist, cleanup it ++ if (util_dir_exists(image_tmp_path)) { ++ nret = util_scan_subdirs(image_tmp_path, remove_image_tmpdir_cb, NULL); ++ if (nret != 0) { ++ ERROR("Failed to scan isulad tmp subdirs"); ++ ret = -1; ++ goto out; ++ } + } + + if (util_mkdir_p(image_tmp_path, TEMP_DIRECTORY_MODE)) { +diff --git a/src/daemon/modules/image/oci/oci_image.h b/src/daemon/modules/image/oci/oci_image.h +index c7304897..482091d6 100644 +--- a/src/daemon/modules/image/oci/oci_image.h ++++ b/src/daemon/modules/image/oci/oci_image.h +@@ -38,6 +38,10 @@ struct oci_image_module_data { + char **insecure_registries; + size_t insecure_registries_len; + }; ++ ++#define LOAD_TMPDIR_PREFIX "oci-image-load-" ++#define REGISTRY_TMPDIR_PREFIX "registry-" ++ + struct oci_image_module_data *get_oci_image_data(void); + + int oci_init(const isulad_daemon_configs *args); +diff --git a/src/daemon/modules/image/oci/oci_load.c b/src/daemon/modules/image/oci/oci_load.c +index 31ae3849..534e2647 100644 +--- a/src/daemon/modules/image/oci/oci_load.c ++++ b/src/daemon/modules/image/oci/oci_load.c +@@ -1048,7 +1048,7 @@ static char *oci_load_path_create() + goto out; + } + +- nret = snprintf(tmp_dir, PATH_MAX, "%s/oci-image-load-XXXXXX", image_tmp_path); ++ nret = snprintf(tmp_dir, PATH_MAX, "%s/%sXXXXXX", image_tmp_path, LOAD_TMPDIR_PREFIX); + if (nret < 0 || (size_t)nret >= sizeof(tmp_dir)) { + ERROR("Path is too long"); + ret = -1; +diff --git a/src/daemon/modules/image/oci/registry/registry.c b/src/daemon/modules/image/oci/registry/registry.c +index 751a8727..aed3057a 100644 +--- a/src/daemon/modules/image/oci/registry/registry.c ++++ b/src/daemon/modules/image/oci/registry/registry.c +@@ -1908,7 +1908,7 @@ static int prepare_pull_desc(pull_descriptor *desc, registry_pull_options *optio + goto out; + } + +- sret = snprintf(blobpath, PATH_MAX, "%s/registry-XXXXXX", image_tmp_path); ++ sret = snprintf(blobpath, PATH_MAX, "%s/%sXXXXXX", image_tmp_path, REGISTRY_TMPDIR_PREFIX); + if (sret < 0 || (size_t)sret >= PATH_MAX) { + ERROR("image tmp work path too long"); + ret = -1; +-- +2.34.1 + diff --git a/0006-skip-devmapper-ut.patch b/0006-skip-devmapper-ut.patch new file mode 100644 index 0000000..912b3c7 --- /dev/null +++ b/0006-skip-devmapper-ut.patch @@ -0,0 +1,26 @@ +From b290e7fb553c5cc6746c9dcfe4896098f74bc7d7 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Tue, 30 Jan 2024 12:35:58 +0800 +Subject: [PATCH 06/43] skip devmapper ut + +Signed-off-by: jikai +--- + CI/make-and-install.sh | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh +index c1d26ff1..9bb984cd 100755 +--- a/CI/make-and-install.sh ++++ b/CI/make-and-install.sh +@@ -75,7 +75,7 @@ mkdir build && cd build + cmake -DCMAKE_BUILD_TYPE=Debug -DENABLE_UT=ON -DENABLE_CRI_API_V1=ON -DENABLE_SHIM_V2=ON -DENABLE_METRICS=ON .. + make -j $(nproc) + make install +-ctest -T memcheck --output-on-failure ++ctest -E "driver_devmapper_ut" -T memcheck --output-on-failure + echo_success "===================RUN DT-LLT TESTCASES END=========================" + + # build fuzz +-- +2.34.1 + diff --git a/0007-update-annotations-and-add-ci-cases.patch b/0007-update-annotations-and-add-ci-cases.patch new file mode 100644 index 0000000..07b857e --- /dev/null +++ b/0007-update-annotations-and-add-ci-cases.patch @@ -0,0 +1,174 @@ +From ed4b71b2027a6e9fdf15931fe93aa9e0bb3dc79d Mon Sep 17 00:00:00 2001 +From: leizhongkai +Date: Wed, 31 Jan 2024 18:17:52 +0800 +Subject: [PATCH 07/43] update annotations and add ci cases + +Signed-off-by: leizhongkai +--- + .../container_cases/dev_cgroup_rule.sh | 24 +++++++++++ + src/daemon/modules/api/specs_api.h | 2 + + .../modules/service/service_container.c | 18 +++++++- + src/daemon/modules/spec/specs.c | 41 ++++++++++++++++++- + 4 files changed, 82 insertions(+), 3 deletions(-) + +diff --git a/CI/test_cases/container_cases/dev_cgroup_rule.sh b/CI/test_cases/container_cases/dev_cgroup_rule.sh +index 839a546c..5616d37a 100755 +--- a/CI/test_cases/container_cases/dev_cgroup_rule.sh ++++ b/CI/test_cases/container_cases/dev_cgroup_rule.sh +@@ -29,6 +29,9 @@ function test_cpu_dev_cgoup_rule_spec() + local image="busybox" + local test="container device cgroup rule test with (${runtime}) => (${FUNCNAME[@]})" + local test_dev="/dev/testA" ++ local default_config="/etc/default/isulad/config.json" ++ local default_config_bak="/etc/default/isulad/config.json.bak" ++ local test_cgroup_parent="/testABC" + + msg_info "${test} starting..." + +@@ -54,6 +57,27 @@ function test_cpu_dev_cgoup_rule_spec() + [[ $? -ne 0 ]] && [[ $cnt -le $priv_minor_88_cnt ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check device minor failed" && ((ret++)) + isula rm -f $priv_cid + ++ def_cid=$(isula run -tid --runtime $runtime -m 10m $image /bin/sh) ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - run container failed" && ((ret++)) ++ cp $default_config $default_config_bak ++ sed -i '/"linux": {/a \ \t\t"devices": [\n\t\t{\n\t\t\t"type": "c",\n\t\t\t"path": "\/dev\/testABC",\n\t\t\t"major": 88,\n\t\t\t"minor": 88\n\t\t}\n\t\t],' $default_config ++ stop_isulad_without_valgrind ++ start_isulad_with_valgrind --cgroup-parent $test_cgroup_parent ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start isulad failed" && ((ret++)) ++ isula restart -t 0 $def_cid ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - restart container failed" && ((ret++)) ++ cat /sys/fs/cgroup/memory/$test_cgroup_parent/$def_cid/memory.limit_in_bytes | grep ^10485760$ ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - --cgroup-parent cannot work" && ((ret++)) ++ cnt=$(cat ${RUNTIME_ROOT_PATH}/${runtime}/$def_cid/config.json | grep "major\": 88" | wc -l) ++ [[ $? -ne 0 ]]&& [[ $cnt -ne 2 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check device major failed" && ((ret++)) ++ cnt=$(cat ${RUNTIME_ROOT_PATH}/${runtime}/$def_cid/config.json | grep "minor\": 88" | wc -l) ++ [[ $? -ne 0 ]] && [[ $cnt -ne 2 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check device minor failed" && ((ret++)) ++ isula rm -f $def_cid ++ cp $default_config_bak $default_config ++ stop_isulad_without_valgrind ++ start_isulad_with_valgrind ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start isulad failed" && ((ret++)) ++ + cid=$(isula run -tid --device "$test_dev:$test_dev" --runtime $runtime $image /bin/sh) + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - run container failed" && ((ret++)) + cnt=$(cat ${RUNTIME_ROOT_PATH}/${runtime}/$cid/config.json | grep "major\": 88" | wc -l) +diff --git a/src/daemon/modules/api/specs_api.h b/src/daemon/modules/api/specs_api.h +index 7c904614..f5f6ad8b 100644 +--- a/src/daemon/modules/api/specs_api.h ++++ b/src/daemon/modules/api/specs_api.h +@@ -41,6 +41,8 @@ int merge_share_namespace(oci_runtime_spec *oci_spec, const host_config *host_sp + const container_config_v2_common_config *v2_spec, + const container_network_settings *network_settings); + ++int update_spec_annotations(oci_runtime_spec *oci_spec, container_config *container_spec, host_config *host_spec); ++ + oci_runtime_spec *load_oci_config(const char *rootpath, const char *name); + + oci_runtime_spec *default_spec(bool system_container); +diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c +index 97f73768..239783b8 100644 +--- a/src/daemon/modules/service/service_container.c ++++ b/src/daemon/modules/service/service_container.c +@@ -691,11 +691,18 @@ out: + epoll_loop_close(&descr); + } + +-static int do_oci_spec_update(const char *id, oci_runtime_spec *oci_spec, host_config *hostconfig) ++static int do_oci_spec_update(const char *id, oci_runtime_spec *oci_spec, container_config *container_spec, host_config *hostconfig) + { + __isula_auto_free char *cgroup_parent = NULL; + int ret; + ++ // First renew annotations for oci spec, cgroup path, rootfs.mount, native.mask ++ // for iSulad daemon might get updated ++ ret = update_spec_annotations(oci_spec, container_spec, hostconfig); ++ if (ret < 0) { ++ return -1; ++ } ++ + // If isulad daemon cgroup parent updated, we should update this config into oci spec + cgroup_parent = merge_container_cgroups_path(id, hostconfig); + if (cgroup_parent == NULL) { +@@ -802,13 +809,20 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo + } + + // Update possible changes +- nret = do_oci_spec_update(id, oci_spec, cont->hostconfig); ++ nret = do_oci_spec_update(id, oci_spec, cont->common_config->config, cont->hostconfig); + if (nret != 0) { + ERROR("Failed to update possible changes for oci spec"); + ret = -1; + goto close_exit_fd; + } + ++ nret = container_to_disk(cont); ++ if (nret != 0) { ++ ERROR("Failed to save container info to disk"); ++ ret = -1; ++ goto close_exit_fd; ++ } ++ + nret = setup_ipc_dirs(cont->hostconfig, cont->common_config); + if (nret != 0) { + ERROR("Failed to setup ipc dirs"); +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index cc49d85f..62e340b1 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -385,6 +385,44 @@ out: + return ret; + } + ++int update_spec_annotations(oci_runtime_spec *oci_spec, container_config *container_spec, host_config *host_spec) ++{ ++ int ret = 0; ++ if (oci_spec == NULL || container_spec == NULL || host_spec == NULL) { ++ return -1; ++ } ++ ++ ret = make_sure_container_spec_annotations(container_spec); ++ if (ret < 0) { ++ return -1; ++ } ++ ++ ret = make_annotations_cgroup_dir(container_spec, host_spec); ++ if (ret != 0) { ++ return -1; ++ } ++ ++ /* add rootfs.mount */ ++ ret = add_rootfs_mount(container_spec); ++ if (ret != 0) { ++ ERROR("Failed to add rootfs mount"); ++ return -1; ++ } ++ ++ /* add native.umask */ ++ ret = add_native_umask(container_spec); ++ if (ret != 0) { ++ ERROR("Failed to add native umask"); ++ return -1; ++ } ++ ++ if (merge_annotations(oci_spec, container_spec)) { ++ return -1; ++ } ++ ++ return 0; ++} ++ + static int make_sure_oci_spec_root(oci_runtime_spec *oci_spec) + { + if (oci_spec->root == NULL) { +@@ -2501,4 +2539,5 @@ int spec_module_init(void) + return -1; + } + return 0; +-} +\ No newline at end of file ++} ++ +-- +2.34.1 + diff --git a/0008-bug-fix-for-device-cgroup-ulimt-oci-update.patch b/0008-bug-fix-for-device-cgroup-ulimt-oci-update.patch new file mode 100644 index 0000000..d433c46 --- /dev/null +++ b/0008-bug-fix-for-device-cgroup-ulimt-oci-update.patch @@ -0,0 +1,184 @@ +From fe3413bb8ebae90f29ce3cc02373f3fc2b5d2fd2 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Mon, 22 Jan 2024 20:19:29 +0800 +Subject: [PATCH 08/43] bug fix for device/cgroup/ulimt oci update + +Signed-off-by: jikai +--- + .../executor/container_cb/execution_create.c | 7 ++- + src/daemon/modules/api/specs_api.h | 4 ++ + .../modules/service/service_container.c | 18 +++--- + src/daemon/modules/spec/specs.c | 60 +++++++++++++++---- + 4 files changed, 63 insertions(+), 26 deletions(-) + +diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c +index ca2a9163..e00afb68 100644 +--- a/src/daemon/executor/container_cb/execution_create.c ++++ b/src/daemon/executor/container_cb/execution_create.c +@@ -533,12 +533,15 @@ static int merge_config_for_syscontainer(const container_create_request *request + value = request->rootfs; + } + +- if (append_json_map_string_string(oci_spec->annotations, "rootfs.mount", value)) { ++ // should also update to container spec ++ if (append_json_map_string_string(container_spec->annotations, "rootfs.mount", value) ++ || append_json_map_string_string(oci_spec->annotations, "rootfs.mount", value)) { + ERROR("Realloc annotations failed"); + ret = -1; + goto out; + } +- if (request->rootfs != NULL && append_json_map_string_string(oci_spec->annotations, "external.rootfs", "true")) { ++ if (request->rootfs != NULL && (append_json_map_string_string(container_spec->annotations, "external.rootfs", "true") ++ || append_json_map_string_string(oci_spec->annotations, "external.rootfs", "true"))) { + ERROR("Realloc annotations failed"); + ret = -1; + goto out; +diff --git a/src/daemon/modules/api/specs_api.h b/src/daemon/modules/api/specs_api.h +index f5f6ad8b..f54c0d31 100644 +--- a/src/daemon/modules/api/specs_api.h ++++ b/src/daemon/modules/api/specs_api.h +@@ -47,6 +47,10 @@ oci_runtime_spec *load_oci_config(const char *rootpath, const char *name); + + oci_runtime_spec *default_spec(bool system_container); + ++int update_oci_container_cgroups_path(const char *id, oci_runtime_spec *oci_spec, const host_config *host_spec); ++ ++int update_oci_ulimit(oci_runtime_spec *oci_spec, const host_config *host_spec); ++ + const oci_runtime_spec *get_readonly_default_oci_spec(bool system_container); + + int spec_module_init(void); +diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c +index 239783b8..a3606a82 100644 +--- a/src/daemon/modules/service/service_container.c ++++ b/src/daemon/modules/service/service_container.c +@@ -693,26 +693,21 @@ out: + + static int do_oci_spec_update(const char *id, oci_runtime_spec *oci_spec, container_config *container_spec, host_config *hostconfig) + { +- __isula_auto_free char *cgroup_parent = NULL; + int ret; + +- // First renew annotations for oci spec, cgroup path, rootfs.mount, native.mask +- // for iSulad daemon might get updated ++ // Renew annotations for oci spec, cgroup path only, ++ // since lxc uses the "cgroup.dir" in oci annotations to create cgroup ++ // should ensure that container spec has the same annotations as oci spec + ret = update_spec_annotations(oci_spec, container_spec, hostconfig); + if (ret < 0) { + return -1; + } + + // If isulad daemon cgroup parent updated, we should update this config into oci spec +- cgroup_parent = merge_container_cgroups_path(id, hostconfig); +- if (cgroup_parent == NULL) { ++ ret = update_oci_container_cgroups_path(id, oci_spec, hostconfig); ++ if (ret < 0) { + return -1; + } +- if (oci_spec->linux->cgroups_path != NULL && strcmp(oci_spec->linux->cgroups_path, cgroup_parent) != 0) { +- free(oci_spec->linux->cgroups_path); +- oci_spec->linux->cgroups_path = cgroup_parent; +- cgroup_parent = NULL; +- } + + // For Linux.Resources, isula update will save changes into oci spec; + // so we just skip it; +@@ -725,7 +720,8 @@ static int do_oci_spec_update(const char *id, oci_runtime_spec *oci_spec, contai + } + + // If isulad daemon ulimit updated, we should update this config into oci spec. +- if (merge_global_ulimit(oci_spec) != 0) { ++ ret = update_oci_ulimit(oci_spec, hostconfig); ++ if (ret < 0) { + return -1; + } + +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index 62e340b1..464b4fb4 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -402,19 +402,8 @@ int update_spec_annotations(oci_runtime_spec *oci_spec, container_config *contai + return -1; + } + +- /* add rootfs.mount */ +- ret = add_rootfs_mount(container_spec); +- if (ret != 0) { +- ERROR("Failed to add rootfs mount"); +- return -1; +- } +- +- /* add native.umask */ +- ret = add_native_umask(container_spec); +- if (ret != 0) { +- ERROR("Failed to add native umask"); +- return -1; +- } ++ // other annotations will either not be updated after containers created ++ // or for rootfs mnt and umask, we do not support the update operation + + if (merge_annotations(oci_spec, container_spec)) { + return -1; +@@ -2302,6 +2291,27 @@ char *merge_container_cgroups_path(const char *id, const host_config *host_spec) + return util_path_join(path, id); + } + ++int update_oci_container_cgroups_path(const char *id, oci_runtime_spec *oci_spec, const host_config *hostconfig) ++{ ++ if (oci_spec == NULL || oci_spec->linux == NULL) { ++ ERROR("Invalid arguments"); ++ return -1; ++ } ++ ++ __isula_auto_free char *cgroup_parent = merge_container_cgroups_path(id, hostconfig); ++ if (cgroup_parent == NULL) { ++ return -1; ++ } ++ ++ if (oci_spec->linux->cgroups_path != NULL && strcmp(oci_spec->linux->cgroups_path, cgroup_parent) != 0) { ++ free(oci_spec->linux->cgroups_path); ++ oci_spec->linux->cgroups_path = cgroup_parent; ++ cgroup_parent = NULL; ++ } ++ ++ return 0; ++} ++ + static int merge_oci_cgroups_path(const char *id, oci_runtime_spec *oci_spec, const host_config *host_spec) + { + if (id == NULL || oci_spec == NULL || host_spec == NULL) { +@@ -2445,6 +2455,30 @@ out: + return ret; + } + ++int update_oci_ulimit(oci_runtime_spec *oci_spec, const host_config *hostconfig) { ++ if (oci_spec == NULL || hostconfig == NULL) { ++ ERROR("Invalid arguments"); ++ return -1; ++ } ++ ++ size_t i = 0; ++ if (oci_spec->process != NULL) { ++ for (i = 0; i < oci_spec->process->rlimits_len; i++) { ++ free_defs_process_rlimits_element(oci_spec->process->rlimits[i]); ++ oci_spec->process->rlimits[i] = NULL; ++ } ++ free(oci_spec->process->rlimits); ++ oci_spec->process->rlimits = NULL; ++ oci_spec->process->rlimits_len = 0; ++ } ++ ++ if (merge_conf_ulimits(oci_spec, hostconfig) != 0 || merge_global_ulimit(oci_spec) != 0) { ++ return -1; ++ } ++ ++ return 0; ++} ++ + /* read oci config */ + oci_runtime_spec *load_oci_config(const char *rootpath, const char *name) + { +-- +2.34.1 + diff --git a/0009-improve-dt-for-oci-spec-update.patch b/0009-improve-dt-for-oci-spec-update.patch new file mode 100644 index 0000000..07c4b82 --- /dev/null +++ b/0009-improve-dt-for-oci-spec-update.patch @@ -0,0 +1,281 @@ +From 82dd5a1db70fdb3f4934a3f9c0ee290ce5bee1b2 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Sat, 27 Jan 2024 15:30:05 +0800 +Subject: [PATCH 09/43] improve dt for oci spec update + +Signed-off-by: jikai +--- + .../container_cases/dev_cgroup_rule.sh | 4 +- + CI/test_cases/container_cases/ulimit.sh | 19 ++ + test/specs/specs/CMakeLists.txt | 1 + + test/specs/specs/specs_ut.cc | 168 ++++++++++++++++++ + 4 files changed, 190 insertions(+), 2 deletions(-) + +diff --git a/CI/test_cases/container_cases/dev_cgroup_rule.sh b/CI/test_cases/container_cases/dev_cgroup_rule.sh +index 5616d37a..33a839c5 100755 +--- a/CI/test_cases/container_cases/dev_cgroup_rule.sh ++++ b/CI/test_cases/container_cases/dev_cgroup_rule.sh +@@ -60,13 +60,13 @@ function test_cpu_dev_cgoup_rule_spec() + def_cid=$(isula run -tid --runtime $runtime -m 10m $image /bin/sh) + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - run container failed" && ((ret++)) + cp $default_config $default_config_bak +- sed -i '/"linux": {/a \ \t\t"devices": [\n\t\t{\n\t\t\t"type": "c",\n\t\t\t"path": "\/dev\/testABC",\n\t\t\t"major": 88,\n\t\t\t"minor": 88\n\t\t}\n\t\t],' $default_config ++ sed -i '/"linux": {/a \ \t\t"devices": [\n\t\t{\n\t\t\t"type": "c",\n\t\t\t"path": "\/dev\/testA",\n\t\t\t"major": 88,\n\t\t\t"minor": 88\n\t\t}\n\t\t],' $default_config + stop_isulad_without_valgrind + start_isulad_with_valgrind --cgroup-parent $test_cgroup_parent + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start isulad failed" && ((ret++)) + isula restart -t 0 $def_cid + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - restart container failed" && ((ret++)) +- cat /sys/fs/cgroup/memory/$test_cgroup_parent/$def_cid/memory.limit_in_bytes | grep ^10485760$ ++ isula exec -it $def_cid sh -c "cat /sys/fs/cgroup/memory/memory.limit_in_bytes | grep ^10485760$" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - --cgroup-parent cannot work" && ((ret++)) + cnt=$(cat ${RUNTIME_ROOT_PATH}/${runtime}/$def_cid/config.json | grep "major\": 88" | wc -l) + [[ $? -ne 0 ]]&& [[ $cnt -ne 2 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check device major failed" && ((ret++)) +diff --git a/CI/test_cases/container_cases/ulimit.sh b/CI/test_cases/container_cases/ulimit.sh +index f823dc1c..41cdcece 100755 +--- a/CI/test_cases/container_cases/ulimit.sh ++++ b/CI/test_cases/container_cases/ulimit.sh +@@ -49,9 +49,28 @@ function test_ulimit() + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - restart failed" && ((ret++)) + cat ${RUNTIME_ROOT_PATH}/${runtime}/$cid/config.json | grep "RLIMIT_" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check rlimit failed after restart" && ((ret++)) ++ ++ check_valgrind_log ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop isulad failed" && ((ret++)) ++ ++ start_isulad_with_valgrind ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start isulad failed" && ((ret++)) ++ ++ # if default ulimit of isulad changed, isula start should do update ulimit of oci spec ++ isula restart -t 0 $cid ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - restart failed" && ((ret++)) ++ cat ${RUNTIME_ROOT_PATH}/${runtime}/$cid/config.json | grep "RLIMIT_" ++ [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check rlimit failed after restart" && ((ret++)) ++ + isula rm -f $cid + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - rm container failed" && ((ret++)) + ++ check_valgrind_log ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop isulad failed" && ((ret++)) ++ ++ start_isulad_with_valgrind --default-ulimit nproc=2048:4096 --default-ulimit nproc=2048:8192 --default-ulimit nofile=1024:4096 ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start isulad failed" && ((ret++)) ++ + isula run --ulimit nproc= $image --runtime $runtime /bin/sh > $ulimitlog 2>&1 + cat $ulimitlog | grep "delimiter '=' can't be the first or the last character" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check failed" && ((ret++)) +diff --git a/test/specs/specs/CMakeLists.txt b/test/specs/specs/CMakeLists.txt +index a9dbc52c..508123fa 100644 +--- a/test/specs/specs/CMakeLists.txt ++++ b/test/specs/specs/CMakeLists.txt +@@ -84,6 +84,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks + ) + ++set_target_properties(${EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,util_common_calloc_s -Wl,--wrap,util_smart_calloc_s -Wl,--wrap,get_readonly_default_oci_spec") + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) + set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/specs/specs/specs_ut.cc b/test/specs/specs/specs_ut.cc +index ad903a3f..47e4ca6e 100644 +--- a/test/specs/specs/specs_ut.cc ++++ b/test/specs/specs/specs_ut.cc +@@ -19,6 +19,7 @@ + #include "mock.h" + #include "isula_libutils/oci_runtime_spec.h" + #include "specs_api.h" ++#include "specs_mount.h" + #include "specs_namespace.h" + #include "specs_security.h" + #include "isula_libutils/host_config.h" +@@ -43,6 +44,46 @@ using ::testing::_; + + using namespace std; + ++static int g_malloc_count = 0; ++static int g_malloc_match = 1; ++ ++extern "C" { ++ DECLARE_WRAPPER_V(util_common_calloc_s, void *, (size_t size)); ++ DEFINE_WRAPPER_V(util_common_calloc_s, void *, (size_t size), (size)); ++ ++ DECLARE_WRAPPER_V(util_smart_calloc_s, void *, (size_t size, size_t len)); ++ DEFINE_WRAPPER_V(util_smart_calloc_s, void *, (size_t size, size_t len), (size, len)); ++ ++ DECLARE_WRAPPER(get_readonly_default_oci_spec, const oci_runtime_spec *, (bool system_container)); ++ DEFINE_WRAPPER(get_readonly_default_oci_spec, const oci_runtime_spec *, (bool system_container), (system_container)); ++} ++ ++void *util_common_calloc_s_fail(size_t size) ++{ ++ g_malloc_count++; ++ ++ if (g_malloc_count == g_malloc_match) { ++ g_malloc_match++; ++ g_malloc_count = 0; ++ return nullptr; ++ } else { ++ return __real_util_common_calloc_s(size); ++ } ++} ++ ++void *util_smart_calloc_s_fail(size_t size, size_t len) ++{ ++ g_malloc_count++; ++ ++ if (g_malloc_count == g_malloc_match) { ++ g_malloc_match++; ++ g_malloc_count = 0; ++ return nullptr; ++ } else { ++ return __real_util_smart_calloc_s(size, len); ++ } ++} ++ + class SpecsUnitTest : public testing::Test { + public: + void SetUp() override +@@ -234,6 +275,32 @@ char *invoke_conf_get_isulad_cgroup_parent() + return util_strdup_s("/var/lib/isulad/engines/lcr"); + } + ++int invoke_conf_get_isulad_default_ulimit_empty(host_config_ulimits_element ***ulimit) ++{ ++ if (ulimit == nullptr) { ++ return -1; ++ } ++ return 0; ++} ++ ++int invoke_conf_get_isulad_default_ulimit(host_config_ulimits_element ***ulimit) ++{ ++ if (ulimit == nullptr) { ++ return -1; ++ } ++ host_config_ulimits_element *ele = static_cast(util_common_calloc_s(sizeof(host_config_ulimits_element))); ++ if (ele == nullptr) { ++ return -1; ++ } ++ ele->hard = 8192; ++ ele->soft = 2048; ++ ele->name = util_strdup_s("NPROC"); ++ ++ int ret = ulimit_array_append(ulimit, ele, ulimit_array_len(*ulimit)); ++ free_host_config_ulimits_element(ele); ++ return ret; ++} ++ + TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_1) + { + ASSERT_EQ(merge_container_cgroups_path(nullptr, nullptr), nullptr); +@@ -347,6 +414,107 @@ TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_5) + testing::Mock::VerifyAndClearExpectations(&m_isulad_conf); + } + ++TEST_F(SpecsUnitTest, test_update_oci_container_cgroups_path) ++{ ++ parser_error err = nullptr; ++ host_config *hostspec = static_cast(util_common_calloc_s(sizeof(host_config))); ++ ASSERT_NE(hostspec, nullptr); ++ ++ oci_runtime_spec *ocispec = oci_runtime_spec_parse_data("{\"ociVersion\": \"1.0.1\", \"linux\": \ ++ {} }", nullptr, &err); ++ ASSERT_NE(ocispec, nullptr); ++ ++ ocispec->linux->cgroups_path = util_strdup_s("/isulad"); ++ ASSERT_EQ(update_oci_container_cgroups_path("abcdef", nullptr, nullptr), -1); ++ EXPECT_CALL(m_isulad_conf, GetCgroupParent()).WillRepeatedly(Invoke(invoke_conf_get_isulad_cgroup_parent)); ++ ASSERT_EQ(update_oci_container_cgroups_path("abcdef", ocispec, hostspec), 0); ++ ASSERT_STREQ(ocispec->linux->cgroups_path, "/var/lib/isulad/engines/lcr/abcdef"); ++ ++ free(err); ++ free_host_config(hostspec); ++ free_oci_runtime_spec(ocispec); ++ ++ testing::Mock::VerifyAndClearExpectations(&m_isulad_conf); ++} ++ ++TEST_F(SpecsUnitTest, test_update_oci_ulimit) ++{ ++ parser_error err = nullptr; ++ host_config *hostspec = static_cast(util_common_calloc_s(sizeof(host_config))); ++ ASSERT_NE(hostspec, nullptr); ++ ++ char *oci_config_file = json_path(OCI_RUNTIME_SPEC_FILE); ++ ASSERT_TRUE(oci_config_file != nullptr); ++ oci_runtime_spec *ocispec = oci_runtime_spec_parse_file(oci_config_file, nullptr, &err); ++ ASSERT_NE(ocispec, nullptr); ++ ++ ASSERT_EQ(update_oci_ulimit(nullptr, nullptr), -1); ++ EXPECT_CALL(m_isulad_conf, GetUlimit(_)).WillRepeatedly(Invoke(invoke_conf_get_isulad_default_ulimit)); ++ ASSERT_EQ(update_oci_ulimit(ocispec, hostspec), 0); ++ ASSERT_EQ(ocispec->process->rlimits_len, 1); ++ ASSERT_EQ(ocispec->process->rlimits[0]->hard, 8192); ++ ASSERT_EQ(ocispec->process->rlimits[0]->soft, 2048); ++ ASSERT_STREQ(ocispec->process->rlimits[0]->type, "RLIMIT_NPROC"); ++ EXPECT_CALL(m_isulad_conf, GetUlimit(_)).WillRepeatedly(Invoke(invoke_conf_get_isulad_default_ulimit_empty)); ++ ASSERT_EQ(update_oci_ulimit(ocispec, hostspec), 0); ++ ASSERT_EQ(ocispec->process->rlimits_len, 0); ++ ++ free(err); ++ free(oci_config_file); ++ free_host_config(hostspec); ++ free_oci_runtime_spec(ocispec); ++ testing::Mock::VerifyAndClearExpectations(&m_isulad_conf); ++} ++ ++TEST_F(SpecsUnitTest, test_update_devcies_for_oci_spec) ++{ ++ parser_error err = nullptr; ++ oci_runtime_spec *readonly_spec = oci_runtime_spec_parse_data("{\"ociVersion\": \"1.0.1\", \"linux\": \ ++ { \"devices\": \ ++ [ { \"type\": \"c\", \"path\": \"/dev/testA\", \ ++ \"fileMode\": 8612, \"major\": 99, \"minor\": 99} ], \ ++ \"resources\": { \"devices\": [ { \"allow\": false, \ ++ \"type\": \"a\", \"major\": -1, \ ++ \"minor\": -1, \"access\": \"rwm\" } ] } } }", nullptr, &err); ++ ASSERT_NE(readonly_spec, nullptr); ++ free(err); ++ err = nullptr; ++ host_config *hostspec = static_cast(util_common_calloc_s(sizeof(host_config))); ++ ASSERT_NE(hostspec, nullptr); ++ ++ oci_runtime_spec *ocispec = oci_runtime_spec_parse_data("{\"ociVersion\": \"1.0.1\", \"linux\": \ ++ { \"devices\": [ ], \ ++ \"resources\": { \"devices\": [ ] } } }", nullptr, &err); ++ ASSERT_NE(ocispec, nullptr); ++ ++ MOCK_SET(get_readonly_default_oci_spec, readonly_spec); ++ MOCK_SET_V(util_smart_calloc_s, util_smart_calloc_s_fail); ++ MOCK_SET_V(util_common_calloc_s, util_common_calloc_s_fail); ++ ++ ASSERT_EQ(update_devcies_for_oci_spec(ocispec, hostspec), -1); ++ ASSERT_EQ(update_devcies_for_oci_spec(ocispec, hostspec), -1); ++ ASSERT_EQ(update_devcies_for_oci_spec(ocispec, hostspec), -1); ++ free(ocispec->linux->devices[0]); ++ free(ocispec->linux->devices); ++ ocispec->linux->devices = NULL; ++ ocispec->linux->devices_len = 0; ++ ASSERT_EQ(update_devcies_for_oci_spec(ocispec, hostspec), -1); ++ free(ocispec->linux->devices[0]); ++ free(ocispec->linux->devices); ++ ocispec->linux->devices = NULL; ++ ocispec->linux->devices_len = 0; ++ ASSERT_EQ(update_devcies_for_oci_spec(ocispec, hostspec), 0); ++ ++ MOCK_CLEAR(get_readonly_default_oci_spec); ++ MOCK_CLEAR(util_smart_calloc_s); ++ MOCK_CLEAR(util_common_calloc_s); ++ ++ free_oci_runtime_spec(readonly_spec); ++ free_oci_runtime_spec(ocispec); ++ free_host_config(hostspec); ++ free(err); ++} ++ + /********************************* UT for merge caps *******************************************/ + struct capabilities_lens { + size_t bounding_len; +-- +2.34.1 + diff --git a/0010-open-run-container-with-dev-volume-testcase.patch b/0010-open-run-container-with-dev-volume-testcase.patch new file mode 100644 index 0000000..2381125 --- /dev/null +++ b/0010-open-run-container-with-dev-volume-testcase.patch @@ -0,0 +1,30 @@ +From 44d15a7451a922ca7266b756d3f9a83908199cb3 Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu +Date: Tue, 23 Jan 2024 10:35:59 +0800 +Subject: [PATCH 10/43] open run container with dev volume testcase + +Signed-off-by: zhangxiaoyu +--- + CI/test_cases/container_cases/bind_special_dir.sh | 5 ++--- + 1 file changed, 2 insertions(+), 3 deletions(-) + +diff --git a/CI/test_cases/container_cases/bind_special_dir.sh b/CI/test_cases/container_cases/bind_special_dir.sh +index 545d5099..04bf437a 100755 +--- a/CI/test_cases/container_cases/bind_special_dir.sh ++++ b/CI/test_cases/container_cases/bind_special_dir.sh +@@ -40,10 +40,9 @@ function test_bind_special_dir() + # when create container in container, runc not support to mount /dev + # adapt fedora base image, we just remove rshared option of sys dir + if [ $runtime == "runc" ]; then +- c_id=`isula run -itd -v -itd --runtime=$runtime -v /sys/fs:/sys/fs:rw -v /proc:/proc -v /dev/pts:/dev/pts:rw busybox sh` ++ c_id=`isula run -itd --runtime=$runtime -v /sys/fs:/sys/fs:rw -v /proc:/proc -v /dev/pts:/dev/pts:rw busybox sh` + else +- # lxc 5.X cannot support mount /dev directory +- c_id=`isula run --runtime=$runtime -itd -v -itd -v /sys/fs:/sys/fs:rw -v /proc:/proc busybox sh` ++ c_id=`isula run --runtime=$runtime -itd -v /sys/fs:/sys/fs:rw -v /proc:/proc -v /dev:/dev:ro -v /dev/pts:/dev/pts:rw busybox sh` + fi + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + +-- +2.34.1 + diff --git a/0011-add-cpu-usage-nano-cores-for-sandbox.patch b/0011-add-cpu-usage-nano-cores-for-sandbox.patch new file mode 100644 index 0000000..27be05d --- /dev/null +++ b/0011-add-cpu-usage-nano-cores-for-sandbox.patch @@ -0,0 +1,81 @@ +From 3dc12d7806fda8d5ceee183595e993079bee4056 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Fri, 12 Jan 2024 17:38:09 +0800 +Subject: [PATCH 11/43] add cpu usage nano cores for sandbox + +Signed-off-by: jikai +--- + .../v1/v1_cri_pod_sandbox_manager_service.cc | 17 ++++++++++++++--- + .../cri/v1/v1_cri_pod_sandbox_manager_service.h | 1 + + 2 files changed, 15 insertions(+), 3 deletions(-) + +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +index a0c45111..76fa17bc 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +@@ -1024,6 +1024,7 @@ void PodSandboxManagerService::PodSandboxStatsToGRPC(const std::string &id, cons + const std::vector &netMetrics, + const std::unique_ptr &containerManager, + std::unique_ptr &podStats, ++ sandbox::StatsInfo &oldStatsRec, + Errors &error) + { + std::unique_ptr podStatsPtr( +@@ -1044,8 +1045,13 @@ void PodSandboxManagerService::PodSandboxStatsToGRPC(const std::string &id, cons + auto cpu = podStatsPtr->mutable_linux()->mutable_cpu(); + cpu->set_timestamp(timestamp); + cpu->mutable_usage_core_nano_seconds()->set_value(cgroupMetrics.cgcpu_metrics.cpu_use_nanos); +- // todo +- // cpu->mutable_usage_nano_cores()->set_value(getNanoCores()); ++ if (oldStatsRec.cpuUseNanos != 0 && timestamp > oldStatsRec.timestamp && ++ cgroupMetrics.cgcpu_metrics.cpu_use_nanos > oldStatsRec.cpuUseNanos) { ++ uint64_t usage = cgroupMetrics.cgcpu_metrics.cpu_use_nanos - oldStatsRec.cpuUseNanos; ++ uint64_t nanoSeconds = timestamp - oldStatsRec.timestamp; ++ uint64_t usage_nano_cores = (uint64_t)(((double)usage / (double)nanoSeconds) * (double)Time_Second); ++ cpu->mutable_usage_nano_cores()->set_value(usage_nano_cores); ++ } + + // Memory + auto memory = podStatsPtr->mutable_linux()->mutable_memory(); +@@ -1114,6 +1120,7 @@ auto PodSandboxManagerService::PodSandboxStats(const std::string &podSandboxID, + return nullptr; + } + auto &config = sandbox->GetSandboxConfig(); ++ auto oldStatsRec = sandbox->GetStatsInfo(); + + auto status = PodSandboxStatus(sandbox->GetId(), tmpErr); + if (error.NotEmpty()) { +@@ -1136,13 +1143,17 @@ auto PodSandboxManagerService::PodSandboxStats(const std::string &podSandboxID, + tmpErr.Clear(); + } + +- PodSandboxStatsToGRPC(sandbox->GetId(), cgroupMetrics, netMetrics, containerManager, podStats, tmpErr); ++ PodSandboxStatsToGRPC(sandbox->GetId(), cgroupMetrics, netMetrics, containerManager, podStats, oldStatsRec, tmpErr); + if (tmpErr.NotEmpty()) { + ERROR("Failed to set PodSandboxStats: %s", tmpErr.GetCMessage()); + error.Errorf("Failed to set PodSandboxStats"); + return nullptr; + } + ++ // update stats info that sandbox recorded ++ sandbox::StatsInfo newStatsRec { podStats->linux().cpu().timestamp(), podStats->linux().cpu().usage_core_nano_seconds().value() }; ++ sandbox->UpdateStatsInfo(newStatsRec); ++ + return podStats; + } + +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h +index 2bd28007..c3d98b8c 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h +@@ -123,6 +123,7 @@ private: + const std::vector &netMetrics, + const std::unique_ptr &containerManager, + std::unique_ptr &podStats, ++ sandbox::StatsInfo &statsInfo, + Errors &error); + void GetFilterPodSandbox(const runtime::v1::PodSandboxStatsFilter *filter, + std::vector &podSandboxIDs, Errors &error); +-- +2.34.1 + diff --git a/0012-sleep-some-time-in-ServiceWorkThread-to-prevent-the-.patch b/0012-sleep-some-time-in-ServiceWorkThread-to-prevent-the-.patch new file mode 100644 index 0000000..9d70f92 --- /dev/null +++ b/0012-sleep-some-time-in-ServiceWorkThread-to-prevent-the-.patch @@ -0,0 +1,27 @@ +From 384cf7870c155d41f742b1928a4cb1b56aa46c94 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Tue, 6 Feb 2024 20:05:05 +0800 +Subject: [PATCH 12/43] sleep some time in ServiceWorkThread to prevent the CPU + from being occupied all the time + +Signed-off-by: zhongtao +--- + src/daemon/entry/cri/streams/websocket/ws_server.cc | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/src/daemon/entry/cri/streams/websocket/ws_server.cc b/src/daemon/entry/cri/streams/websocket/ws_server.cc +index 6319a67f..a8d89b36 100644 +--- a/src/daemon/entry/cri/streams/websocket/ws_server.cc ++++ b/src/daemon/entry/cri/streams/websocket/ws_server.cc +@@ -551,6 +551,8 @@ void WebsocketServer::ServiceWorkThread(int threadid) + + while (n >= 0 && m_forceExit == 0) { + n = lws_service(m_context, 0); ++ // sleep some time to prevent the CPU from being occupied all the time ++ std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + } + +-- +2.34.1 + diff --git a/0013-restore-name-for-rename-failed.patch b/0013-restore-name-for-rename-failed.patch new file mode 100644 index 0000000..be7d59c --- /dev/null +++ b/0013-restore-name-for-rename-failed.patch @@ -0,0 +1,29 @@ +From 2df7a67ad2cb0249b18ca5eba46f9aab8f72038f Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Sun, 18 Feb 2024 11:32:55 +0800 +Subject: [PATCH 13/43] restore name for rename failed + +Signed-off-by: zhongtao +--- + src/daemon/executor/container_cb/execution_information.c | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/src/daemon/executor/container_cb/execution_information.c b/src/daemon/executor/container_cb/execution_information.c +index 2a71e82a..c02cc830 100644 +--- a/src/daemon/executor/container_cb/execution_information.c ++++ b/src/daemon/executor/container_cb/execution_information.c +@@ -1116,6 +1116,11 @@ static void restore_names_at_fail(container_t *cont, const char *ori_name, const + if (!container_name_index_rename(ori_name, new_name, id)) { + ERROR("Failed to restore name from \"%s\" to \"%s\" for container %s", new_name, ori_name, id); + } ++ ++ // restore name in id-name manager ++ if (!id_name_manager_rename(ori_name, new_name)) { ++ ERROR("Failed to restore name from \"%s\" to \"%s\" in id-name manager", new_name, ori_name); ++ } + } + + static int container_rename(container_t *cont, const char *new_name) +-- +2.34.1 + diff --git a/0014-2371-Allow-iSulad-to-pull-load-image-with-symlink.patch b/0014-2371-Allow-iSulad-to-pull-load-image-with-symlink.patch new file mode 100644 index 0000000..4a1fa1f --- /dev/null +++ b/0014-2371-Allow-iSulad-to-pull-load-image-with-symlink.patch @@ -0,0 +1,45 @@ +From fd4c80b8de768d7132cef0720cd46167173a653b Mon Sep 17 00:00:00 2001 +From: xuxuepeng +Date: Mon, 19 Feb 2024 01:05:18 +0000 +Subject: [PATCH 14/43] !2371 Allow iSulad to pull/load image with symlink * + Allow iSulad to pull/load image with symlink + +--- + src/utils/tar/util_archive.c | 13 ++++++++++--- + 1 file changed, 10 insertions(+), 3 deletions(-) + +diff --git a/src/utils/tar/util_archive.c b/src/utils/tar/util_archive.c +index e4c302bc..52b51162 100644 +--- a/src/utils/tar/util_archive.c ++++ b/src/utils/tar/util_archive.c +@@ -637,6 +637,11 @@ static void try_to_replace_exited_dst(const char *dst_path, struct archive_entry + } + } + ++/** ++ * This function has to be used with chroot to prevent a potential attack from manipulating ++ * the path of the file to be extracted, such as using a symbolic link to extract the file to ++ * a location outside the path. ++ */ + int archive_unpack_handler(const struct io_read_wrapper *content, const struct archive_options *options) + { + int ret = 0; +@@ -671,10 +676,12 @@ int archive_unpack_handler(const struct io_read_wrapper *content, const struct a + flags |= ARCHIVE_EXTRACT_PERM; + flags |= ARCHIVE_EXTRACT_ACL; + flags |= ARCHIVE_EXTRACT_FFLAGS; +- flags |= ARCHIVE_EXTRACT_SECURE_SYMLINKS; +- flags |= ARCHIVE_EXTRACT_SECURE_NODOTDOT; + flags |= ARCHIVE_EXTRACT_XATTR; +- flags |= ARCHIVE_EXTRACT_SECURE_NOABSOLUTEPATHS; ++ /** ++ * ARCHIVE_EXTRACT_SECURE_SYMLINKS, ARCHIVE_EXTRACT_SECURE_NODOTDOT, ++ * ARCHIVE_EXTRACT_SECURE_NOABSOLUTEPATHS flags are not set here, ++ * since this function is called after chroot, the security of the path is guaranteed. ++ */ + + a = archive_read_new(); + if (a == NULL) { +-- +2.34.1 + diff --git a/0015-Replace-http-parser-dependency-with-lcr.patch b/0015-Replace-http-parser-dependency-with-lcr.patch new file mode 100644 index 0000000..eccb44e --- /dev/null +++ b/0015-Replace-http-parser-dependency-with-lcr.patch @@ -0,0 +1,427 @@ +From fb76605985166c4d2172270c8d633ed26d62f698 Mon Sep 17 00:00:00 2001 +From: xuxuepeng +Date: Mon, 19 Feb 2024 23:52:47 +0800 +Subject: [PATCH 15/43] Replace http-parser dependency with lcr + +Signed-off-by: xuxuepeng +--- + CI/dockerfiles/Dockerfile-centos | 13 ------------- + CI/dockerfiles/Dockerfile-fedora | 1 - + CI/dockerfiles/Dockerfile-ubuntu | 1 - + CI/pr-gateway.sh | 2 +- + Dockerfile | 13 ------------- + cmake/checker.cmake | 5 ----- + docs/build_docs/guide/build_guide.md | 13 ------------- + docs/build_docs/guide/build_guide_with_rpm.md | 4 +--- + docs/build_docs/guide/build_guide_with_rpm_zh.md | 4 +--- + docs/build_docs/guide/build_guide_zh.md | 13 ------------- + .../guide/script/install_iSulad_on_Centos_7.sh | 11 ----------- + .../script/install_iSulad_on_Ubuntu_20_04_LTS.sh | 2 +- + .../isulad_build_in_openeuler.Dockerfile | 2 +- + docs/vs_other_engines/vs_docker_command.md | 1 - + iSulad.spec | 3 +-- + src/CMakeLists.txt | 1 - + src/contrib/env_checkconfig | 1 - + .../modules/image/oci/registry/registry_apiv1.c | 2 +- + .../modules/image/oci/registry/registry_apiv2.c | 2 +- + src/utils/http/CMakeLists.txt | 2 +- + src/utils/http/parser.c | 2 +- + src/utils/http/parser.h | 2 +- + src/utils/http/rest_common.c | 2 +- + 23 files changed, 12 insertions(+), 90 deletions(-) + +diff --git a/CI/dockerfiles/Dockerfile-centos b/CI/dockerfiles/Dockerfile-centos +index 1d76b4ec..af3ce035 100644 +--- a/CI/dockerfiles/Dockerfile-centos ++++ b/CI/dockerfiles/Dockerfile-centos +@@ -207,19 +207,6 @@ RUN export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH && \ + make install && \ + ldconfig + +-# install http-parser +-RUN export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH && \ +- set -x && \ +- cd ~ && \ +- git clone https://gitee.com/src-openeuler/http-parser.git && \ +- cd http-parser && \ +- git checkout -b openEuler-20.03-LTS-tag openEuler-20.03-LTS-tag && \ +- tar -xzvf http-parser-2.9.2.tar.gz && \ +- cd http-parser-2.9.2 && \ +- make -j CFLAGS="-Wno-error" && \ +- make CFLAGS="-Wno-error" install && \ +- ldconfig +- + # install libwebsockets + RUN export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH && \ + set -x && \ +diff --git a/CI/dockerfiles/Dockerfile-fedora b/CI/dockerfiles/Dockerfile-fedora +index a105cbb4..be2bf412 100644 +--- a/CI/dockerfiles/Dockerfile-fedora ++++ b/CI/dockerfiles/Dockerfile-fedora +@@ -42,7 +42,6 @@ RUN dnf update -y && dnf install -y automake \ + grpc-plugins \ + libevent-devel \ + libwebsockets-devel \ +- http-parser-devel \ + gtest-devel \ + gmock-devel \ + libarchive-devel \ +diff --git a/CI/dockerfiles/Dockerfile-ubuntu b/CI/dockerfiles/Dockerfile-ubuntu +index 2441a7ce..09a20eb5 100644 +--- a/CI/dockerfiles/Dockerfile-ubuntu ++++ b/CI/dockerfiles/Dockerfile-ubuntu +@@ -71,7 +71,6 @@ RUN apt update -y && apt upgrade -y && \ + language-pack-en \ + curl \ + cmake \ +- libhttp-parser-dev \ + libprotobuf-dev \ + libgrpc-dev \ + libgrpc++-dev \ +diff --git a/CI/pr-gateway.sh b/CI/pr-gateway.sh +index e5bf627e..e3613e8e 100755 +--- a/CI/pr-gateway.sh ++++ b/CI/pr-gateway.sh +@@ -22,7 +22,7 @@ sed -i "s#http://repo.openeuler.org#https://repo.huaweicloud.com/openeuler#g" /e + + dnf update -y + +-dnf install -y docbook2X doxygen gtest-devel gmock-devel diffutils cmake gcc-c++ yajl-devel patch make libtool libevent-devel libevhtp-devel grpc grpc-plugins grpc-devel protobuf-devel libcurl libcurl-devel sqlite-devel libarchive-devel device-mapper-devel http-parser-devel libseccomp-devel libcap-devel libselinux-devel libwebsockets libwebsockets-devel systemd-devel git chrpath ncurses-devel ++dnf install -y docbook2X doxygen gtest-devel gmock-devel diffutils cmake gcc-c++ yajl-devel patch make libtool libevent-devel libevhtp-devel grpc grpc-plugins grpc-devel protobuf-devel libcurl libcurl-devel sqlite-devel libarchive-devel device-mapper-devel libseccomp-devel libcap-devel libselinux-devel libwebsockets libwebsockets-devel systemd-devel git chrpath ncurses-devel + if [ $? -ne 0 ]; then + echo "install dependences failed" + exit 1 +diff --git a/Dockerfile b/Dockerfile +index 3b284630..a30ed12a 100644 +--- a/Dockerfile ++++ b/Dockerfile +@@ -197,19 +197,6 @@ RUN export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH && \ + make install && \ + ldconfig + +-# install http-parser +-RUN export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH && \ +- set -x && \ +- cd ~ && \ +- git clone https://gitee.com/src-openeuler/http-parser.git && \ +- cd http-parser && \ +- git checkout -b openEuler-20.03-LTS-tag openEuler-20.03-LTS-tag && \ +- tar -xzvf http-parser-2.9.2.tar.gz && \ +- cd http-parser-2.9.2 && \ +- make -j CFLAGS="-Wno-error" && \ +- make CFLAGS="-Wno-error" install && \ +- ldconfig +- + # install libwebsockets + RUN export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH && \ + set -x && \ +diff --git a/cmake/checker.cmake b/cmake/checker.cmake +index b0c395ef..13f0fd62 100644 +--- a/cmake/checker.cmake ++++ b/cmake/checker.cmake +@@ -66,11 +66,6 @@ if (ANDROID OR MUSL) + _CHECK(CRYPTO_LIBRARY "LIBSSL_LIBRARY-NOTFOUND" "libssl.so") + endif() + +-find_path(HTTP_PARSER_INCLUDE_DIR http_parser.h) +-_CHECK(HTTP_PARSER_INCLUDE_DIR "HTTP_PARSER_INCLUDE_DIR-NOTFOUND" "http_parser.h") +-find_library(HTTP_PARSER_LIBRARY http_parser) +-_CHECK(HTTP_PARSER_LIBRARY "HTTP_PARSER_LIBRARY-NOTFOUND" "libhttp_parser.so") +- + pkg_check_modules(PC_CURL "libcurl>=7.4.0") + find_path(CURL_INCLUDE_DIR "curl/curl.h" + HINTS ${PC_CURL_INCLUDEDIR} ${PC_CURL_INCLUDE_DIRS}) +diff --git a/docs/build_docs/guide/build_guide.md b/docs/build_docs/guide/build_guide.md +index 741abddd..73a0d9d8 100644 +--- a/docs/build_docs/guide/build_guide.md ++++ b/docs/build_docs/guide/build_guide.md +@@ -150,19 +150,6 @@ $ sudo -E make install + $ sudo -E ldconfig + ``` + +-#### build and install http-parser +- +-```bash +-$ git clone https://gitee.com/src-openeuler/http-parser.git +-$ cd http-parser +-$ git checkout openEuler-20.03-LTS-tag +-$ tar -xzvf http-parser-2.9.2.tar.gz +-$ cd http-parser-2.9.2 +-$ sudo -E make -j CFLAGS="-Wno-error" +-$ sudo -E make CFLAGS="-Wno-error" install +-$ sudo -E ldconfig +-``` +- + #### build and install libwebsockets + + ```bash +diff --git a/docs/build_docs/guide/build_guide_with_rpm.md b/docs/build_docs/guide/build_guide_with_rpm.md +index 181b2ef5..acf8e7c0 100644 +--- a/docs/build_docs/guide/build_guide_with_rpm.md ++++ b/docs/build_docs/guide/build_guide_with_rpm.md +@@ -29,7 +29,7 @@ BUILD BUILDROOT RPMS SOURCES SPECS SRPMS + + ```shell + dnf install -y patch automake autoconf libtool cmake make libcap libcap-devel libselinux libselinux-devel libseccomp libseccomp-devel git libcgroup tar python3 python3-pip libcurl-devel zlib-devel glibc-headers openssl-devel gcc gcc-c++ systemd-devel systemd-libs golang libtar && \ +-dnf --enablerepo=powertools install -y yajl-devel device-mapper-devel http-parser-devel && \ ++dnf --enablerepo=powertools install -y yajl-devel device-mapper-devel && \ + dnf install -y epel-release && \ + dnf --enablerepo=powertools install libuv-devel &&\ + dnf install libwebsockets-devel +@@ -281,7 +281,6 @@ rpm -Uvh libarchive-devel-3.4.3-4.x86_64.rpm + ### 9.1 install iSulad dependencies + + ```shell +-dnf --enablerepo=powertools install http-parser-devel + dnf install -y sqlite-devel + ``` + +@@ -317,7 +316,6 @@ dnf install libwebsockets-devel + then, you can install iSulad + + ```shell +-dnf --enablerepo=powertools install http-parser-devel + dnf install -y sqlite-devel.x86_64 + rpm -Uvh iSulad-2.1.0-1.x86_64.rpm + ``` +\ No newline at end of file +diff --git a/docs/build_docs/guide/build_guide_with_rpm_zh.md b/docs/build_docs/guide/build_guide_with_rpm_zh.md +index edb565e3..b9574b4e 100644 +--- a/docs/build_docs/guide/build_guide_with_rpm_zh.md ++++ b/docs/build_docs/guide/build_guide_with_rpm_zh.md +@@ -25,7 +25,7 @@ BUILD BUILDROOT RPMS SOURCES SPECS SRPMS + + ```shell + dnf install -y patch automake autoconf libtool cmake make libcap libcap-devel libselinux libselinux-devel libseccomp libseccomp-devel git libcgroup tar python3 python3-pip libcurl-devel zlib-devel glibc-headers openssl-devel gcc gcc-c++ systemd-devel systemd-libs golang libtar && \ +-dnf --enablerepo=powertools install -y yajl-devel device-mapper-devel http-parser-devel && \ ++dnf --enablerepo=powertools install -y yajl-devel device-mapper-devel && \ + dnf install -y epel-release && \ + dnf --enablerepo=powertools install libuv-devel &&\ + dnf install libwebsockets-devel +@@ -272,7 +272,6 @@ rpm -Uvh libarchive-devel-3.4.3-4.x86_64.rpm + ### 9.1 安装iSulad的依赖 + + ```shell +-dnf --enablerepo=powertools install http-parser-devel + dnf install -y sqlite-devel + ``` + +@@ -310,7 +309,6 @@ dnf install libwebsockets-devel + 再安装isulad： + + ```shell +-dnf --enablerepo=powertools install http-parser-devel + dnf install -y sqlite-devel.x86_64 + rpm -Uvh iSulad-2.1.0-1.x86_64.rpm + ``` +\ No newline at end of file +diff --git a/docs/build_docs/guide/build_guide_zh.md b/docs/build_docs/guide/build_guide_zh.md +index bfdc69dc..2d853d9c 100644 +--- a/docs/build_docs/guide/build_guide_zh.md ++++ b/docs/build_docs/guide/build_guide_zh.md +@@ -150,19 +150,6 @@ $ sudo -E make install + $ sudo -E ldconfig + ``` + +-#### 编译安装http-parser +- +-```bash +-$ git clone https://gitee.com/src-openeuler/http-parser.git +-$ cd http-parser +-$ git checkout openEuler-20.03-LTS-tag +-$ tar -xzvf http-parser-2.9.2.tar.gz +-$ cd http-parser-2.9.2 +-$ sudo -E make -j CFLAGS="-Wno-error" +-$ sudo -E make CFLAGS="-Wno-error" install +-$ sudo -E ldconfig +-``` +- + #### 编译安装libwebsockets + + ```bash +diff --git a/docs/build_docs/guide/script/install_iSulad_on_Centos_7.sh b/docs/build_docs/guide/script/install_iSulad_on_Centos_7.sh +index 3834d333..b268d777 100755 +--- a/docs/build_docs/guide/script/install_iSulad_on_Centos_7.sh ++++ b/docs/build_docs/guide/script/install_iSulad_on_Centos_7.sh +@@ -68,17 +68,6 @@ make -j $(nproc) + make install + ldconfig + +-# build http_parser +-cd $BUILD_DIR +-git clone https://gitee.com/src-openeuler/http-parser.git +-cd http-parser +-git checkout openEuler-20.03-LTS-tag +-tar -xzvf http-parser-2.9.2.tar.gz +-cd http-parser-2.9.2 +-make -j CFLAGS="-Wno-error" +-make CFLAGS="-Wno-error" install +-ldconfig +- + # build libwebsockets + cd $BUILD_DIR + git clone https://gitee.com/src-openeuler/libwebsockets.git +diff --git a/docs/build_docs/guide/script/install_iSulad_on_Ubuntu_20_04_LTS.sh b/docs/build_docs/guide/script/install_iSulad_on_Ubuntu_20_04_LTS.sh +index 4f27244e..f44bddb4 100755 +--- a/docs/build_docs/guide/script/install_iSulad_on_Ubuntu_20_04_LTS.sh ++++ b/docs/build_docs/guide/script/install_iSulad_on_Ubuntu_20_04_LTS.sh +@@ -7,7 +7,7 @@ set -e + export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH + export LD_LIBRARY_PATH=/usr/local/lib:/usr/lib:/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH + echo "/usr/local/lib" >> /etc/ld.so.conf +-apt install -y g++ libprotobuf-dev protobuf-compiler protobuf-compiler-grpc libgrpc++-dev libgrpc-dev libtool automake autoconf cmake make pkg-config libyajl-dev zlib1g-dev libselinux1-dev libseccomp-dev libcap-dev libsystemd-dev git libarchive-dev libcurl4-gnutls-dev openssl libdevmapper-dev python3 libtar0 libtar-dev libhttp-parser-dev libwebsockets-dev ++apt install -y g++ libprotobuf-dev protobuf-compiler protobuf-compiler-grpc libgrpc++-dev libgrpc-dev libtool automake autoconf cmake make pkg-config libyajl-dev zlib1g-dev libselinux1-dev libseccomp-dev libcap-dev libsystemd-dev git libarchive-dev libcurl4-gnutls-dev openssl libdevmapper-dev python3 libtar0 libtar-dev libwebsockets-dev + + BUILD_DIR=/tmp/build_isulad + +diff --git a/docs/dockerfiles/isulad_build_in_openeuler.Dockerfile b/docs/dockerfiles/isulad_build_in_openeuler.Dockerfile +index 5049f783..a081b009 100644 +--- a/docs/dockerfiles/isulad_build_in_openeuler.Dockerfile ++++ b/docs/dockerfiles/isulad_build_in_openeuler.Dockerfile +@@ -26,7 +26,7 @@ RUN dnf install -y rust rust-packaging cargo + RUN dnf install -y grpc grpc-plugins grpc-devel protobuf-devel libwebsockets libwebsockets-devel + + # depends for image module and restful client of iSulad +-RUN dnf install -y libcurl libcurl-devel libarchive-devel http-parser-devel ++RUN dnf install -y libcurl libcurl-devel libarchive-devel + + # depends for embedded image of iSulad: -DENABLE_EMBEDDED=ON + RUN dnf install -y sqlite-devel +diff --git a/docs/vs_other_engines/vs_docker_command.md b/docs/vs_other_engines/vs_docker_command.md +index 5c38d31b..d7b587ea 100644 +--- a/docs/vs_other_engines/vs_docker_command.md ++++ b/docs/vs_other_engines/vs_docker_command.md +@@ -114,7 +114,6 @@ Installing dependencies: + abseil-cpp + clibcni + grpc +- http-parser + lcr + lib-shim-v2 + libwebsockets +diff --git a/iSulad.spec b/iSulad.spec +index eda87f7a..eafa82a0 100644 +--- a/iSulad.spec ++++ b/iSulad.spec +@@ -48,7 +48,6 @@ BuildRequires: libisula-devel > %{lcrver_lower} libisula-devel < %{lcrver_upper} + BuildRequires: cmake gcc-c++ yajl-devel + BuildRequires: grpc grpc-plugins grpc-devel protobuf-devel + BuildRequires: libcurl libcurl-devel libarchive-devel device-mapper-devel +-BuildRequires: http-parser-devel + BuildRequires: libseccomp-devel libcap-devel libselinux-devel libwebsockets libwebsockets-devel + BuildRequires: systemd-devel git + BuildRequires: libevhtp-devel libevent-devel +@@ -60,7 +59,7 @@ BuildRequires: lib-shim-v2 lib-shim-v2-devel + Requires: libisula > %{lcrver_lower} libisula < %{lcrver_upper} + Requires: grpc protobuf + Requires: libcurl +-Requires: http-parser libseccomp ++Requires: libseccomp + Requires: libcap libselinux libwebsockets libarchive device-mapper + Requires: systemd + Requires: (docker-runc or runc) +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index d1bc65f9..48c1bad0 100644 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -3,7 +3,6 @@ set(CHECKED_INCLUDE_DIRS + ${STD_HEADER_SYS_PARAM} + ${LIBYAJL_INCLUDE_DIR} + ${LIBARCHIVE_INCLUDE_DIR} +- ${HTTP_PARSER_INCLUDE_DIR} + ${OPENSSL_INCLUDE_DIR} + ${CURL_INCLUDE_DIR} + ${SYSTEMD_INCLUDE_DIR} +diff --git a/src/contrib/env_checkconfig b/src/contrib/env_checkconfig +index 62a91354..9080e01a 100755 +--- a/src/contrib/env_checkconfig ++++ b/src/contrib/env_checkconfig +@@ -164,7 +164,6 @@ config_set CONFIG_SMP && echo -n "Cpuset Cgroup Result: " && config_enable CONFI + echo "" + echo "--- Third-party Packages ---" + echo -n "libyajl: " && has_lib libyajl +-echo -n "libhttp_parser: " && has_lib libhttp_parser + echo -n "libevhtp.so.1.2.16: " && has_lib libevhtp.so.1.2.16 + echo -n "libseccomp: " && has_lib libseccomp + echo -n "libcap.so: " && has_lib libcap.so +diff --git a/src/daemon/modules/image/oci/registry/registry_apiv1.c b/src/daemon/modules/image/oci/registry/registry_apiv1.c +index 414eb65c..6da24c1d 100644 +--- a/src/daemon/modules/image/oci/registry/registry_apiv1.c ++++ b/src/daemon/modules/image/oci/registry/registry_apiv1.c +@@ -18,7 +18,7 @@ + #include + #include + #include +-#include ++#include + #include + #include + #include +diff --git a/src/daemon/modules/image/oci/registry/registry_apiv2.c b/src/daemon/modules/image/oci/registry/registry_apiv2.c +index 3b3bbd93..dd49fab7 100644 +--- a/src/daemon/modules/image/oci/registry/registry_apiv2.c ++++ b/src/daemon/modules/image/oci/registry/registry_apiv2.c +@@ -18,7 +18,7 @@ + #include + #include + #include +-#include ++#include + #include + #include + #include +diff --git a/src/utils/http/CMakeLists.txt b/src/utils/http/CMakeLists.txt +index ad7d0747..23d92cdc 100644 +--- a/src/utils/http/CMakeLists.txt ++++ b/src/utils/http/CMakeLists.txt +@@ -20,7 +20,7 @@ target_include_directories(libhttpclient PUBLIC + + # set libhttpclient FLAGS + set_target_properties(libhttpclient PROPERTIES PREFIX "") +-target_link_libraries(libhttpclient ${HTTP_PARSER_LIBRARY} ${CURL_LIBRARY}) ++target_link_libraries(libhttpclient ${CURL_LIBRARY}) + + set_target_properties(libhttpclient PROPERTIES LINKER_LANGUAGE "C") + +diff --git a/src/utils/http/parser.c b/src/utils/http/parser.c +index cf8425e4..3e910efc 100644 +--- a/src/utils/http/parser.c ++++ b/src/utils/http/parser.c +@@ -38,7 +38,7 @@ + */ + + #include "parser.h" +-#include ++#include + #include + #include + #include +diff --git a/src/utils/http/parser.h b/src/utils/http/parser.h +index ce5fe5e7..fd8f150d 100644 +--- a/src/utils/http/parser.h ++++ b/src/utils/http/parser.h +@@ -42,7 +42,7 @@ + + #include + +-#include "http_parser.h" ++#include + + #undef TRUE + #define TRUE 1 +diff --git a/src/utils/http/rest_common.c b/src/utils/http/rest_common.c +index 885375f2..d851ba96 100644 +--- a/src/utils/http/rest_common.c ++++ b/src/utils/http/rest_common.c +@@ -16,7 +16,7 @@ + + #include + #include +-#include ++#include + #include + + #include "isula_libutils/log.h" +-- +2.34.1 + diff --git a/0016-add-more-detailed-log-information-for-load-sandbox.patch b/0016-add-more-detailed-log-information-for-load-sandbox.patch new file mode 100644 index 0000000..db8bf17 --- /dev/null +++ b/0016-add-more-detailed-log-information-for-load-sandbox.patch @@ -0,0 +1,26 @@ +From e4facfcd2947b5277789d58a452090b61ca2d383 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Wed, 21 Feb 2024 15:04:27 +0800 +Subject: [PATCH 16/43] add more detailed log information for load sandbox + +Signed-off-by: zhongtao +--- + src/daemon/sandbox/sandbox_manager.cc | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/daemon/sandbox/sandbox_manager.cc b/src/daemon/sandbox/sandbox_manager.cc +index d3db4fb4..cee444f4 100644 +--- a/src/daemon/sandbox/sandbox_manager.cc ++++ b/src/daemon/sandbox/sandbox_manager.cc +@@ -461,7 +461,7 @@ auto SandboxManager::LoadSandbox(std::string &id) -> std::shared_ptr + Errors tmpError; + + if (!sandbox->Load(tmpError)) { +- ERROR("Failed to load subdir:%s", id.c_str()); ++ ERROR("Failed to load subdir:%s: %s", id.c_str(), tmpError.GetMessage().c_str()); + return nullptr; + } + +-- +2.34.1 + diff --git a/0017-bugfix-for-the-concurrency-competition-between-the-r.patch b/0017-bugfix-for-the-concurrency-competition-between-the-r.patch new file mode 100644 index 0000000..796370b --- /dev/null +++ b/0017-bugfix-for-the-concurrency-competition-between-the-r.patch @@ -0,0 +1,83 @@ +From 0099190e7f18e890185e36c5a657e9ce95179bc8 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Fri, 1 Mar 2024 15:04:09 +0800 +Subject: [PATCH 17/43] bugfix for the concurrency competition between the + reuse layer and the creation layer + +Signed-off-by: zhongtao +--- + src/daemon/modules/image/oci/oci_load.c | 9 ++++++++- + src/daemon/modules/image/oci/registry/registry.c | 9 ++++++++- + 2 files changed, 16 insertions(+), 2 deletions(-) + +diff --git a/src/daemon/modules/image/oci/oci_load.c b/src/daemon/modules/image/oci/oci_load.c +index 534e2647..85127f35 100644 +--- a/src/daemon/modules/image/oci/oci_load.c ++++ b/src/daemon/modules/image/oci/oci_load.c +@@ -680,6 +680,12 @@ static int oci_load_set_layers_info(load_image_t *im, const image_manifest_items + char *parent_chain_id_sha256 = ""; + char *id = NULL; + char *parent_chain_id = NULL; ++ // exist_flag is used to mark whether a non-existent layer has been encountered during this layer reuse process. ++ // 1.exist_flag is true if the layers are currently reusable; ++ // 2.exist_flag is false if encounter an uncreated layer that cannot be reused ++ // Prevent concurrent competition between the creation layer function ++ // and the reuse layer function on the im -> layer_of_hold_refs variable ++ bool exist_flag = true; + + if (im == NULL || manifest == NULL || dstdir == NULL) { + ERROR("Invalid input params image or manifest is null"); +@@ -761,7 +767,7 @@ static int oci_load_set_layers_info(load_image_t *im, const image_manifest_items + goto out; + } + +- if (storage_inc_hold_refs(id) == 0) { ++ if (exist_flag && storage_inc_hold_refs(id) == 0) { + free(im->layer_of_hold_refs); + im->layer_of_hold_refs = util_strdup_s(id); + if (parent_chain_id != NULL && storage_dec_hold_refs(parent_chain_id) != 0) { +@@ -781,6 +787,7 @@ static int oci_load_set_layers_info(load_image_t *im, const image_manifest_items + continue; + } + ++ exist_flag = false; + if (check_and_set_digest_from_tarball(im->layers[i], conf->rootfs->diff_ids[i]) != 0) { + ERROR("Check layer digest failed"); + ret = -1; +diff --git a/src/daemon/modules/image/oci/registry/registry.c b/src/daemon/modules/image/oci/registry/registry.c +index aed3057a..66fa0076 100644 +--- a/src/daemon/modules/image/oci/registry/registry.c ++++ b/src/daemon/modules/image/oci/registry/registry.c +@@ -1516,6 +1516,12 @@ static int fetch_all(pull_descriptor *desc) + struct layer_list *list = NULL; + pthread_t tid = 0; + struct timespec ts = { 0 }; ++ // exist_flag is used to mark whether a non-existent layer has been encountered during this layer reuse process. ++ // 1.exist_flag is true if the layers are currently reusable; ++ // 2.exist_flag is false if encounter an uncreated layer that cannot be reused ++ // Prevent concurrent competition between the creation layer function ++ // and the reuse layer function on the im -> layer_of_hold_refs variable ++ bool exist_flag = true; + + if (desc == NULL) { + ERROR("Invalid NULL param"); +@@ -1547,7 +1553,7 @@ static int fetch_all(pull_descriptor *desc) + + // Skip layer that already exist in local store + list = storage_layers_get_by_compress_digest(desc->layers[i].digest); +- if (list != NULL) { ++ if (exist_flag && list != NULL) { + for (j = 0; j < list->layers_len; j++) { + if ((list->layers[j]->parent == NULL && i == 0) || + (parent_chain_id != NULL && list->layers[j]->parent != NULL && +@@ -1579,6 +1585,7 @@ static int fetch_all(pull_descriptor *desc) + continue; + } + } ++ exist_flag = false; + + // parent_chain_id = NULL means no parent chain match from now on, so no longer need + // to get layers by compressed digest to reuse layer. +-- +2.34.1 + diff --git a/0018-add-concurrent-load-test.patch b/0018-add-concurrent-load-test.patch new file mode 100644 index 0000000..2256aba --- /dev/null +++ b/0018-add-concurrent-load-test.patch @@ -0,0 +1,73 @@ +From 2af906d42a155a7b779dce017a2779b96dba2b61 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Fri, 1 Mar 2024 15:04:35 +0800 +Subject: [PATCH 18/43] add concurrent load test + +Signed-off-by: zhongtao +--- + CI/test_cases/image_cases/image_load.sh | 47 +++++++++++++++++++++++++ + 1 file changed, 47 insertions(+) + +diff --git a/CI/test_cases/image_cases/image_load.sh b/CI/test_cases/image_cases/image_load.sh +index 52b713d4..a2cada5f 100755 +--- a/CI/test_cases/image_cases/image_load.sh ++++ b/CI/test_cases/image_cases/image_load.sh +@@ -79,8 +79,55 @@ function test_image_load() + return ${ret} + } + ++function test_concurrent_load() ++{ ++ local ret=0 ++ local test="isula load image test => (${FUNCNAME[@]})" ++ ++ msg_info "${test} starting..." ++ ++ # clean exist image ++ ubuntu_id=`isula inspect -f '{{.image.id}}' ubuntu` ++ busybox_id=`isula inspect -f '{{.image.id}}' busybox` ++ isula rmi $ubuntu_id $busybox_id ++ ++ concurrent_time=10 ++ for i in `seq 1 $concurrent_time` ++ do ++ isula load -i $mult_image & ++ pids[$i]=$! ++ done ++ ++ for i in `seq 1 $concurrent_time`;do ++ wait ${pids[$i]} ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - fail to do isulad load $i" && ((ret++)) ++ done ++ ++ ubuntu_id=`isula inspect -f '{{.image.id}}' ubuntu` ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - fail to inspect image: ubuntu" && ((ret++)) ++ ++ top_layer_id=$(isula inspect -f '{{.image.top_layer}}' ${ubuntu_id}) ++ ++ busybox_id=`isula inspect -f '{{.image.id}}' busybox` ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - fail to inspect image: busybox" && ((ret++)) ++ ++ # delete image after concurrent load ++ isula rmi $ubuntu_id $busybox_id ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to remove image ${ubuntu_id} and ${busybox_id}" && ((ret++)) ++ ++ ls -l /var/lib/isulad/storage/overlay-layers ++ local top_layer_dir=/var/lib/isulad/storage/overlay-layers/${top_layer_id} ++ test -e ${top_layer_dir} ++ [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - top layer dir ${top_layer_id} exist after delete image" && ((ret++)) ++ ++ msg_info "${test} finished with return ${ret}..." ++ return ${ret} ++} ++ + declare -i ans=0 + ++test_concurrent_load || ((ans++)) ++ + test_image_load || ((ans++)) + + show_result ${ans} "${curr_path}/${0}" +-- +2.34.1 + diff --git a/0019-get-the-realpath-of-the-host-path-for-archive-when-c.patch b/0019-get-the-realpath-of-the-host-path-for-archive-when-c.patch new file mode 100644 index 0000000..e1a2fda --- /dev/null +++ b/0019-get-the-realpath-of-the-host-path-for-archive-when-c.patch @@ -0,0 +1,63 @@ +From 96dfd32ee5d9a133ad63af13723402f10cd7cf7b Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Mon, 11 Mar 2024 15:50:45 +0800 +Subject: [PATCH 19/43] get the realpath of the host path for archive when cp + +Signed-off-by: zhongtao +--- + src/utils/tar/isulad_tar.c | 16 ++++++++++++++-- + 1 file changed, 14 insertions(+), 2 deletions(-) + +diff --git a/src/utils/tar/isulad_tar.c b/src/utils/tar/isulad_tar.c +index bbe4c3b2..fe514acc 100644 +--- a/src/utils/tar/isulad_tar.c ++++ b/src/utils/tar/isulad_tar.c +@@ -390,6 +390,7 @@ int archive_copy_to(const struct io_read_wrapper *content, const struct archive_ + { + int ret = -1; + struct archive_copy_info *dstinfo = NULL; ++ char cleanpath[PATH_MAX] = { 0 }; + char *dstdir = NULL; + char *src_base = NULL; + char *dst_base = NULL; +@@ -410,7 +411,12 @@ int archive_copy_to(const struct io_read_wrapper *content, const struct archive_ + goto cleanup; + } + +- ret = archive_chroot_untar_stream(content, dstdir, ".", src_base, dst_base, root_dir, err); ++ if (realpath(dstdir, cleanpath) == NULL) { ++ ERROR("Failed to get real path for %s", dstdir); ++ return -1; ++ } ++ ++ ret = archive_chroot_untar_stream(content, cleanpath, ".", src_base, dst_base, root_dir, err); + + cleanup: + free_archive_copy_info(dstinfo); +@@ -428,6 +434,7 @@ static int tar_resource_rebase(const char *path, const char *rebase, const char + struct stat st; + char *srcdir = NULL; + char *srcbase = NULL; ++ char cleanpath[PATH_MAX] = { 0 }; + + if (lstat(path, &st) < 0) { + SYSERROR("lstat %s failed", path); +@@ -438,9 +445,14 @@ static int tar_resource_rebase(const char *path, const char *rebase, const char + ERROR("Can not split path: %s", path); + goto cleanup; + } ++ ++ if (realpath(srcdir, cleanpath) == NULL) { ++ ERROR("Failed to get real path for %s", srcdir); ++ return -1; ++ } + + DEBUG("chroot tar stream srcdir(%s) srcbase(%s) rebase(%s)", srcdir, srcbase, rebase); +- nret = archive_chroot_tar_stream(srcdir, srcbase, srcbase, rebase, root_dir, archive_reader); ++ nret = archive_chroot_tar_stream(cleanpath, srcbase, srcbase, rebase, root_dir, archive_reader); + if (nret < 0) { + ERROR("Can not archive path: %s", path); + goto cleanup; +-- +2.34.1 + diff --git a/0020-bugfix-for-wrong-goto-branch.patch b/0020-bugfix-for-wrong-goto-branch.patch new file mode 100644 index 0000000..7c8318c --- /dev/null +++ b/0020-bugfix-for-wrong-goto-branch.patch @@ -0,0 +1,35 @@ +From c67760ce928f67d9a8beeaf2e2d51c8f2239f69e Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Tue, 12 Mar 2024 11:15:26 +0800 +Subject: [PATCH 20/43] bugfix for wrong goto branch + +Signed-off-by: zhongtao +--- + src/utils/tar/isulad_tar.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/src/utils/tar/isulad_tar.c b/src/utils/tar/isulad_tar.c +index fe514acc..13343922 100644 +--- a/src/utils/tar/isulad_tar.c ++++ b/src/utils/tar/isulad_tar.c +@@ -413,7 +413,7 @@ int archive_copy_to(const struct io_read_wrapper *content, const struct archive_ + + if (realpath(dstdir, cleanpath) == NULL) { + ERROR("Failed to get real path for %s", dstdir); +- return -1; ++ goto cleanup; + } + + ret = archive_chroot_untar_stream(content, cleanpath, ".", src_base, dst_base, root_dir, err); +@@ -448,7 +448,7 @@ static int tar_resource_rebase(const char *path, const char *rebase, const char + + if (realpath(srcdir, cleanpath) == NULL) { + ERROR("Failed to get real path for %s", srcdir); +- return -1; ++ goto cleanup; + } + + DEBUG("chroot tar stream srcdir(%s) srcbase(%s) rebase(%s)", srcdir, srcbase, rebase); +-- +2.34.1 + diff --git a/0021-bugfix-for-wrong-dynamic-allocation-object-type.patch b/0021-bugfix-for-wrong-dynamic-allocation-object-type.patch new file mode 100644 index 0000000..bb08db6 --- /dev/null +++ b/0021-bugfix-for-wrong-dynamic-allocation-object-type.patch @@ -0,0 +1,29 @@ +From 0ef23c6caae4a97228705574b0c8f3445c6e65dc Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?=E8=AE=B8=E7=BA=A2=E5=BC=BA?= <277922995@qq.com> +Date: Wed, 13 Mar 2024 17:00:16 +0800 +Subject: [PATCH 21/43] bugfix for wrong dynamic allocation object type +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Signed-off-by: 许红强 <277922995@qq.com> +--- + src/daemon/modules/image/image.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/daemon/modules/image/image.c b/src/daemon/modules/image/image.c +index e7aa81b8..871f5f39 100644 +--- a/src/daemon/modules/image/image.c ++++ b/src/daemon/modules/image/image.c +@@ -1535,7 +1535,7 @@ int im_tag_image(const im_tag_request *request, im_tag_response **response) + return -1; + } + +- *response = util_common_calloc_s(sizeof(im_remove_response)); ++ *response = util_common_calloc_s(sizeof(im_tag_response)); + if (*response == NULL) { + ERROR("Out of memory"); + return -1; +-- +2.34.1 + diff --git a/0022-add-swap-usage-in-cri.patch b/0022-add-swap-usage-in-cri.patch new file mode 100644 index 0000000..cb773e6 --- /dev/null +++ b/0022-add-swap-usage-in-cri.patch @@ -0,0 +1,80 @@ +From ed569ccbf7e5029e83c40521255e0e406f285bae Mon Sep 17 00:00:00 2001 +From: jikai +Date: Fri, 12 Jan 2024 11:31:59 +0800 +Subject: [PATCH 22/43] add swap usage in cri + +Signed-off-by: jikai +--- + .../entry/cri/v1/v1_cri_container_manager_service.cc | 11 +++++++++++ + src/daemon/executor/container_cb/execution_extend.c | 2 ++ + src/daemon/modules/api/runtime_api.h | 3 +++ + src/daemon/modules/runtime/isula/isula_rt_ops.c | 4 ++++ + 4 files changed, 20 insertions(+) + +diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +index 2dda1e16..47a33c2c 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +@@ -890,6 +890,17 @@ void ContainerManagerService::ContainerStatsToGRPC( + if (response->container_stats[i]->major_page_faults != 0u) { + container->mutable_memory()->mutable_major_page_faults()->set_value(response->container_stats[i]->major_page_faults); + } ++ ++ // Swap ++ container->mutable_swap()->set_timestamp(timestamp); ++ if (response->container_stats[i]->swap_used != 0u) { ++ container->mutable_swap()->mutable_swap_usage_bytes()->set_value(response->container_stats[i]->swap_used); ++ } ++ if (response->container_stats[i]->swap_limit >= response->container_stats[i]->swap_used) { ++ container->mutable_swap()->mutable_swap_available_bytes()->set_value(response->container_stats[i]->swap_limit ++ - response->container_stats[i]->swap_used); ++ } ++ + containerstats.push_back(std::move(container)); + } + } +diff --git a/src/daemon/executor/container_cb/execution_extend.c b/src/daemon/executor/container_cb/execution_extend.c +index 25ec5d3b..52401633 100644 +--- a/src/daemon/executor/container_cb/execution_extend.c ++++ b/src/daemon/executor/container_cb/execution_extend.c +@@ -259,6 +259,8 @@ static container_info *get_container_stats(const container_t *cont, + info->major_page_faults = einfo->major_page_faults; + info->kmem_used = einfo->kmem_used; + info->kmem_limit = einfo->kmem_limit; ++ info->swap_used = einfo->swap_used; ++ info->swap_limit = einfo->swap_limit; + info->timestamp = util_get_now_time_nanos(); + + // workingset is zero if memory used < total inactive file +diff --git a/src/daemon/modules/api/runtime_api.h b/src/daemon/modules/api/runtime_api.h +index d2679c2d..bd170c30 100644 +--- a/src/daemon/modules/api/runtime_api.h ++++ b/src/daemon/modules/api/runtime_api.h +@@ -66,6 +66,9 @@ struct runtime_container_resources_stats_info { + uint64_t cache; + uint64_t cache_total; + uint64_t inactive_file_total; ++ /* Swap usage*/ ++ uint64_t swap_used; ++ uint64_t swap_limit; + }; + + typedef struct _rt_create_params_t { +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 2c92cc59..745154bb 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -647,6 +647,10 @@ static void transform_stats_info_from_runtime(shim_client_runtime_stats *stats, + info->page_faults = memory->raw->pgfault; + info->major_page_faults = memory->raw->pgmajfault; + } ++ if (memory != NULL && memory->swap != NULL) { ++ info->swap_used = memory->swap->usage; ++ info->swap_limit = memory->swap->limit; ++ } + shim_client_runtime_stats_data_blkio *blkio = stats->data->blkio; + if (blkio == NULL) { + return; +-- +2.34.1 + diff --git a/0023-add-benchmark-result-of-perf-test-in-cri.patch b/0023-add-benchmark-result-of-perf-test-in-cri.patch new file mode 100644 index 0000000..92efaeb --- /dev/null +++ b/0023-add-benchmark-result-of-perf-test-in-cri.patch @@ -0,0 +1,256 @@ +From 16a0cf7e9c2c059cb5537f48a022e63df457f186 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Sat, 2 Mar 2024 11:49:08 +0800 +Subject: [PATCH 23/43] add benchmark, result of perf test in cri + +Signed-off-by: jikai +--- + docs/performance/cri_performance_test.md | 60 ++++++++ + tools/benchmark/cri_perf_test.sh | 168 +++++++++++++++++++++++ + 2 files changed, 228 insertions(+) + create mode 100644 docs/performance/cri_performance_test.md + create mode 100755 tools/benchmark/cri_perf_test.sh + +diff --git a/docs/performance/cri_performance_test.md b/docs/performance/cri_performance_test.md +new file mode 100644 +index 00000000..594f0da7 +--- /dev/null ++++ b/docs/performance/cri_performance_test.md +@@ -0,0 +1,60 @@ ++## Machine configuration ++ ++ARM machine: ++ ++| Configuration | Information | ++| ------------- | -------------------------------------- | ++| OS | openEuler 22.03-LTS | ++| Kernel | linux 5.10.0-136.12.0.86.oe2203.aarch64 | ++| CPU | 96 cores | ++| Memory | 128 GB | ++ ++## Version of Softwares ++ ++| Name | Version | ++| ------ | ------------------------------------------------------------ | ++| iSulad | Version: 2.1.5 , Git commit: 5ebca976dd591a5676527be1bde950e5ce93eac0 | ++| containerd | Version: v2.0.0-beta.2, Git commit: 290194fe77d48521d3ea78ec02e2e406c4bf91b6 | ++| crio | version: 1.30.0, Git commit: b43e0d63a8af3277dbfc555f62d07bb2305a72c7 | ++ ++## Test tool ++ ++tools/benchmark/cri_perf_test.sh ++ ++## Compare with other container engines ++ ++### run operator once ++ ++#### ARM ++ ++run 1 pod and 1 container ++ ++| measure | iSulad | containerd | crio | vs containerd | vs crio | ++| ----------------- | ------ | ------ | ------ | ------ | ------ | ++| time(ms) | 580 | 812 | 567 | -28.5% | 2.3% | ++| engine mem(kb) | 38704 | 66806 | 58760 | -42.0% | -34.2% | ++| shim mem(kb) | 1700 | 13876 | 4648 | -87.7% | -63.4% | ++ ++run 10 pods and 10 containers ++ ++| measure | iSulad | containerd | crio | vs containerd | vs crio | ++| ----------------- | ------ | ------ | ------ | ------ | ------ | ++| time(ms) | 1141 | 4000 | 1749 | -71.5% | -34.8% | ++| engine mem(kb) | 47688 | 82580 | 86128 | -42.2% | -44.6% | ++| shim mem(kb) | 16764 | 154872 | 46836 | -89.2% | -64.2% | ++ ++run 50 pods and 50 containers ++ ++| measure | iSulad | containerd | crio | vs containerd | vs crio | ++| ----------------- | ------ | ------ | ------ | ------ | ------ | ++| time(ms) | 4544 | 19963 | 8503 | -77.2% | -46.9% | ++| engine mem(kb) | 88700 | 134384 | 115560 | -34.0% | -23.2% | ++| shim mem(kb) | 83892 | 750924 | 233480 | -88.8% | -64.0% | ++ ++run 100 pods and 100 containers ++ ++| measure | iSulad | containerd | crio | vs containerd | vs crio | ++| ----------------- | ------ | ------ | ------ | ------ | ------ | ++| time(ms) | 10012 | 39629 | 18278 | -74.7% | -45.5% | ++| engine mem(kb) | 148464 | 185700 | 147836 | -20.0% | 0.4% | ++| shim mem(kb) | 168420 | 1506268| 462000 | -88.8% | -63.3% | +diff --git a/tools/benchmark/cri_perf_test.sh b/tools/benchmark/cri_perf_test.sh +new file mode 100755 +index 00000000..54ee24f5 +--- /dev/null ++++ b/tools/benchmark/cri_perf_test.sh +@@ -0,0 +1,168 @@ ++#!/bin/bash ++####################################################################### ++##- Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. ++# - iSulad licensed under the Mulan PSL v2. ++# - You can use this software according to the terms and conditions of the Mulan PSL v2. ++# - You may obtain a copy of Mulan PSL v2 at: ++# - http://license.coscl.org.cn/MulanPSL2 ++# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++# - PURPOSE. ++# - See the Mulan PSL v2 for more details. ++##- @Description: perf test ++##- @Author: jikai ++##- @Create: 2024-02-29 ++####################################################################### ++ ++# cri_perf_test.sh -e $engine -p $parallel ++ ++engine=isulad ++runtime="unix:///var/run/isulad.sock" ++shim="isulad-shim" ++parallel=1 ++while getopts ":e:p:" opt ++do ++ case $opt in ++ e) ++ engine=${OPTARG} ++ # compare if OPTARG is in ["isulad", "containerd", "crio"]} ++ if [ $engine == "isulad" ]; then ++ runtime="unix:///var/run/isulad.sock" ++ shim="isulad-shim" ++ elif [ $engine == "containerd" ]; then ++ runtime="unix:///var/run/containerd/containerd.sock" ++ shim="containerd-shim" ++ elif [ $engine == "crio" ]; then ++ runtime="unix:///var/run/crio/crio.sock" ++ shim="conmon" ++ else ++ echo "Unknown engine: ${OPTARG}, only support isulad, containerd, crio." ++ exit 1 ++ fi ++ ;; ++ p) ++ parallel=${OPTARG} ++ ;; ++ ?) ++ echo "Unknown parameter" ++ exit 1;; ++ esac ++done ++ ++workdir="$(pwd)" ++tmpdir="$workdir/cri_perf_test_tmpdata" ++mkdir -p $tmpdir/container/ ++mkdir -p $tmpdir/pod/ ++mkdir -p $workdir/cri_perf_test_result/ ++result_data=$workdir/cri_perf_test_result/${engine}-${parallel}-result.dat ++rm -f $result_data ++ ++# Get the interval time(ms) ++function getTiming(){ ++ start=$1 ++ end=$2 ++ ++ start_s=$(echo $start | cut -d '.' -f 1) ++ start_ns=$(echo $start | cut -d '.' -f 2) ++ end_s=$(echo $end | cut -d '.' -f 1) ++ end_ns=$(echo $end | cut -d '.' -f 2) ++ ++ time=$(( ( 10#$end_s - 10#$start_s ) * 1000 + ( 10#$end_ns / 1000000 - 10#$start_ns / 1000000 ) )) ++ ++ echo "$time" ++} ++ ++# Kill all pods and containers running ++crictl --runtime-endpoint $runtime rmp -af ++ ++# Create $parallel container.json and pod.json ++for((i=0;i<$parallel;i++)) ++do ++ cat > $tmpdir/container/container_$i.json << EOF ++{ ++ "metadata": { ++ "name": "testcontainer$i" ++ }, ++ "image": { ++ "image": "busybox" ++ }, ++ "command": [ ++ "/bin/sh", "-c", "sleep 1d" ++ ], ++ "log_path": "console$i.log", ++ "linux": { ++ "security_context": { ++ "capabilities": {} ++ } ++ } ++} ++EOF ++ ++ cat > $tmpdir/pod/pod_$i.json <> ${result_data} ++else ++ echo "${boot_time}ms is too long, please check the environment." ++fi ++ ++# get pids ++shim_pids=$(ps -ef | grep -v grep | grep -i $shim | awk '{print$2}') ++ ++# calc memory of pids ++engine_mem=$(cat /proc/$engine_pid/status | grep VmRSS | awk '{print $2}') ++shim_mem=0 ++for pid in $shim_pids ++do ++ let shim_mem+=$(cat /proc/$pid/status | grep VmRSS | awk '{print $2}') ++done ++echo "Engine Mem: ${engine_mem}KB" ++echo "engine-mem: ${engine_mem}" >> ${result_data} ++echo "Shim Mem Total: ${shim_mem}KB" ++echo "shim-mem: ${shim_mem}" >> ${result_data} ++ ++# clean resources ++crictl --runtime-endpoint $runtime rmp -af ++rm -rf $tmpdir +-- +2.34.1 + diff --git a/0024-add-support-for-systemd-cgroup-driver.patch b/0024-add-support-for-systemd-cgroup-driver.patch new file mode 100644 index 0000000..d6c363b --- /dev/null +++ b/0024-add-support-for-systemd-cgroup-driver.patch @@ -0,0 +1,523 @@ +From 167af3ce0cff3906c9976b249432d41167b15eb2 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Thu, 11 Jan 2024 17:06:57 +0800 +Subject: [PATCH 24/43] add support for systemd cgroup driver + +Signed-off-by: jikai +--- + src/cmd/isulad-shim/process.c | 6 + + src/cmd/isulad/isulad_commands.h | 2 + + src/contrib/config/daemon.json | 3 +- + src/daemon/common/cgroup.h | 4 + + src/daemon/common/cgroup_v1.c | 116 ++++++++++++++++++ + src/daemon/config/isulad_config.c | 23 ++++ + src/daemon/config/isulad_config.h | 2 + + .../cri/v1/cri_v1_runtime_runtime_service.cc | 25 ++++ + .../cri/v1/cri_v1_runtime_runtime_service.h | 4 + + .../cri/v1/v1_cri_runtime_manager_service.cc | 12 ++ + .../cri/v1/v1_cri_runtime_manager_service.h | 2 + + .../entry/cri/v1/v1_cri_runtime_service.h | 2 + + .../cri/v1/v1_cri_runtime_service_impl.cc | 5 + + .../cri/v1/v1_cri_runtime_service_impl.h | 2 + + src/daemon/executor/container_cb/execution.c | 26 ++++ + .../modules/runtime/engines/lcr/lcr_rt_ops.c | 7 ++ + .../modules/runtime/isula/isula_rt_ops.c | 1 + + src/daemon/modules/runtime/shim/shim_rt_ops.c | 6 + + src/daemon/modules/spec/specs.c | 22 +++- + 19 files changed, 268 insertions(+), 2 deletions(-) + +diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c +index bf67c414..8a4ca175 100644 +--- a/src/cmd/isulad-shim/process.c ++++ b/src/cmd/isulad-shim/process.c +@@ -1287,6 +1287,12 @@ static void get_runtime_cmd(process_t *p, const char *log_path, const char *pid_ + params[i++] = p->state->cwd; + } + } else { ++ // the --systemd-cgroup argument is not in oci spec, but up to now, ++ // the latest version of runc, crun, youki, runsc, kata-runtime all support this argument ++ // should ensure that this is supported for oci runtime ++ if (p->state->systemd_cgroup) { ++ params[i++] = "--systemd-cgroup"; ++ } + params[i++] = "create"; + params[i++] = "--bundle"; + params[i++] = p->bundle; +diff --git a/src/cmd/isulad/isulad_commands.h b/src/cmd/isulad/isulad_commands.h +index ba4b838d..cf5f65f5 100644 +--- a/src/cmd/isulad/isulad_commands.h ++++ b/src/cmd/isulad/isulad_commands.h +@@ -226,6 +226,8 @@ int command_default_ulimit_append(command_option_t *option, const char *arg); + &(cmdargs)->json_confs->storage_opts, \ + "Storage driver options", \ + command_append_array }, \ ++ { CMD_OPT_TYPE_BOOL, false, "systemd-cgroup", 0, &(cmdargs)->json_confs->systemd_cgroup, \ ++ "Use systemd cgroup driver(default false)", NULL }, \ + SUP_GROUPS_OPT(cmdargs) \ + { CMD_OPT_TYPE_CALLBACK, \ + false, \ +diff --git a/src/contrib/config/daemon.json b/src/contrib/config/daemon.json +index 966e016a..69362c26 100644 +--- a/src/contrib/config/daemon.json ++++ b/src/contrib/config/daemon.json +@@ -36,5 +36,6 @@ + "cri-runtimes": { + "kata": "io.containerd.kata.v2" + }, +- "enable-cri-v1": false ++ "enable-cri-v1": false, ++ "systemd-cgroup": false + } +diff --git a/src/daemon/common/cgroup.h b/src/daemon/common/cgroup.h +index 6664fb15..fa20f42c 100644 +--- a/src/daemon/common/cgroup.h ++++ b/src/daemon/common/cgroup.h +@@ -141,6 +141,10 @@ typedef struct { + + int common_get_cgroup_v1_metrics(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics); + ++char *common_get_init_cgroup(const char *subsystem); ++ ++char *common_get_own_cgroup(const char *subsystem); ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/common/cgroup_v1.c b/src/daemon/common/cgroup_v1.c +index 9004becc..e38fc03e 100644 +--- a/src/daemon/common/cgroup_v1.c ++++ b/src/daemon/common/cgroup_v1.c +@@ -554,3 +554,119 @@ int common_get_cgroup_v1_metrics(const char *cgroup_path, cgroup_metrics_t *cgro + return 0; + } + ++/* parse cgroup files, such as /proc/self/cgroup or /proc/1/cgroup */ ++static int parse_cgroup_file(const char *path, char ***nlist, char ***plist) ++{ ++ int ret = 0; ++ size_t length = 0; ++ __isula_auto_file FILE *fp = NULL; ++ __isula_auto_free char *pline = NULL; ++ ++ fp = util_fopen(path, "r"); ++ if (fp == NULL) { ++ return -1; ++ } ++ ++ while (getline(&pline, &length, fp) != -1) { ++ char *pos = NULL; ++ char *pos2 = NULL; ++ char *pos3 = NULL; ++ char *ptoken = NULL; ++ char *psave = NULL; ++ pos = strchr(pline, ':'); ++ if (pos == NULL) { ++ ERROR("Invalid cgroup entry: must contain at least two colons: %s", pline); ++ ret = -1; ++ goto out; ++ } ++ pos++; ++ pos2 = strchr(pos, ':'); ++ if (pos2 == NULL) { ++ ERROR("Invalid cgroup entry: must contain at least two colons: %s", pline); ++ ret = -1; ++ goto out; ++ } ++ pos3 = strchr(pos2, '\n'); ++ if (pos3 != NULL) { ++ *pos3 = '\0'; ++ } ++ *pos2 = '\0'; ++ ++ if ((pos2 - pos) == 0) { ++ INFO("Cgroup entry: %s not supported by cgroup v1", pline); ++ continue; ++ } ++ ++ for (ptoken = strtok_r(pos, ",", &psave); ptoken; ptoken = strtok_r(NULL, ",", &psave)) { ++ ret = util_array_append(nlist, ptoken); ++ if (ret != 0) { ++ ERROR("Failed to append string"); ++ goto out; ++ } ++ ++ ret = util_array_append(plist, pos2 + 1); ++ if (ret != 0) { ++ ERROR("Failed to append string"); ++ goto out; ++ } ++ } ++ } ++ ++out: ++ if (ret != 0) { ++ util_free_array(*nlist); ++ *nlist = NULL; ++ util_free_array(*plist); ++ *plist = NULL; ++ } ++ return ret; ++} ++ ++static char *common_get_cgroup_path(const char *path, const char *subsystem) ++{ ++ char **nlist = NULL, **plist = NULL; ++ size_t i = 0; ++ char *res = NULL; ++ if (path == NULL) { ++ ERROR("Invalid NULL param"); ++ return NULL; ++ } ++ ++ if (parse_cgroup_file(path, &nlist, &plist) < 0) { ++ return NULL; ++ } ++ ++ for (i = 0; i < util_array_len((const char **)nlist); i++) { ++ const char *prefix = "name="; ++ bool find_sub = (strcmp(nlist[i], subsystem) == 0 || (strncmp(nlist[i], prefix, strlen(prefix)) == 0 ++ && strcmp(nlist[i]+strlen(prefix), subsystem) == 0)); ++ if (find_sub) { ++ res = util_strdup_s(plist[i]); ++ break; ++ } ++ } ++ ++ util_free_array(nlist); ++ util_free_array(plist); ++ return res; ++} ++ ++char *common_get_init_cgroup(const char *subsystem) ++{ ++ if (common_get_cgroup_version() != CGROUP_VERSION_1) { ++ ERROR("Not implemented for cgroup v2 hierarchy"); ++ return NULL; ++ } ++ ++ return common_get_cgroup_path("/proc/1/cgroup", subsystem); ++} ++ ++char *common_get_own_cgroup(const char *subsystem) ++{ ++ if (common_get_cgroup_version() != CGROUP_VERSION_1) { ++ ERROR("Not implemented for cgroup v2 hierarchy"); ++ return NULL; ++ } ++ ++ return common_get_cgroup_path("/proc/self/cgroup", subsystem); ++} +diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c +index 51758adb..8179558e 100644 +--- a/src/daemon/config/isulad_config.c ++++ b/src/daemon/config/isulad_config.c +@@ -1762,6 +1762,8 @@ int merge_json_confs_into_global(struct service_arguments *args) + args->json_confs->enable_cri_v1 = tmp_json_confs->enable_cri_v1; + #endif + ++ args->json_confs->systemd_cgroup = tmp_json_confs->systemd_cgroup; ++ + if (merge_cri_runtimes_into_global(args, tmp_json_confs)) { + ret = -1; + goto out; +@@ -1895,3 +1897,24 @@ isulad_daemon_constants *get_isulad_daemon_constants(void) + { + return g_isulad_daemon_constants; + } ++ ++bool conf_get_systemd_cgroup() ++{ ++ bool systemd_cgroup = false; ++ struct service_arguments *conf = NULL; ++ ++ if (isulad_server_conf_rdlock() != 0) { ++ return false; ++ } ++ ++ conf = conf_get_server_conf(); ++ if (conf == NULL || conf->json_confs == NULL) { ++ goto out; ++ } ++ ++ systemd_cgroup = conf->json_confs->systemd_cgroup; ++ ++out: ++ (void)isulad_server_conf_unlock(); ++ return systemd_cgroup; ++} +diff --git a/src/daemon/config/isulad_config.h b/src/daemon/config/isulad_config.h +index 459ea331..f29cd564 100644 +--- a/src/daemon/config/isulad_config.h ++++ b/src/daemon/config/isulad_config.h +@@ -114,6 +114,8 @@ char *conf_get_isulad_monitor_fifo_path(void); + int init_isulad_daemon_constants(void); + isulad_daemon_constants *get_isulad_daemon_constants(void); + ++bool conf_get_systemd_cgroup(void); ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc +index ba9459f6..76e393f3 100644 +--- a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc ++++ b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc +@@ -632,3 +632,28 @@ grpc::Status RuntimeV1RuntimeServiceImpl::Status(grpc::ServerContext *context, + + return grpc::Status::OK; + } ++ ++grpc::Status ++RuntimeV1RuntimeServiceImpl::RuntimeConfig(grpc::ServerContext *context, ++ const runtime::v1::RuntimeConfigRequest *request, ++ runtime::v1::RuntimeConfigResponse *reply) ++{ ++ Errors error; ++ ++ if (request == nullptr) { ++ ERROR("Invalid input arguments"); ++ return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, "Invalid input arguments"); ++ } ++ ++ EVENT("Event: {Object: CRI, Type: Runtime Config}"); ++ ++ m_rService->RuntimeConfig(reply, error); ++ if (!error.Empty()) { ++ ERROR("Object: CRI, Type: Failed to get runtime config:%s", error.GetMessage().c_str()); ++ return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ } ++ ++ EVENT("Event: {Object: CRI, Type: Runtime Config}"); ++ ++ return grpc::Status::OK; ++} +diff --git a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h +index 09ebe68b..52cc6b99 100644 +--- a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h ++++ b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h +@@ -101,6 +101,10 @@ public: + grpc::Status Status(grpc::ServerContext *context, const runtime::v1::StatusRequest *request, + runtime::v1::StatusResponse *reply) override; + ++ grpc::Status RuntimeConfig(grpc::ServerContext *context, ++ const runtime::v1::RuntimeConfigRequest *request, ++ runtime::v1::RuntimeConfigResponse *reply) override; ++ + private: + std::unique_ptr m_rService; + }; +diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.cc +index 1afac3e2..de489aae 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.cc +@@ -16,6 +16,7 @@ + #include "isula_libutils/log.h" + #include "v1_cri_helpers.h" + #include "cri_helpers.h" ++#include "isulad_config.h" + + namespace CRIV1 { + void RuntimeManagerService::UpdateRuntimeConfig(const runtime::v1::RuntimeConfig &config, Errors & /*error*/) +@@ -69,4 +70,15 @@ auto RuntimeManagerService::Status(Errors &error) -> std::unique_ptrmutable_linux()->set_cgroup_driver(conf_get_systemd_cgroup() ? runtime::v1::SYSTEMD : runtime::v1::CGROUPFS); ++} ++ + } // namespace CRI +diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.h b/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.h +index 3800ec30..2160064e 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.h ++++ b/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.h +@@ -39,6 +39,8 @@ public: + + auto Status(Errors &error) -> std::unique_ptr; + ++ void RuntimeConfig(runtime::v1::RuntimeConfigResponse *reply, Errors &error); ++ + private: + service_executor_t *m_cb; + std::shared_ptr m_pluginManager; +diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service.h b/src/daemon/entry/cri/v1/v1_cri_runtime_service.h +index 87394173..839f6724 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_runtime_service.h ++++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service.h +@@ -89,6 +89,8 @@ public: + virtual void UpdateRuntimeConfig(const runtime::v1::RuntimeConfig &config, Errors &error) = 0; + + virtual auto Status(Errors &error) -> std::unique_ptr = 0; ++ ++ virtual void RuntimeConfig(runtime::v1::RuntimeConfigResponse *reply, Errors &error) = 0; + }; + } // namespace CRIV1 + #endif // DAEMON_ENTRY_CRI_V1_CRI_RUNTIME_SERVICE_INTERFACE_H +diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc +index 67cda5ed..aa5ae516 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc +@@ -161,4 +161,9 @@ auto CRIRuntimeServiceImpl::Status(Errors &error) -> std::unique_ptrStatus(error); + } + ++void CRIRuntimeServiceImpl::RuntimeConfig(runtime::v1::RuntimeConfigResponse *reply, Errors &error) ++{ ++ m_runtimeManager->RuntimeConfig(reply, error); ++} ++ + } // namespace CRIV1 +diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h +index 23866648..0a25749f 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h ++++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h +@@ -92,6 +92,8 @@ public: + + auto Status(Errors &error) -> std::unique_ptr override; + ++ void RuntimeConfig(runtime::v1::RuntimeConfigResponse *reply, Errors &error) override; ++ + protected: + std::unique_ptr m_runtimeVersioner; + std::unique_ptr m_containerManager; +diff --git a/src/daemon/executor/container_cb/execution.c b/src/daemon/executor/container_cb/execution.c +index 63d8143c..7ed8e837 100644 +--- a/src/daemon/executor/container_cb/execution.c ++++ b/src/daemon/executor/container_cb/execution.c +@@ -427,6 +427,32 @@ static int cpurt_controller_init(const char *id, const host_config *host_spec) + return 0; + } + ++ if (conf_get_systemd_cgroup()) { ++ // currently it is the same as docker, yet it is unclear that ++ // if systemd cgroup is used and cgroup parent is set to a slice rather than system.slice ++ // should iSulad set cpu.rt_runtime_us and cpu.rt_period_us for the parent path? ++ // in fact, even if system.slice is used, ++ // cpu.rt_runtime_us and cpu.rt_period_us might still needed to be set manually ++ __isula_auto_free char *init_cgroup = common_get_init_cgroup("cpu"); ++ if (init_cgroup == NULL) { ++ ERROR("Failed to get init cgroup"); ++ return -1; ++ } ++ // make sure that the own cgroup path for cpu existed ++ __isula_auto_free char *own_cgroup = common_get_own_cgroup("cpu"); ++ if (own_cgroup == NULL) { ++ ERROR("Failed to get own cgroup"); ++ return -1; ++ } ++ char *new_cgroups_path = util_path_join(init_cgroup, cgroups_path); ++ if (new_cgroups_path == NULL) { ++ ERROR("Failed to join path"); ++ return -1; ++ } ++ free(cgroups_path); ++ cgroups_path = new_cgroups_path; ++ } ++ + mnt_root = sysinfo_cgroup_controller_cpurt_mnt_path(); + if (mnt_root == NULL) { + ERROR("Failed to get cpu rt controller mnt root path"); +diff --git a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c +index 8c3c5f1a..6b862958 100644 +--- a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c ++++ b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c +@@ -53,6 +53,13 @@ int rt_lcr_create(const char *name, const char *runtime, const rt_create_params_ + char *runtime_root = NULL; + struct engine_operation *engine_ops = NULL; + ++ if (conf_get_systemd_cgroup()) { ++ ERROR("Systemd cgroup not supported for lcr runtime"); ++ isulad_set_error_message("Systemd cgroup not supported for lcr runtime"); ++ ret = -1; ++ goto out; ++ } ++ + runtime_root = conf_get_routine_rootdir(runtime); + if (runtime_root == NULL) { + ERROR("Root path is NULL"); +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 745154bb..b9aba3e3 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -1157,6 +1157,7 @@ int rt_isula_create(const char *id, const char *runtime, const rt_create_params_ + p.runtime_args = (char **)runtime_args; + p.runtime_args_len = runtime_args_len; + p.attach_socket = attach_socket; ++ p.systemd_cgroup = conf_get_systemd_cgroup(); + copy_process(&p, config->process); + copy_annotations(&p, config->annotations); + +diff --git a/src/daemon/modules/runtime/shim/shim_rt_ops.c b/src/daemon/modules/runtime/shim/shim_rt_ops.c +index 81daf224..fc4d8e3a 100644 +--- a/src/daemon/modules/runtime/shim/shim_rt_ops.c ++++ b/src/daemon/modules/runtime/shim/shim_rt_ops.c +@@ -35,6 +35,7 @@ + #include "engine.h" + #include "shim_rt_monitor.h" + #include "supervisor.h" ++#include "isulad_config.h" + + #define EXIT_SIGNAL_OFFSET_X 128 + +@@ -370,6 +371,11 @@ int rt_shim_create(const char *id, const char *runtime, const rt_create_params_t + return -1; + } + ++ if (conf_get_systemd_cgroup()) { ++ ERROR("ShimV2 does not support systemd cgroup yet"); ++ return -1; ++ } ++ + exit_fifo_path = util_path_dir(params->exit_fifo); + if (exit_fifo_path == NULL) { + ERROR("%s: failed to get exit fifo dir from %s", id, params->exit_fifo); +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index 464b4fb4..b4d2b0f6 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -188,7 +188,10 @@ static char *do_get_container_cgroup_path(const host_config *host_spec) + } + + if (path == NULL) { +- // third, all faild, just use default '/isulad' ++ // third, all faild, just use default '/isulad' for cgroupfs or "system.slice" for systemd ++ if (conf_get_systemd_cgroup()) { ++ return util_strdup_s("system.slice"); ++ } + path = util_strdup_s("/isulad"); + } + +@@ -2288,6 +2291,23 @@ char *merge_container_cgroups_path(const char *id, const host_config *host_spec) + + path = do_get_container_cgroup_path(host_spec); + ++ if (conf_get_systemd_cgroup()) { ++ // systemd cgroup path has the form of [slice]:[prefix]:[name] ++#define SYSTEMD_CGROUP_PATH_LEN 3 ++ if (!util_has_suffix(path, ".slice")) { ++ ERROR("Invalid cgroup path %s for systemd", path); ++ isulad_set_error_message("Invalid cgroup path %s for systemd", path); ++ return NULL; ++ } ++ ++ // slice must not contain slashes ++ // convert test.slice/test-a.slice/test-a-b.slice to become test-a-b.slice ++ __isula_auto_free char *base = util_path_base(path); ++ const char *isulad_prefix = "isulad"; ++ const char *parts[SYSTEMD_CGROUP_PATH_LEN] = {base, isulad_prefix, id}; ++ return util_string_join(":", parts, SYSTEMD_CGROUP_PATH_LEN); ++ } ++ + return util_path_join(path, id); + } + +-- +2.34.1 + diff --git a/0025-add-ci-cases-for-systemd-cgroup-driver.patch b/0025-add-ci-cases-for-systemd-cgroup-driver.patch new file mode 100644 index 0000000..2d2ac71 --- /dev/null +++ b/0025-add-ci-cases-for-systemd-cgroup-driver.patch @@ -0,0 +1,219 @@ +From f5f100f5b244be2debebe815aaed3afad8950daf Mon Sep 17 00:00:00 2001 +From: jikai +Date: Tue, 6 Feb 2024 17:33:17 +0800 +Subject: [PATCH 25/43] add ci cases for systemd cgroup driver + +Signed-off-by: jikai +--- + .../container_cases/systemd_cgroup.sh | 80 +++++++++++++++++++ + test/mocks/isulad_config_mock.cc | 8 ++ + test/mocks/isulad_config_mock.h | 1 + + test/specs/specs/specs_ut.cc | 49 ++++++++++++ + 4 files changed, 138 insertions(+) + create mode 100755 CI/test_cases/container_cases/systemd_cgroup.sh + +diff --git a/CI/test_cases/container_cases/systemd_cgroup.sh b/CI/test_cases/container_cases/systemd_cgroup.sh +new file mode 100755 +index 00000000..ac1288e1 +--- /dev/null ++++ b/CI/test_cases/container_cases/systemd_cgroup.sh +@@ -0,0 +1,80 @@ ++#!/bin/bash ++# ++# attributes: isulad systemd cgroup run ++# concurrent: NO ++# spend time: 18 ++ ++####################################################################### ++##- Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. ++# - iSulad licensed under the Mulan PSL v2. ++# - You can use this software according to the terms and conditions of the Mulan PSL v2. ++# - You may obtain a copy of Mulan PSL v2 at: ++# - http://license.coscl.org.cn/MulanPSL2 ++# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++# - PURPOSE. ++# - See the Mulan PSL v2 for more details. ++##- @Description:CI ++##- @Author: jikai ++##- @Create: 2024-02-05 ++####################################################################### ++ ++curr_path=$(dirname $(readlink -f "$0")) ++data_path=$(realpath $curr_path/../data) ++source ../helpers.sh ++ ++function test_systemd_cgroup() ++{ ++ local ret=0 ++ local runtime=$1 ++ local image="busybox" ++ ++ local test="systemd cgroup driver test with (${runtime})=> (${FUNCNAME[@]})" ++ msg_info "${test} starting..." ++ ++ check_valgrind_log ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop isulad failed" && ((ret++)) ++ ++ start_isulad_with_valgrind --systemd-cgroup ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start isulad failed" && ((ret++)) ++ ++ cid1=$(isula run -tid --runtime $runtime -m 10M $image /bin/sh) ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start container failed" && ((ret++)) ++ cat /sys/fs/cgroup/memory/system.slice/isulad-$cid1.scope/memory.limit_in_bytes | grep ^10485760$ ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check memory limit failed" && ((ret++)) ++ ++ cid2=$(isula run -tid --runtime $runtime --cgroup-parent /test $image /bin/sh) ++ [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start container failed" && ((ret++)) ++ ++ cid3=$(isula run -tid --runtime $runtime -m 10M --cgroup-parent test-a-b.slice $image /bin/sh) ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start container failed" && ((ret++)) ++ cat /sys/fs/cgroup/memory/test.slice/test-a.slice/test-a-b.slice/isulad-$cid3.scope/memory.limit_in_bytes | grep ^10485760$ ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check memory limit failed" && ((ret++)) ++ ++ isula rm -f $cid1 $cid2 $cid3 ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - rm container failed" && ((ret++)) ++ ++ check_valgrind_log ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop isulad failed" && ((ret++)) ++ ++ start_isulad_with_valgrind ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start isulad failed" && ((ret++)) ++ ++ rm -rf $ulimitlog ++ ++ msg_info "${test} finished with return ${ret}..." ++ return ${ret} ++} ++ ++declare -i ans=0 ++ ++for element in ${RUNTIME_LIST[@]}; ++do ++ # lcr does not support systemd cgroup driver ++ if [ "$element" == "lcr" ];then ++ continue ++ fi ++ test_systemd_cgroup $element || ((ans++)) ++done ++ ++show_result ${ans} "${curr_path}/${0}" +diff --git a/test/mocks/isulad_config_mock.cc b/test/mocks/isulad_config_mock.cc +index 7ba4fa57..65b00563 100644 +--- a/test/mocks/isulad_config_mock.cc ++++ b/test/mocks/isulad_config_mock.cc +@@ -210,3 +210,11 @@ char *conf_get_isulad_loglevel(void) + } + return nullptr; + } ++ ++bool conf_get_systemd_cgroup(void) ++{ ++ if (g_isulad_conf_mock != nullptr) { ++ return g_isulad_conf_mock->ConfGetSystemdCgroup(); ++ } ++ return false; ++} +diff --git a/test/mocks/isulad_config_mock.h b/test/mocks/isulad_config_mock.h +index 6793fa51..d59c5938 100644 +--- a/test/mocks/isulad_config_mock.h ++++ b/test/mocks/isulad_config_mock.h +@@ -45,6 +45,7 @@ public: + MOCK_METHOD0(ConfGetSandboxStatePath, char *(void)); + MOCK_METHOD0(ConfGetEngineLogFile, char *(void)); + MOCK_METHOD0(ConfGetIsuladLogLevel, char *(void)); ++ MOCK_METHOD0(ConfGetSystemdCgroup, bool(void)); + }; + + void MockIsuladConf_SetMock(MockIsuladConf *mock); +diff --git a/test/specs/specs/specs_ut.cc b/test/specs/specs/specs_ut.cc +index 47e4ca6e..6c42216d 100644 +--- a/test/specs/specs/specs_ut.cc ++++ b/test/specs/specs/specs_ut.cc +@@ -319,6 +319,7 @@ TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_2) + ASSERT_TRUE(host_spec != nullptr); + + EXPECT_CALL(m_isulad_conf, GetCgroupParent()).WillRepeatedly(Invoke(invoke_conf_get_isulad_cgroup_parent_null)); ++ EXPECT_CALL(m_isulad_conf, ConfGetSystemdCgroup()).WillRepeatedly(Return(false)); + + merged_cp = merge_container_cgroups_path("123", host_spec); + ASSERT_NE(merged_cp, nullptr); +@@ -347,6 +348,7 @@ TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_3) + host_spec->cgroup_parent = util_strdup_s("/test"); + + EXPECT_CALL(m_isulad_conf, GetCgroupParent()).WillRepeatedly(Invoke(invoke_conf_get_isulad_cgroup_parent_null)); ++ EXPECT_CALL(m_isulad_conf, ConfGetSystemdCgroup()).WillRepeatedly(Return(false)); + + merged_cp = merge_container_cgroups_path("123", host_spec); + ASSERT_NE(merged_cp, nullptr); +@@ -373,6 +375,7 @@ TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_4) + ASSERT_TRUE(host_spec != nullptr); + + EXPECT_CALL(m_isulad_conf, GetCgroupParent()).WillRepeatedly(Invoke(invoke_conf_get_isulad_cgroup_parent)); ++ EXPECT_CALL(m_isulad_conf, ConfGetSystemdCgroup()).WillRepeatedly(Return(false)); + + merged_cp = merge_container_cgroups_path("123", host_spec); + ASSERT_NE(merged_cp, nullptr); +@@ -401,6 +404,7 @@ TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_5) + host_spec->cgroup_parent = util_strdup_s("/test"); + + EXPECT_CALL(m_isulad_conf, GetCgroupParent()).WillRepeatedly(Invoke(invoke_conf_get_isulad_cgroup_parent)); ++ EXPECT_CALL(m_isulad_conf, ConfGetSystemdCgroup()).WillRepeatedly(Return(false)); + + merged_cp = merge_container_cgroups_path("123", host_spec); + ASSERT_NE(merged_cp, nullptr); +@@ -414,6 +418,51 @@ TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_5) + testing::Mock::VerifyAndClearExpectations(&m_isulad_conf); + } + ++// systemd cgroup test ++TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_6) ++{ ++ oci_runtime_spec *oci_spec = nullptr; ++ host_config *host_spec = nullptr; ++ char *merged_cp = nullptr; ++ ++ oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec)); ++ ASSERT_TRUE(oci_spec != nullptr); ++ ++ host_spec = (host_config *)util_common_calloc_s(sizeof(host_config)); ++ ASSERT_TRUE(host_spec != nullptr); ++ ++ EXPECT_CALL(m_isulad_conf, GetCgroupParent()).WillRepeatedly(Invoke(invoke_conf_get_isulad_cgroup_parent_null)); ++ EXPECT_CALL(m_isulad_conf, ConfGetSystemdCgroup()).WillRepeatedly(Return(true)); ++ ++ merged_cp = merge_container_cgroups_path("123", host_spec); ++ ASSERT_NE(merged_cp, nullptr); ++ ASSERT_STREQ(merged_cp, "system.slice:isulad:123"); ++ free(merged_cp); ++ ++ host_spec->cgroup_parent = util_strdup_s("/test"); ++ merged_cp = merge_container_cgroups_path("123", host_spec); ++ ASSERT_EQ(merged_cp, nullptr); ++ free(host_spec->cgroup_parent); ++ ++ host_spec->cgroup_parent = util_strdup_s("test.slice"); ++ merged_cp = merge_container_cgroups_path("123", host_spec); ++ ASSERT_NE(merged_cp, nullptr); ++ ASSERT_STREQ(merged_cp, "test.slice:isulad:123"); ++ free(merged_cp); ++ free(host_spec->cgroup_parent); ++ ++ host_spec->cgroup_parent = util_strdup_s("test/test-a/test-a-b.slice"); ++ merged_cp = merge_container_cgroups_path("123", host_spec); ++ ASSERT_NE(merged_cp, nullptr); ++ ASSERT_STREQ(merged_cp, "test-a-b.slice:isulad:123"); ++ ++ free_oci_runtime_spec(oci_spec); ++ free_host_config(host_spec); ++ free(merged_cp); ++ ++ testing::Mock::VerifyAndClearExpectations(&m_isulad_conf); ++} ++ + TEST_F(SpecsUnitTest, test_update_oci_container_cgroups_path) + { + parser_error err = nullptr; +-- +2.34.1 + diff --git a/0026-move-systemd_cgroup-CI-test-to-manual-cases.patch b/0026-move-systemd_cgroup-CI-test-to-manual-cases.patch new file mode 100644 index 0000000..ea60213 --- /dev/null +++ b/0026-move-systemd_cgroup-CI-test-to-manual-cases.patch @@ -0,0 +1,18 @@ +From b93647205db5c4a5d74fb245c9b1e15ca1ffd3fe Mon Sep 17 00:00:00 2001 +From: jikai +Date: Sat, 16 Mar 2024 09:35:22 +0800 +Subject: [PATCH 26/43] move systemd_cgroup CI test to manual cases + +Signed-off-by: jikai +--- + CI/test_cases/{container_cases => manual_cases}/systemd_cgroup.sh | 0 + 1 file changed, 0 insertions(+), 0 deletions(-) + rename CI/test_cases/{container_cases => manual_cases}/systemd_cgroup.sh (100%) + +diff --git a/CI/test_cases/container_cases/systemd_cgroup.sh b/CI/test_cases/manual_cases/systemd_cgroup.sh +similarity index 100% +rename from CI/test_cases/container_cases/systemd_cgroup.sh +rename to CI/test_cases/manual_cases/systemd_cgroup.sh +-- +2.34.1 + diff --git a/0027-feature-add-support-for-cgroup-v2-metrics.patch b/0027-feature-add-support-for-cgroup-v2-metrics.patch new file mode 100644 index 0000000..f5b4073 --- /dev/null +++ b/0027-feature-add-support-for-cgroup-v2-metrics.patch @@ -0,0 +1,1084 @@ +From 7c7cd82619ed1f7e36d34da1afc2b417a90b3040 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Tue, 23 Jan 2024 17:18:51 +0800 +Subject: [PATCH 27/43] =?UTF-8?q?=E3=80=90feature=E3=80=91add=20support=20?= + =?UTF-8?q?for=20cgroup=20v2=20metrics?= +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Signed-off-by: zhongtao +--- + src/daemon/common/cgroup.c | 237 ++-------- + src/daemon/common/cgroup.h | 11 +- + src/daemon/common/cgroup_v1.c | 194 +++------ + src/daemon/common/cgroup_v2.c | 406 ++++++++++++++++++ + .../v1/v1_cri_pod_sandbox_manager_service.cc | 2 +- + .../cri_pod_sandbox_manager_service.cc | 2 +- + src/utils/cutils/utils_array.h | 6 + + 7 files changed, 506 insertions(+), 352 deletions(-) + create mode 100644 src/daemon/common/cgroup_v2.c + +diff --git a/src/daemon/common/cgroup.c b/src/daemon/common/cgroup.c +index 2d1cabb2..3c58f7fa 100644 +--- a/src/daemon/common/cgroup.c ++++ b/src/daemon/common/cgroup.c +@@ -29,32 +29,6 @@ + #include "utils_array.h" + #include "sysinfo.h" + +-// Cgroup V2 Item Definition +-#define CGROUP2_CPU_WEIGHT "cpu.weight" +-#define CGROUP2_CPU_MAX "cpu.max" +-#define CGROUP2_CPUSET_CPUS_EFFECTIVE "cpuset.cpus.effective" +-#define CGROUP2_CPUSET_MEMS_EFFECTIVE "cpuset.mems.effective" +-#define CGROUP2_CPUSET_CPUS "cpuset.cpus" +-#define CGROUP2_CPUSET_MEMS "cpuset.mems" +-#define CGROUP2_IO_WEIGHT "io.weight" +-#define CGROUP2_IO_BFQ_WEIGHT "io.bfq.weight" +-#define CGROUP2_IO_MAX "io.max" +-#define CGROUP2_MEMORY_MAX "memory.max" +-#define CGROUP2_MEMORY_LOW "memory.low" +-#define CGROUP2_MEMORY_SWAP_MAX "memory.swap.max" +-#define CGROUP2_HUGETLB_MAX "hugetlb.%s.max" +-#define CGROUP2_PIDS_MAX "pids.max" +-#define CGROUP2_FILES_LIMIT "files.limit" +- +-#define CGROUP2_CONTROLLERS_PATH CGROUP_MOUNTPOINT"/cgroup.controllers" +-#define CGROUP2_SUBTREE_CONTROLLER_PATH CGROUP_MOUNTPOINT"/cgroup.subtree_control" +-#define CGROUP2_CPUSET_CPUS_EFFECTIVE_PATH CGROUP_MOUNTPOINT"/cpuset.cpus.effective" +-#define CGROUP2_CPUSET_MEMS_EFFECTIVE_PATH CGROUP_MOUNTPOINT"/cpuset.mems.effective" +- +-#ifndef CGROUP2_SUPER_MAGIC +-#define CGROUP2_SUPER_MAGIC 0x63677270 +-#endif +- + #ifndef CGROUP_SUPER_MAGIC + #define CGROUP_SUPER_MAGIC 0x27e0eb + #endif +@@ -485,20 +459,6 @@ out: + return layers; + } + +-/* cgroup enabled */ +-static bool cgroup_enabled(const char *mountpoint, const char *name) +-{ +- char path[PATH_MAX] = { 0 }; +- int nret; +- +- nret = snprintf(path, sizeof(path), "%s/%s", mountpoint, name); +- if (nret < 0 || (size_t)nret >= sizeof(path)) { +- ERROR("Path is too long"); +- return false; +- } +- return util_file_exists(path); +-} +- + char *common_find_cgroup_subsystem_mountpoint(const cgroup_layer_t *layers, const char *subsystem) + { + size_t i; +@@ -607,188 +567,59 @@ int common_get_cgroup_version(void) + return CGROUP_VERSION_1; + } + +-static int cgroup2_enable_all() ++static int get_value_ull(const char *content, void *result) + { +- int ret = 0; +- int nret = 0; +- int n = 0; +- size_t i = 0; +- const char *space = ""; +- char *controllers_str = NULL; +- char *subtree_controller_str = NULL; +- char **controllers = NULL; +- char enable_controllers[PATH_MAX] = { 0 }; +- +- controllers_str = util_read_content_from_file(CGROUP2_CONTROLLERS_PATH); +- if (controllers_str == NULL || strlen(controllers_str) == 0 || strcmp(controllers_str, "\n") == 0) { +- WARN("no cgroup controller found"); +- goto out; +- } +- +- subtree_controller_str = util_read_content_from_file(CGROUP2_SUBTREE_CONTROLLER_PATH); +- if (subtree_controller_str != NULL && strcmp(controllers_str, subtree_controller_str) == 0) { +- goto out; +- } ++ uint64_t ull_result = 0; + +- controllers = util_string_split(controllers_str, ' '); +- if (controllers == NULL) { +- ERROR("split %s failed", controllers_str); +- ret = -1; +- goto out; +- } +- +- for (i = 0; i < util_array_len((const char **)controllers); i++) { +- nret = snprintf(enable_controllers + n, PATH_MAX - n, "%s+%s", space, controllers[i]); +- if (nret < 0 || (size_t)nret >= PATH_MAX - n) { +- ERROR("Path is too long"); +- goto out; +- } +- n += nret; +- space = " "; +- } +- ret = util_write_file(CGROUP2_SUBTREE_CONTROLLER_PATH, enable_controllers, strlen(enable_controllers), +- DEFAULT_CGROUP_FILE_MODE); +- if (ret != 0) { +- SYSERROR("write %s to %s failed", enable_controllers, CGROUP2_SUBTREE_CONTROLLER_PATH); +- goto out; ++ if (util_safe_uint64(content, &ull_result) != 0) { ++ ERROR("Failed to convert %s to uint64", content); ++ return -1; + } + +-out: +- util_free_array(controllers); +- free(controllers_str); +- free(subtree_controller_str); +- +- return ret; ++ *(uint64_t *)result = ull_result; ++ return 0; + } + +-#if defined (__ANDROID__) || defined(__MUSL__) +-static bool cgroup2_no_controller() ++int get_match_value_ull(const char *content, const char *match, void *result) + { +- char *controllers_str = NULL; ++ __isula_auto_free char *llu_string = NULL; ++ __isula_auto_free char *match_with_space = NULL; ++ __isula_auto_array_t char **lines = NULL; ++ char **worker = NULL; + +- controllers_str = util_read_content_from_file(CGROUP2_CONTROLLERS_PATH); +- if (controllers_str == NULL || strlen(controllers_str) == 0 || strcmp(controllers_str, "\n") == 0) { +- free(controllers_str); +- return true; ++ if (match == NULL) { ++ return get_value_ull(content, result); + } + +- free(controllers_str); +- return false; +-} +-#endif +- +-static int make_sure_cgroup2_isulad_path_exist() +-{ +- int ret = 0; +- +- if (util_dir_exists(CGROUP_ISULAD_PATH)) { +- return 0; ++ // match full string ++ match_with_space = util_string_append(" ", match); ++ if (match_with_space == NULL) { ++ ERROR("Failed to append string"); ++ return -1; + } + +- if (cgroup2_enable_all() != 0) { ++ lines = util_string_split(content, '\n'); ++ if (lines == NULL) { ++ ERROR("Failed to split content %s", content); + return -1; + } + +-#if defined (__ANDROID__) || defined(__MUSL__) +- if (cgroup2_no_controller()) { +- DEBUG("no cgroup controller found"); +- return 0; ++ for (worker = lines; worker && *worker; worker++) { ++ if (util_has_prefix(*worker, match_with_space)) { ++ break; ++ } + } +-#endif +- +- ret = mkdir(CGROUP_ISULAD_PATH, DEFAULT_CGROUP_DIR_MODE); +- if (ret != 0 && (errno != EEXIST || !util_dir_exists(CGROUP_ISULAD_PATH))) { ++ if (*worker == NULL) { ++ ERROR("Cannot find match string %s", match); + return -1; + } + +- return ret; +-} +- +-int common_get_cgroup_info_v2(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, +- cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, +- cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, +- cgroup_files_info_t *filesinfo, bool quiet) +-{ +- int ret = 0; +- int nret = 0; +- char *size = NULL; +- char path[PATH_MAX] = { 0 }; +- +- if (make_sure_cgroup2_isulad_path_exist() != 0) { ++ llu_string = util_sub_string(*worker, strlen(match_with_space), strlen(*worker) - strlen(match_with_space)); ++ if (llu_string == NULL) { ++ ERROR("Failed to sub string"); + return -1; + } ++ llu_string = util_trim_space(llu_string); + +- // cpu cgroup +- cpuinfo->cpu_shares = cgroup_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPU_WEIGHT); +- common_cgroup_do_log(quiet, !(cpuinfo->cpu_shares), "Your kernel does not support cgroup2 cpu weight"); +- +- cpuinfo->cpu_cfs_period = cgroup_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPU_MAX); +- cpuinfo->cpu_cfs_quota = cpuinfo->cpu_cfs_period; +- common_cgroup_do_log(quiet, !(cpuinfo->cpu_cfs_period), "Your kernel does not support cgroup2 cpu max"); +- +- cpusetinfo->cpuset = cgroup_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPUSET_CPUS_EFFECTIVE) && +- cgroup_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPUSET_CPUS) && +- cgroup_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPUSET_MEMS_EFFECTIVE) && +- cgroup_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPUSET_MEMS); +- common_cgroup_do_log(quiet, !(cpusetinfo->cpuset), "Your kernel does not support cpuset"); +- if (cpusetinfo->cpuset) { +- cpusetinfo->cpus = util_read_content_from_file(CGROUP2_CPUSET_CPUS_EFFECTIVE_PATH); +- cpusetinfo->mems = util_read_content_from_file(CGROUP2_CPUSET_MEMS_EFFECTIVE_PATH); +- if (cpusetinfo->cpus == NULL || cpusetinfo->mems == NULL) { +- ERROR("read cpus or mems failed"); +- return -1; +- } +- cpusetinfo->cpus = util_trim_space(cpusetinfo->cpus); +- cpusetinfo->mems = util_trim_space(cpusetinfo->mems); +- } +- +- // io cgroup +- blkioinfo->blkio_weight = cgroup_enabled(CGROUP_ISULAD_PATH, CGROUP2_IO_BFQ_WEIGHT) || +- cgroup_enabled(CGROUP_ISULAD_PATH, CGROUP2_IO_WEIGHT); +- blkioinfo->blkio_weight_device = blkioinfo->blkio_weight; +- common_cgroup_do_log(quiet, !(blkioinfo->blkio_weight), "Your kernel does not support cgroup2 io weight"); +- +- blkioinfo->blkio_read_bps_device = cgroup_enabled(CGROUP_ISULAD_PATH, CGROUP2_IO_MAX); +- blkioinfo->blkio_write_bps_device = blkioinfo->blkio_read_bps_device; +- blkioinfo->blkio_read_iops_device = blkioinfo->blkio_read_bps_device; +- blkioinfo->blkio_write_iops_device = blkioinfo->blkio_read_bps_device; +- common_cgroup_do_log(quiet, !(blkioinfo->blkio_read_bps_device), "Your kernel does not support cgroup2 io max"); +- +- // memory cgroup +- meminfo->limit = cgroup_enabled(CGROUP_ISULAD_PATH, CGROUP2_MEMORY_MAX); +- common_cgroup_do_log(quiet, !(meminfo->limit), "Your kernel does not support cgroup2 memory max"); +- +- meminfo->reservation = cgroup_enabled(CGROUP_ISULAD_PATH, CGROUP2_MEMORY_LOW); +- common_cgroup_do_log(quiet, !(meminfo->reservation), "Your kernel does not support cgroup2 memory low"); +- +- meminfo->swap = cgroup_enabled(CGROUP_ISULAD_PATH, CGROUP2_MEMORY_SWAP_MAX); +- common_cgroup_do_log(quiet, !(meminfo->swap), "Your kernel does not support cgroup2 memory swap max"); +- +- // pids cgroup +- pidsinfo->pidslimit = cgroup_enabled(CGROUP_ISULAD_PATH, CGROUP2_PIDS_MAX); +- common_cgroup_do_log(quiet, !(pidsinfo->pidslimit), "Your kernel does not support cgroup2 pids max"); +- +- // hugetlb cgroup +- size = get_default_huge_page_size(); +- if (size != NULL) { +- nret = snprintf(path, sizeof(path), CGROUP2_HUGETLB_MAX, size); +- if (nret < 0 || (size_t)nret >= sizeof(path)) { +- WARN("Failed to print hugetlb path"); +- ret = -1; +- goto out; +- } +- hugetlbinfo->hugetlblimit = cgroup_enabled(CGROUP_ISULAD_PATH, path); +- common_cgroup_do_log(quiet, !hugetlbinfo->hugetlblimit, "Your kernel does not support cgroup2 hugetlb limit"); +- } else { +- WARN("Your kernel does not support cgroup2 hugetlb limit"); +- } +- +- // files cgroup +- filesinfo->fileslimit = cgroup_enabled(CGROUP_ISULAD_PATH, CGROUP2_FILES_LIMIT); +- common_cgroup_do_log(quiet, !(filesinfo->fileslimit), "Your kernel does not support cgroup2 files limit"); +- +-out: +- free(size); +- +- return ret; +-} ++ return get_value_ull(llu_string, result); ++} +\ No newline at end of file +diff --git a/src/daemon/common/cgroup.h b/src/daemon/common/cgroup.h +index fa20f42c..251e3a3d 100644 +--- a/src/daemon/common/cgroup.h ++++ b/src/daemon/common/cgroup.h +@@ -31,6 +31,15 @@ extern "C" { + #define CGROUP_MOUNTPOINT "/sys/fs/cgroup" + #define CGROUP_ISULAD_PATH CGROUP_MOUNTPOINT"/isulad" + ++struct cgfile_t { ++ char *name; ++ char *file; ++ char *match; ++ int (*get_value)(const char *content, const char *match, void *result); ++}; ++ ++int get_match_value_ull(const char *content, const char *match, void *result); ++ + int common_get_cgroup_version(void); + + int common_find_cgroup_mnt_and_root(const char *subsystem, char **mountpoint, char **root); +@@ -42,7 +51,6 @@ static inline void common_cgroup_do_log(bool quiet, bool do_log, const char *msg + } + } + +- + typedef struct { + char **controllers; + char *mountpoint; +@@ -140,6 +148,7 @@ typedef struct { + } cgroup_metrics_t; + + int common_get_cgroup_v1_metrics(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics); ++int common_get_cgroup_v2_metrics(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics); + + char *common_get_init_cgroup(const char *subsystem); + +diff --git a/src/daemon/common/cgroup_v1.c b/src/daemon/common/cgroup_v1.c +index e38fc03e..e34100bc 100644 +--- a/src/daemon/common/cgroup_v1.c ++++ b/src/daemon/common/cgroup_v1.c +@@ -23,20 +23,8 @@ + + #define CGROUP_HUGETLB_LIMIT "hugetlb.%s.limit_in_bytes" + +-typedef struct { +- char *match; +-} cgfile_callback_args_t; +- +-struct cgfile_t { +- char *name; +- char *file; +- int (*get_value)(const char *content, const cgfile_callback_args_t *args, void *result); +-}; +- +-static int get_value_ll(const char *content, const cgfile_callback_args_t *args, void *result); +-static int get_value_ull(const char *content, const cgfile_callback_args_t *args, void *result); +-static int get_match_value_ull(const char *content, const cgfile_callback_args_t *args, void *result); +-static int get_value_string(const char *content, const cgfile_callback_args_t *args, void *result); ++static int get_value_ll(const char *content, const char *match, void *result); ++static int get_value_string(const char *content, const char *match, void *result); + + typedef enum { + // CPU subsystem +@@ -63,46 +51,46 @@ typedef enum { + + static struct cgfile_t g_cgroup_v1_files[] = { + // CPU subsystem +- [CPU_RT_PERIOD] = {"cpu_rt_period", "cpu.rt_period_us", get_value_ull}, +- [CPU_RT_RUNTIME] = {"cpu_rt_runtime", "cpu.rt_runtime_us", get_value_ull}, +- [CPU_SHARES] = {"cpu_shares", "cpu.shares", get_value_ull}, +- [CPU_CFS_PERIOD] = {"cpu_cfs_period", "cpu.cfs_period_us", get_value_ull}, +- [CPU_CFS_QUOTA] = {"cpu_cfs_quota", "cpu.cfs_quota_us", get_value_ll}, ++ [CPU_RT_PERIOD] = {"cpu_rt_period", "cpu.rt_period_us", NULL, get_match_value_ull}, ++ [CPU_RT_RUNTIME] = {"cpu_rt_runtime", "cpu.rt_runtime_us", NULL, get_match_value_ull}, ++ [CPU_SHARES] = {"cpu_shares", "cpu.shares", NULL, get_match_value_ull}, ++ [CPU_CFS_PERIOD] = {"cpu_cfs_period", "cpu.cfs_period_us", NULL, get_match_value_ull}, ++ [CPU_CFS_QUOTA] = {"cpu_cfs_quota", "cpu.cfs_quota_us", NULL, get_value_ll}, + // CPUSET subsystem +- [CPUSET_CPUS] = {"cpuset_cpus", "cpuset.cpus", get_value_string}, +- [CPUSET_MEMS] = {"cpuset_mems", "cpuset.mems", get_value_string}, ++ [CPUSET_CPUS] = {"cpuset_cpus", "cpuset.cpus", NULL, get_value_string}, ++ [CPUSET_MEMS] = {"cpuset_mems", "cpuset.mems", NULL, get_value_string}, + // CPUACCT subsystem +- [CPUACCT_USE_NANOS] = {"cpu_use_nanos", "cpuacct.usage", get_value_ull}, +- [CPUACCT_USE_USER] = {"cpu_use_user", "cpuacct.stat", get_match_value_ull}, +- [CPUACCT_USE_SYS] = {"cpu_use_sys", "cpuacct.stat", get_match_value_ull}, ++ [CPUACCT_USE_NANOS] = {"cpu_use_nanos", "cpuacct.usage", NULL, get_match_value_ull}, ++ [CPUACCT_USE_USER] = {"cpu_use_user", "cpuacct.stat", NULL, get_match_value_ull}, ++ [CPUACCT_USE_SYS] = {"cpu_use_sys", "cpuacct.stat", NULL, get_match_value_ull}, + // MEMORY subsystem +- [MEMORY_LIMIT] = {"mem_limit", "memory.limit_in_bytes", get_value_ull}, +- [MEMORY_USAGE] = {"mem_usage", "memory.usage_in_bytes", get_value_ull}, +- [MEMORY_SOFT_LIMIT] = {"mem_soft_limit", "memory.soft_limit_in_bytes", get_value_ull}, +- [MEMORY_KMEM_LIMIT] = {"kmem_limit", "memory.kmem.limit_in_bytes", get_value_ull}, +- [MEMORY_KMEM_USAGE] = {"kmem_usage", "memory.kmem.usage_in_bytes", get_value_ull}, +- [MEMORY_SWAPPINESS] = {"swappiness", "memory.swappiness", NULL}, +- [MEMORY_SW_LIMIT] = {"memsw_limit", "memory.memsw.limit_in_bytes", get_value_ull}, +- [MEMORY_SW_USAGE] = {"memsw_usage", "memory.memsw.usage_in_bytes", get_value_ull}, +- [MEMORY_CACHE] = {"cache", "memory.stat", get_match_value_ull}, +- [MEMORY_CACHE_TOTAL] = {"cache_total", "memory.stat", get_match_value_ull}, +- [MEMORY_TOTAL_RSS] = {"total_rss", "memory.stat", get_match_value_ull}, +- [MEMORY_TOTAL_PGFAULT] = {"total_page_fault", "memory.stat", get_match_value_ull}, +- [MEMORY_TOTAL_PGMAJFAULT] = {"total_page_majfault", "memory.stat", get_match_value_ull}, +- [MEMORY_TOTAL_INACTIVE_FILE] = {"total_inactive_file", "memory.stat", get_match_value_ull}, +- [MEMORY_OOM_CONTROL] = {"oom_control", "memory.oom_control", NULL}, ++ [MEMORY_LIMIT] = {"mem_limit", "memory.limit_in_bytes", NULL, get_match_value_ull}, ++ [MEMORY_USAGE] = {"mem_usage", "memory.usage_in_bytes", NULL, get_match_value_ull}, ++ [MEMORY_SOFT_LIMIT] = {"mem_soft_limit", "memory.soft_limit_in_bytes", NULL, get_match_value_ull}, ++ [MEMORY_KMEM_LIMIT] = {"kmem_limit", "memory.kmem.limit_in_bytes", NULL, get_match_value_ull}, ++ [MEMORY_KMEM_USAGE] = {"kmem_usage", "memory.kmem.usage_in_bytes", NULL, get_match_value_ull}, ++ [MEMORY_SWAPPINESS] = {"swappiness", "memory.swappiness", NULL, NULL}, ++ [MEMORY_SW_LIMIT] = {"memsw_limit", "memory.memsw.limit_in_bytes", NULL, get_match_value_ull}, ++ [MEMORY_SW_USAGE] = {"memsw_usage", "memory.memsw.usage_in_bytes", NULL, get_match_value_ull}, ++ [MEMORY_CACHE] = {"cache", "memory.stat", NULL, get_match_value_ull}, ++ [MEMORY_CACHE_TOTAL] = {"cache_total", "memory.stat", NULL, get_match_value_ull}, ++ [MEMORY_TOTAL_RSS] = {"total_rss", "memory.stat", "total_rss", get_match_value_ull}, ++ [MEMORY_TOTAL_PGFAULT] = {"total_page_fault", "memory.stat", "total_pgfault", get_match_value_ull}, ++ [MEMORY_TOTAL_PGMAJFAULT] = {"total_page_majfault", "memory.stat", "total_pgmajfault", get_match_value_ull}, ++ [MEMORY_TOTAL_INACTIVE_FILE] = {"total_inactive_file", "memory.stat", "total_inactive_file", get_match_value_ull}, ++ [MEMORY_OOM_CONTROL] = {"oom_control", "memory.oom_control", NULL, NULL}, + // BLKIO subsystem +- [BLKIO_WEIGTH] = {"blkio_weigth", "blkio.weight", NULL}, +- [BLKIO_WEIGTH_DEVICE] = {"blkio_weigth_device", "blkio.weight_device", NULL}, +- [BLKIO_READ_BPS] = {"blkio_read_bps", "blkio.throttle.read_bps_device", NULL}, +- [BLKIO_WRITE_BPS] = {"blkio_write_bps", "blkio.throttle.write_bps_device", NULL}, +- [BLKIO_READ_IOPS] = {"blkio_read_iops", "blkio.throttle.read_iops_device", NULL}, +- [BLKIO_WRITE_IOPS] = {"blkio_write_iops", "blkio.throttle.write_iops_device", NULL}, ++ [BLKIO_WEIGTH] = {"blkio_weigth", "blkio.weight", NULL, NULL}, ++ [BLKIO_WEIGTH_DEVICE] = {"blkio_weigth_device", "blkio.weight_device", NULL, NULL}, ++ [BLKIO_READ_BPS] = {"blkio_read_bps", "blkio.throttle.read_bps_device", NULL, NULL}, ++ [BLKIO_WRITE_BPS] = {"blkio_write_bps", "blkio.throttle.write_bps_device", NULL, NULL}, ++ [BLKIO_READ_IOPS] = {"blkio_read_iops", "blkio.throttle.read_iops_device", NULL, NULL}, ++ [BLKIO_WRITE_IOPS] = {"blkio_write_iops", "blkio.throttle.write_iops_device", NULL, NULL}, + // PIDS subsystem +- [PIDS_CURRENT] = {"pids_current", "pids.current", get_value_ull}, ++ [PIDS_CURRENT] = {"pids_current", "pids.current", NULL, get_match_value_ull}, + }; + +-static int get_value_ll(const char *content, const cgfile_callback_args_t *args, void *result) ++static int get_value_ll(const char *content, const char *match, void *result) + { + long long ll_result = 0; + +@@ -115,81 +103,7 @@ static int get_value_ll(const char *content, const cgfile_callback_args_t *args, + return 0; + } + +-static int get_value_ull(const char *content, const cgfile_callback_args_t *args, void *result) +-{ +- uint64_t ull_result = 0; +- +- if (util_safe_uint64(content, &ull_result) != 0) { +- ERROR("Failed to convert %s to uint64", content); +- return -1; +- } +- +- *(uint64_t *)result = ull_result; +- return 0; +-} +- +-static int get_match_value_ull(const char *content, const cgfile_callback_args_t *args, void *result) +-{ +- int ret = 0; +- uint64_t llu_result = 0; +- char *llu_string = NULL; +- char *match_with_space = NULL; +- char **lines = NULL; +- char **worker = NULL; +- +- if (args == NULL || args->match == NULL || strlen(args->match) == 0) { +- ERROR("Invalid arguments"); +- return -1; +- } +- +- // match full string +- match_with_space = util_string_append(" ", args->match); +- if (match_with_space == NULL) { +- ERROR("Failed to append string"); +- return -1; +- } +- +- lines = util_string_split(content, '\n'); +- if (lines == NULL) { +- ERROR("Failed to split content %s", content); +- ret = -1; +- goto out; +- } +- +- for (worker = lines; worker && *worker; worker++) { +- if (util_has_prefix(*worker, match_with_space)) { +- break; +- } +- } +- if (*worker == NULL) { +- ERROR("Cannot find match string %s", args->match); +- ret = -1; +- goto out; +- } +- +- llu_string = util_sub_string(*worker, strlen(match_with_space), strlen(*worker) - strlen(match_with_space)); +- if (llu_string == NULL) { +- ERROR("Failed to sub string"); +- ret = -1; +- goto out; +- } +- llu_string = util_trim_space(llu_string); +- +- ret = util_safe_uint64(llu_string, &llu_result); +- if (ret != 0) { +- ERROR("Failed to convert %s to uint64", llu_string); +- } else { +- *(uint64_t *)result = llu_result; +- } +- +-out: +- free(match_with_space); +- free(llu_string); +- util_free_array(lines); +- return ret; +-} +- +-static int get_value_string(const char *content, const cgfile_callback_args_t *args, void *result) ++static int get_value_string(const char *content, const char *match, void *result) + { + *(char **)result = util_strdup_s(content); + return 0; +@@ -228,7 +142,7 @@ static bool check_cgroup_v1_helper(const char *mountpoint, const cgroup_v1_files + } + + static int get_cgroup_v1_value_helper(const char *path, const cgroup_v1_files_index index, +- const cgfile_callback_args_t *args, void *result) ++ void *result) + { + int nret = 0; + char file_path[PATH_MAX] = { 0 }; +@@ -265,7 +179,7 @@ static int get_cgroup_v1_value_helper(const char *path, const cgroup_v1_files_in + util_trim_newline(content); + content = util_trim_space(content); + +- nret = g_cgroup_v1_files[index].get_value(content, args, result); ++ nret = g_cgroup_v1_files[index].get_value(content, g_cgroup_v1_files[index].match, result); + if (nret != 0) { + ERROR("%s: failed to get value", g_cgroup_v1_files[index].name); + } +@@ -308,11 +222,11 @@ static void check_cgroup_v1_cpuset(const cgroup_layer_t *layers, const bool quie + return; + } + +- if (get_cgroup_v1_value_helper(mountpoint, CPUSET_CPUS, NULL, (void *)&cpusetinfo->cpus) != 0) { ++ if (get_cgroup_v1_value_helper(mountpoint, CPUSET_CPUS, (void *)&cpusetinfo->cpus) != 0) { + ERROR("Failed to get cgroup cpuset.cpus data"); + return; + } +- if (get_cgroup_v1_value_helper(mountpoint, CPUSET_MEMS, NULL, (void *)&cpusetinfo->mems) != 0) { ++ if (get_cgroup_v1_value_helper(mountpoint, CPUSET_MEMS, (void *)&cpusetinfo->mems) != 0) { + free(cpusetinfo->cpus); + cpusetinfo->cpus = NULL; + ERROR("Failed to get cgroup cpuset.cpus data"); +@@ -463,7 +377,7 @@ static void get_cgroup_v1_metrics_cpu(const cgroup_layer_t *layers, const char * + return; + } + +- get_cgroup_v1_value_helper(path, CPUACCT_USE_NANOS, NULL, (void *)&cgroup_cpu_metrics->cpu_use_nanos); ++ get_cgroup_v1_value_helper(path, CPUACCT_USE_NANOS, (void *)&cgroup_cpu_metrics->cpu_use_nanos); + } + + static void get_cgroup_v1_metrics_memory(const cgroup_layer_t *layers, const char *cgroup_path, +@@ -472,18 +386,6 @@ static void get_cgroup_v1_metrics_memory(const cgroup_layer_t *layers, const cha + int nret = 0; + char *mountpoint = NULL; + char path[PATH_MAX] = { 0 }; +- const cgfile_callback_args_t total_inactive_file_arg = { +- .match = "total_inactive_file", +- }; +- const cgfile_callback_args_t total_rss_arg = { +- .match = "total_rss", +- }; +- const cgfile_callback_args_t total_pgfault_arg = { +- .match = "total_pgfault", +- }; +- const cgfile_callback_args_t total_pgmajfault_arg = { +- .match = "total_pgmajfault", +- }; + + mountpoint = common_find_cgroup_subsystem_mountpoint(layers, "memory"); + if (mountpoint == NULL) { +@@ -497,14 +399,14 @@ static void get_cgroup_v1_metrics_memory(const cgroup_layer_t *layers, const cha + return; + } + +- get_cgroup_v1_value_helper(path, MEMORY_LIMIT, NULL, (void *)&cgroup_mem_metrics->mem_limit); +- get_cgroup_v1_value_helper(path, MEMORY_USAGE, NULL, (void *)&cgroup_mem_metrics->mem_used); +- get_cgroup_v1_value_helper(path, MEMORY_TOTAL_RSS, &total_rss_arg, (void *)&cgroup_mem_metrics->total_rss); +- get_cgroup_v1_value_helper(path, MEMORY_TOTAL_PGFAULT, &total_pgfault_arg, ++ get_cgroup_v1_value_helper(path, MEMORY_LIMIT, (void *)&cgroup_mem_metrics->mem_limit); ++ get_cgroup_v1_value_helper(path, MEMORY_USAGE, (void *)&cgroup_mem_metrics->mem_used); ++ get_cgroup_v1_value_helper(path, MEMORY_TOTAL_RSS, (void *)&cgroup_mem_metrics->total_rss); ++ get_cgroup_v1_value_helper(path, MEMORY_TOTAL_PGFAULT, + (void *)&cgroup_mem_metrics->total_pgfault); +- get_cgroup_v1_value_helper(path, MEMORY_TOTAL_PGMAJFAULT, &total_pgmajfault_arg, ++ get_cgroup_v1_value_helper(path, MEMORY_TOTAL_PGMAJFAULT, + (void *)&cgroup_mem_metrics->total_pgmajfault); +- get_cgroup_v1_value_helper(path, MEMORY_TOTAL_INACTIVE_FILE, &total_inactive_file_arg, ++ get_cgroup_v1_value_helper(path, MEMORY_TOTAL_INACTIVE_FILE, + (void *)&cgroup_mem_metrics->total_inactive_file); + } + +@@ -527,7 +429,7 @@ static void get_cgroup_v1_metrics_pid(const cgroup_layer_t *layers, const char * + return; + } + +- get_cgroup_v1_value_helper(path, PIDS_CURRENT, NULL, (void *)&cgroup_pids_metrics->pid_current); ++ get_cgroup_v1_value_helper(path, PIDS_CURRENT, (void *)&cgroup_pids_metrics->pid_current); + } + + int common_get_cgroup_v1_metrics(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics) +diff --git a/src/daemon/common/cgroup_v2.c b/src/daemon/common/cgroup_v2.c +new file mode 100644 +index 00000000..25509bda +--- /dev/null ++++ b/src/daemon/common/cgroup_v2.c +@@ -0,0 +1,406 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-01-16 ++ * Description: provide cgroup v2 functions ++ ******************************************************************************/ ++#include "cgroup.h" ++ ++#include ++#include ++#include ++ ++#include ++ ++#include "utils.h" ++#include "path.h" ++#include "sysinfo.h" ++ ++// Cgroup V2 Item Definition ++#define CGROUP2_CPU_WEIGHT "cpu.weight" ++#define CGROUP2_CPU_MAX "cpu.max" ++#define CGROUP2_CPUSET_CPUS_EFFECTIVE "cpuset.cpus.effective" ++#define CGROUP2_CPUSET_MEMS_EFFECTIVE "cpuset.mems.effective" ++#define CGROUP2_CPUSET_CPUS "cpuset.cpus" ++#define CGROUP2_CPUSET_MEMS "cpuset.mems" ++#define CGROUP2_IO_WEIGHT "io.weight" ++#define CGROUP2_IO_BFQ_WEIGHT "io.bfq.weight" ++#define CGROUP2_IO_MAX "io.max" ++#define CGROUP2_MEMORY_MAX "memory.max" ++#define CGROUP2_MEMORY_LOW "memory.low" ++#define CGROUP2_MEMORY_SWAP_MAX "memory.swap.max" ++#define CGROUP2_HUGETLB_MAX "hugetlb.%s.max" ++#define CGROUP2_PIDS_MAX "pids.max" ++#define CGROUP2_FILES_LIMIT "files.limit" ++ ++#define CGROUP2_CONTROLLERS_PATH CGROUP_MOUNTPOINT"/cgroup.controllers" ++#define CGROUP2_SUBTREE_CONTROLLER_PATH CGROUP_MOUNTPOINT"/cgroup.subtree_control" ++#define CGROUP2_CPUSET_CPUS_EFFECTIVE_PATH CGROUP_MOUNTPOINT"/cpuset.cpus.effective" ++#define CGROUP2_CPUSET_MEMS_EFFECTIVE_PATH CGROUP_MOUNTPOINT"/cpuset.mems.effective" ++ ++#ifndef CGROUP2_SUPER_MAGIC ++#define CGROUP2_SUPER_MAGIC 0x63677270 ++#endif ++ ++static int get_value_ull_v2(const char *content, const char *match, void *result) ++{ ++ uint64_t ull_result = 0; ++ __isula_auto_free char *tmp_str = util_strdup_s(content); ++ ++ tmp_str = util_trim_space(tmp_str); ++ if (strcmp(tmp_str, "max") == 0) { ++ *(uint64_t *)result = UINT64_MAX; ++ return 0; ++ } ++ ++ if (util_safe_uint64(content, &ull_result) != 0) { ++ ERROR("Failed to convert %s to uint64", content); ++ return -1; ++ } ++ ++ *(uint64_t *)result = ull_result; ++ return 0; ++} ++ ++typedef enum { ++ // cpu ++ CPUACCT_USE_USER, CPUACCT_USE_SYS, CPUACCT_USE_NANOS, ++ // MEMORY subsystem ++ MEMORY_USAGE, MEMORY_LIMIT, MEMORY_ANON, ++ MEMORY_TOTAL_PGFAULT, MEMORY_TOTAL_INACTIVE_FILE, MEMORY_TOTAL_PGMAJFAULT, ++ MEMORY_CACHE, MEMORY_CACHE_TOTAL, ++ // BLKIO subsystem ++ BLKIO_READ_BPS, BLKIO_WRITE_BPS, BLKIO_READ_IOPS, BLKIO_WRITE_IOPS, ++ // PIDS subsystem ++ PIDS_CURRENT, ++ // MAX ++ CGROUP_V2_FILES_INDEX_MAXS ++} cgroup_v2_files_index; ++ ++static struct cgfile_t g_cgroup_v2_files[] = { ++ // cpu ++ [CPUACCT_USE_USER] = {"cpu_use_user", "cpu.stat", "user_usec", get_match_value_ull}, ++ [CPUACCT_USE_SYS] = {"cpu_use_sys", "cpu.stat", "system_usec", get_match_value_ull}, ++ [CPUACCT_USE_NANOS] = {"cpu_use_nanos", "cpu.stat", "usage_usec", get_match_value_ull}, ++ // memory ++ [MEMORY_USAGE] = {"mem_usage", "memory.current", NULL, get_value_ull_v2}, ++ [MEMORY_LIMIT] = {"mem_limit", "memory.max", NULL, get_value_ull_v2}, ++ [MEMORY_ANON] = {"mem_anon", "memory.stat", "anon", get_match_value_ull}, ++ [MEMORY_TOTAL_PGFAULT] = {"total_page_fault", "memory.stat", "pgfault", get_match_value_ull}, ++ [MEMORY_TOTAL_PGMAJFAULT] = {"total_page_majfault", "memory.stat", "pgmajfault", get_match_value_ull}, ++ [MEMORY_TOTAL_INACTIVE_FILE] = {"total_inactive_file", "memory.stat", "inactive_file", get_match_value_ull}, ++ [MEMORY_CACHE] = {"cache", "memory.stat", "file", get_match_value_ull}, ++ [MEMORY_CACHE_TOTAL] = {"cache_total", "memory.stat", "file", get_match_value_ull}, ++ // pids ++ [PIDS_CURRENT] = {"pids_current", "pids.current", NULL, get_value_ull_v2}, ++}; ++ ++static int get_cgroup_v2_value_helper(const char *path, const cgroup_v2_files_index index, void *result) ++{ ++ int nret = 0; ++ char file_path[PATH_MAX] = { 0 }; ++ char real_path[PATH_MAX] = { 0 }; ++ char *content = NULL; ++ ++ if (index >= CGROUP_V2_FILES_INDEX_MAXS) { ++ ERROR("Index out of range"); ++ return false; ++ } ++ ++ if (path == NULL || strlen(path) == 0 || result == NULL) { ++ ERROR("%s: Invalid arguments", g_cgroup_v2_files[index].name); ++ return -1; ++ } ++ ++ nret = snprintf(file_path, sizeof(file_path), "%s/%s", path, g_cgroup_v2_files[index].file); ++ if (nret < 0 || (size_t)nret >= sizeof(file_path)) { ++ ERROR("%s: failed to snprintf", g_cgroup_v2_files[index].name); ++ return -1; ++ } ++ ++ if (util_clean_path(file_path, real_path, sizeof(real_path)) == NULL) { ++ ERROR("%s: failed to clean path %s", g_cgroup_v2_files[index].name, file_path); ++ return -1; ++ } ++ ++ content = util_read_content_from_file(real_path); ++ if (content == NULL) { ++ ERROR("%s: failed to read file %s", g_cgroup_v2_files[index].name, real_path); ++ return -1; ++ } ++ ++ util_trim_newline(content); ++ content = util_trim_space(content); ++ ++ nret = g_cgroup_v2_files[index].get_value(content, g_cgroup_v2_files[index].match, result); ++ if (nret != 0) { ++ ERROR("%s: failed to get value", g_cgroup_v2_files[index].name); ++ } ++ ++ free(content); ++ return nret; ++} ++ ++static void get_cgroup_v2_metrics_cpu(const char *cgroup_path, cgroup_cpu_metrics_t *cgroup_cpu_metrics) ++{ ++ int nret = 0; ++ char path[PATH_MAX] = { 0 }; ++ ++ nret = snprintf(path, sizeof(path), "%s/%s", CGROUP_MOUNTPOINT, cgroup_path); ++ if (nret < 0 || (size_t)nret >= sizeof(path)) { ++ ERROR("Failed to snprintf"); ++ return; ++ } ++ ++ get_cgroup_v2_value_helper(path, CPUACCT_USE_NANOS, (void *)&cgroup_cpu_metrics->cpu_use_nanos); ++} ++ ++static void get_cgroup_v2_metrics_memory(const char *cgroup_path, cgroup_mem_metrics_t *cgroup_mem_metrics) ++{ ++ int nret = 0; ++ char path[PATH_MAX] = { 0 }; ++ ++ nret = snprintf(path, sizeof(path), "%s/%s", CGROUP_MOUNTPOINT, cgroup_path); ++ if (nret < 0 || (size_t)nret >= sizeof(path)) { ++ ERROR("Failed to snprintf"); ++ return; ++ } ++ ++ get_cgroup_v2_value_helper(path, MEMORY_LIMIT, (void *)&cgroup_mem_metrics->mem_limit); ++ get_cgroup_v2_value_helper(path, MEMORY_USAGE, (void *)&cgroup_mem_metrics->mem_used); ++ // Use Anon memory for RSS as cAdvisor on cgroupv2 ++ // see https://github.com/google/cadvisor/blob/a9858972e75642c2b1914c8d5428e33e6392c08a/container/libcontainer/handler.go#L799 ++ get_cgroup_v2_value_helper(path, MEMORY_ANON, (void *)&cgroup_mem_metrics->total_rss); ++ get_cgroup_v2_value_helper(path, MEMORY_TOTAL_PGFAULT, ++ (void *)&cgroup_mem_metrics->total_pgfault); ++ get_cgroup_v2_value_helper(path, MEMORY_TOTAL_PGMAJFAULT, ++ (void *)&cgroup_mem_metrics->total_pgmajfault); ++ get_cgroup_v2_value_helper(path, MEMORY_TOTAL_INACTIVE_FILE, ++ (void *)&cgroup_mem_metrics->total_inactive_file); ++} ++ ++static void get_cgroup_v2_metrics_pid(const char *cgroup_path, cgroup_pids_metrics_t *cgroup_pids_metrics) ++{ ++ int nret = 0; ++ char path[PATH_MAX] = { 0 }; ++ ++ nret = snprintf(path, sizeof(path), "%s/%s", CGROUP_MOUNTPOINT, cgroup_path); ++ if (nret < 0 || (size_t)nret >= sizeof(path)) { ++ ERROR("Failed to snprintf"); ++ return; ++ } ++ ++ get_cgroup_v2_value_helper(path, PIDS_CURRENT, (void *)&cgroup_pids_metrics->pid_current); ++} ++ ++static int cgroup2_enable_all() ++{ ++ int ret = 0; ++ int nret = 0; ++ int n = 0; ++ size_t i = 0; ++ const char *space = ""; ++ __isula_auto_free char *controllers_str = NULL; ++ __isula_auto_free char *subtree_controller_str = NULL; ++ __isula_auto_array_t char **controllers = NULL; ++ char enable_controllers[PATH_MAX] = { 0 }; ++ ++ controllers_str = util_read_content_from_file(CGROUP2_CONTROLLERS_PATH); ++ if (controllers_str == NULL || strlen(controllers_str) == 0 || strcmp(controllers_str, "\n") == 0) { ++ WARN("no cgroup controller found"); ++ return ret; ++ } ++ ++ subtree_controller_str = util_read_content_from_file(CGROUP2_SUBTREE_CONTROLLER_PATH); ++ if (subtree_controller_str != NULL && strcmp(controllers_str, subtree_controller_str) == 0) { ++ return ret; ++ } ++ ++ controllers = util_string_split(controllers_str, ' '); ++ if (controllers == NULL) { ++ ERROR("split %s failed", controllers_str); ++ return -1; ++ } ++ ++ for (i = 0; i < util_array_len((const char **)controllers); i++) { ++ nret = snprintf(enable_controllers + n, PATH_MAX - n, "%s+%s", space, controllers[i]); ++ if (nret < 0 || (size_t)nret >= PATH_MAX - n) { ++ ERROR("Path is too long"); ++ return -1; ++ } ++ n += nret; ++ space = " "; ++ } ++ ret = util_write_file(CGROUP2_SUBTREE_CONTROLLER_PATH, enable_controllers, strlen(enable_controllers), ++ DEFAULT_CGROUP_FILE_MODE); ++ if (ret != 0) { ++ SYSERROR("write %s to %s failed", enable_controllers, CGROUP2_SUBTREE_CONTROLLER_PATH); ++ return ret; ++ } ++ ++ return ret; ++} ++ ++#if defined (__ANDROID__) || defined(__MUSL__) ++static bool cgroup2_no_controller() ++{ ++ char *controllers_str = NULL; ++ ++ controllers_str = util_read_content_from_file(CGROUP2_CONTROLLERS_PATH); ++ if (controllers_str == NULL || strlen(controllers_str) == 0 || strcmp(controllers_str, "\n") == 0) { ++ free(controllers_str); ++ return true; ++ } ++ ++ free(controllers_str); ++ return false; ++} ++#endif ++ ++static int make_sure_cgroup2_isulad_path_exist() ++{ ++ int ret = 0; ++ ++ if (util_dir_exists(CGROUP_ISULAD_PATH)) { ++ return 0; ++ } ++ ++ if (cgroup2_enable_all() != 0) { ++ return -1; ++ } ++ ++#if defined (__ANDROID__) || defined(__MUSL__) ++ if (cgroup2_no_controller()) { ++ DEBUG("no cgroup controller found"); ++ return 0; ++ } ++#endif ++ ++ ret = mkdir(CGROUP_ISULAD_PATH, DEFAULT_CGROUP_DIR_MODE); ++ if (ret != 0 && (errno != EEXIST || !util_dir_exists(CGROUP_ISULAD_PATH))) { ++ return -1; ++ } ++ ++ return ret; ++} ++ ++/* cgroup enabled */ ++static bool cgroup_v2_enabled(const char *mountpoint, const char *name) ++{ ++ char path[PATH_MAX] = { 0 }; ++ int nret; ++ ++ nret = snprintf(path, sizeof(path), "%s/%s", mountpoint, name); ++ if (nret < 0 || (size_t)nret >= sizeof(path)) { ++ ERROR("Path is too long"); ++ return false; ++ } ++ return util_file_exists(path); ++} ++ ++int common_get_cgroup_info_v2(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, ++ cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, ++ cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, ++ cgroup_files_info_t *filesinfo, bool quiet) ++{ ++ int ret = 0; ++ int nret = 0; ++ __isula_auto_free char *size = NULL; ++ char path[PATH_MAX] = { 0 }; ++ ++ if (make_sure_cgroup2_isulad_path_exist() != 0) { ++ return -1; ++ } ++ ++ // cpu cgroup ++ cpuinfo->cpu_shares = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPU_WEIGHT); ++ common_cgroup_do_log(quiet, !(cpuinfo->cpu_shares), "Your kernel does not support cgroup2 cpu weight"); ++ ++ cpuinfo->cpu_cfs_period = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPU_MAX); ++ cpuinfo->cpu_cfs_quota = cpuinfo->cpu_cfs_period; ++ common_cgroup_do_log(quiet, !(cpuinfo->cpu_cfs_period), "Your kernel does not support cgroup2 cpu max"); ++ ++ cpusetinfo->cpuset = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPUSET_CPUS_EFFECTIVE) && ++ cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPUSET_CPUS) && ++ cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPUSET_MEMS_EFFECTIVE) && ++ cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPUSET_MEMS); ++ common_cgroup_do_log(quiet, !(cpusetinfo->cpuset), "Your kernel does not support cpuset"); ++ if (cpusetinfo->cpuset) { ++ cpusetinfo->cpus = util_read_content_from_file(CGROUP2_CPUSET_CPUS_EFFECTIVE_PATH); ++ cpusetinfo->mems = util_read_content_from_file(CGROUP2_CPUSET_MEMS_EFFECTIVE_PATH); ++ if (cpusetinfo->cpus == NULL || cpusetinfo->mems == NULL) { ++ ERROR("read cpus or mems failed"); ++ return -1; ++ } ++ cpusetinfo->cpus = util_trim_space(cpusetinfo->cpus); ++ cpusetinfo->mems = util_trim_space(cpusetinfo->mems); ++ } ++ ++ // io cgroup ++ blkioinfo->blkio_weight = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_IO_BFQ_WEIGHT) || ++ cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_IO_WEIGHT); ++ blkioinfo->blkio_weight_device = blkioinfo->blkio_weight; ++ common_cgroup_do_log(quiet, !(blkioinfo->blkio_weight), "Your kernel does not support cgroup2 io weight"); ++ ++ blkioinfo->blkio_read_bps_device = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_IO_MAX); ++ blkioinfo->blkio_write_bps_device = blkioinfo->blkio_read_bps_device; ++ blkioinfo->blkio_read_iops_device = blkioinfo->blkio_read_bps_device; ++ blkioinfo->blkio_write_iops_device = blkioinfo->blkio_read_bps_device; ++ common_cgroup_do_log(quiet, !(blkioinfo->blkio_read_bps_device), "Your kernel does not support cgroup2 io max"); ++ ++ // memory cgroup ++ meminfo->limit = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_MEMORY_MAX); ++ common_cgroup_do_log(quiet, !(meminfo->limit), "Your kernel does not support cgroup2 memory max"); ++ ++ meminfo->reservation = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_MEMORY_LOW); ++ common_cgroup_do_log(quiet, !(meminfo->reservation), "Your kernel does not support cgroup2 memory low"); ++ ++ meminfo->swap = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_MEMORY_SWAP_MAX); ++ common_cgroup_do_log(quiet, !(meminfo->swap), "Your kernel does not support cgroup2 memory swap max"); ++ ++ // pids cgroup ++ pidsinfo->pidslimit = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_PIDS_MAX); ++ common_cgroup_do_log(quiet, !(pidsinfo->pidslimit), "Your kernel does not support cgroup2 pids max"); ++ ++ // hugetlb cgroup ++ size = get_default_huge_page_size(); ++ if (size != NULL) { ++ nret = snprintf(path, sizeof(path), CGROUP2_HUGETLB_MAX, size); ++ if (nret < 0 || (size_t)nret >= sizeof(path)) { ++ WARN("Failed to print hugetlb path"); ++ return -1; ++ } ++ hugetlbinfo->hugetlblimit = cgroup_v2_enabled(CGROUP_ISULAD_PATH, path); ++ common_cgroup_do_log(quiet, !hugetlbinfo->hugetlblimit, "Your kernel does not support cgroup2 hugetlb limit"); ++ } else { ++ WARN("Your kernel does not support cgroup2 hugetlb limit"); ++ } ++ ++ // files cgroup ++ filesinfo->fileslimit = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_FILES_LIMIT); ++ common_cgroup_do_log(quiet, !(filesinfo->fileslimit), "Your kernel does not support cgroup2 files limit"); ++ ++ return ret; ++} ++ ++int common_get_cgroup_v2_metrics(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics) ++{ ++ if (cgroup_path == NULL || strlen(cgroup_path) == 0 || cgroup_metrics == NULL) { ++ ERROR("Invalid arguments"); ++ return -1; ++ } ++ ++ get_cgroup_v2_metrics_cpu(cgroup_path, &cgroup_metrics->cgcpu_metrics); ++ get_cgroup_v2_metrics_memory(cgroup_path, &cgroup_metrics->cgmem_metrics); ++ get_cgroup_v2_metrics_pid(cgroup_path, &cgroup_metrics->cgpids_metrics); ++ ++ return 0; ++} +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +index 76fa17bc..3bdc3af8 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +@@ -881,7 +881,7 @@ void PodSandboxManagerService::GetPodSandboxCgroupMetrics(const std::string &cgr + if (cgroupVersion == CGROUP_VERSION_1) { + nret = common_get_cgroup_v1_metrics(cgroupParent.c_str(), &cgroupMetrics); + } else { +- // todo: get cgroup v2 metrics ++ nret = common_get_cgroup_v2_metrics(cgroupParent.c_str(), &cgroupMetrics); + } + + if (nret != 0) { +diff --git a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +index 4d1d19eb..49a7ca54 100644 +--- a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +@@ -1337,7 +1337,7 @@ void PodSandboxManagerService::GetPodSandboxCgroupMetrics(const container_inspec + if (cgroupVersion == CGROUP_VERSION_1) { + nret = common_get_cgroup_v1_metrics(cgroupParent, &cgroupMetrics); + } else { +- // todo: get cgroup v2 metrics ++ nret = common_get_cgroup_v2_metrics(cgroupParent, &cgroupMetrics); + } + + if (nret != 0) { +diff --git a/src/utils/cutils/utils_array.h b/src/utils/cutils/utils_array.h +index 64f41496..1c084595 100644 +--- a/src/utils/cutils/utils_array.h ++++ b/src/utils/cutils/utils_array.h +@@ -18,6 +18,7 @@ + + #include + #include ++#include + + #ifdef __cplusplus + extern "C" { +@@ -57,6 +58,11 @@ void util_free_sensitive_array(char **array); + + void util_free_sensitive_array_by_len(char **array, size_t len); + ++// define auto free function callback for char * ++define_auto_cleanup_callback(util_free_array, char *); ++// define auto free macro for char * ++#define __isula_auto_array_t auto_cleanup_tag(util_free_array) ++ + #ifdef __cplusplus + } + #endif +-- +2.34.1 + diff --git a/0028-use-supervisor-to-notify-sandbox-exit-event.patch b/0028-use-supervisor-to-notify-sandbox-exit-event.patch new file mode 100644 index 0000000..379eb90 --- /dev/null +++ b/0028-use-supervisor-to-notify-sandbox-exit-event.patch @@ -0,0 +1,254 @@ +From 835185f7c4739993c2ca26d737bb0a45277ad932 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Wed, 20 Mar 2024 15:48:42 +0800 +Subject: [PATCH 28/43] use supervisor to notify sandbox exit event + +Signed-off-by: jikai +--- + src/daemon/modules/api/container_api.h | 2 +- + .../modules/container/restore/restore.c | 6 +++- + .../modules/container/supervisor/supervisor.c | 15 +++++++- + .../modules/service/service_container.c | 9 +++-- + .../controller/shim/shim_controller.cc | 35 ++----------------- + src/daemon/sandbox/sandbox_ops.cc | 23 ++++++++++++ + src/daemon/sandbox/sandbox_ops.h | 2 ++ + 7 files changed, 53 insertions(+), 39 deletions(-) + +diff --git a/src/daemon/modules/api/container_api.h b/src/daemon/modules/api/container_api.h +index 4602d244..43d66d64 100644 +--- a/src/daemon/modules/api/container_api.h ++++ b/src/daemon/modules/api/container_api.h +@@ -270,7 +270,7 @@ bool container_is_valid_state_string(const char *state); + void container_update_health_monitor(const char *container_id); + + extern int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_info, const char *name, +- const char *runtime); ++ const char *runtime, bool sandbox_container); + + extern char *container_exit_fifo_create(const char *cont_state_path); + +diff --git a/src/daemon/modules/container/restore/restore.c b/src/daemon/modules/container/restore/restore.c +index 2669ea22..76868e28 100644 +--- a/src/daemon/modules/container/restore/restore.c ++++ b/src/daemon/modules/container/restore/restore.c +@@ -57,6 +57,7 @@ static int restore_supervisor(const container_t *cont) + char *statepath = cont->state_path; + char *runtime = cont->runtime; + pid_ppid_info_t pid_info = { 0 }; ++ bool sandbox_container = false; + + nret = snprintf(container_state, sizeof(container_state), "%s/%s", statepath, id); + if (nret < 0 || (size_t)nret >= sizeof(container_state)) { +@@ -90,8 +91,11 @@ static int restore_supervisor(const container_t *cont) + pid_info.ppid = cont->state->state->p_pid; + pid_info.start_time = cont->state->state->start_time; + pid_info.pstart_time = cont->state->state->p_start_time; ++#ifdef ENABLE_CRI_API_V1 ++ sandbox_container = is_sandbox_container(cont->common_config->sandbox_info); ++#endif + +- if (container_supervisor_add_exit_monitor(exit_fifo_fd, &pid_info, id, runtime)) { ++ if (container_supervisor_add_exit_monitor(exit_fifo_fd, &pid_info, id, runtime, sandbox_container)) { + ERROR("Failed to add exit monitor to supervisor"); + ret = -1; + goto out; +diff --git a/src/daemon/modules/container/supervisor/supervisor.c b/src/daemon/modules/container/supervisor/supervisor.c +index 1f9a043c..63289283 100644 +--- a/src/daemon/modules/container/supervisor/supervisor.c ++++ b/src/daemon/modules/container/supervisor/supervisor.c +@@ -38,6 +38,9 @@ + #include "container_api.h" + #include "event_type.h" + #include "utils_file.h" ++#ifdef ENABLE_CRI_API_V1 ++#include "sandbox_ops.h" ++#endif + + pthread_mutex_t g_supervisor_lock = PTHREAD_MUTEX_INITIALIZER; + struct epoll_descr g_supervisor_descr; +@@ -47,6 +50,7 @@ struct supervisor_handler_data { + int exit_code; + char *name; + char *runtime; ++ bool is_sandbox_container; + pid_ppid_info_t pid_info; + }; + +@@ -211,6 +215,14 @@ retry: + + (void)isulad_monitor_send_container_event(name, STOPPED, (int)pid, data->exit_code, NULL, NULL); + ++#ifdef ENABLE_CRI_API_V1 ++ if (data->is_sandbox_container) { ++ if (sandbox_on_sandbox_exit(name, data->exit_code) < 0) { ++ ERROR("Failed to handle sandbox %s exit", name); ++ } ++ } ++#endif ++ + supervisor_handler_data_free(data); + + DAEMON_CLEAR_ERRMSG(); +@@ -259,7 +271,7 @@ static int supervisor_exit_cb(int fd, uint32_t events, void *cbdata, struct epol + + /* supervisor add exit monitor */ + int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_info, const char *name, +- const char *runtime) ++ const char *runtime, bool sandbox_container) + { + int ret = 0; + struct supervisor_handler_data *data = NULL; +@@ -285,6 +297,7 @@ int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_inf + data->fd = fd; + data->name = util_strdup_s(name); + data->runtime = util_strdup_s(runtime); ++ data->is_sandbox_container = sandbox_container; + data->pid_info.pid = pid_info->pid; + data->pid_info.start_time = pid_info->start_time; + data->pid_info.ppid = pid_info->ppid; +diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c +index a3606a82..7b34cc7f 100644 +--- a/src/daemon/modules/service/service_container.c ++++ b/src/daemon/modules/service/service_container.c +@@ -275,13 +275,14 @@ static void clean_resources_on_failure(const container_t *cont, const char *engi + return; + } + +-static int do_post_start_on_success(const char *id, const char *runtime, const char *pidfile, int exit_fifo_fd, ++static int do_post_start_on_success(const char *id, const char *runtime, bool sandbox_container, ++ const char *pidfile, int exit_fifo_fd, + const pid_ppid_info_t *pid_info) + { + int ret = 0; + + // exit_fifo_fd was closed in container_supervisor_add_exit_monitor +- if (container_supervisor_add_exit_monitor(exit_fifo_fd, pid_info, id, runtime)) { ++ if (container_supervisor_add_exit_monitor(exit_fifo_fd, pid_info, id, runtime, sandbox_container)) { + ERROR("Failed to add exit monitor to supervisor"); + ret = -1; + } +@@ -749,6 +750,7 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo + oci_runtime_spec *oci_spec = NULL; + rt_create_params_t create_params = { 0 }; + rt_start_params_t start_params = { 0 }; ++ bool sandbox_container; + + nret = snprintf(bundle, sizeof(bundle), "%s/%s", cont->root_path, id); + if (nret < 0 || (size_t)nret >= sizeof(bundle)) { +@@ -897,6 +899,7 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo + if (cont->common_config->sandbox_info != NULL) { + create_params.task_addr = cont->common_config->sandbox_info->task_address; + } ++ sandbox_container = is_sandbox_container(cont->common_config->sandbox_info); + #endif + + if (runtime_create(id, runtime, &create_params) != 0) { +@@ -921,7 +924,7 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo + + ret = runtime_start(id, runtime, &start_params, pid_info); + if (ret == 0) { +- if (do_post_start_on_success(id, runtime, pidfile, exit_fifo_fd, pid_info) != 0) { ++ if (do_post_start_on_success(id, runtime, sandbox_container, pidfile, exit_fifo_fd, pid_info) != 0) { + ERROR("Failed to do post start on runtime start success"); + ret = -1; + goto clean_resources; +diff --git a/src/daemon/sandbox/controller/shim/shim_controller.cc b/src/daemon/sandbox/controller/shim/shim_controller.cc +index 39fcf8ea..593fade9 100644 +--- a/src/daemon/sandbox/controller/shim/shim_controller.cc ++++ b/src/daemon/sandbox/controller/shim/shim_controller.cc +@@ -397,39 +397,8 @@ bool ShimController::Stop(const std::string &sandboxId, uint32_t timeoutSecs, Er + + bool ShimController::Wait(std::shared_ptr cb, const std::string &sandboxId, Errors &error) + { +- std::thread([this, cb, sandboxId]() { +- if (m_cb == nullptr || m_cb->container.wait == nullptr) { +- ERROR("Unimplemented callback"); +- return; +- } +- +- auto requestWrapper = makeUniquePtrCStructWrapper(free_container_wait_request); +- if (requestWrapper == nullptr) { +- ERROR("Out of memory"); +- return; +- } +- auto request = requestWrapper->get(); +- request->id = isula_strdup_s(sandboxId.c_str()); +- request->condition = WAIT_CONDITION_STOPPED; +- container_wait_response *response { nullptr }; +- +- int ret = m_cb->container.wait(request, &response); +- auto responseWrapper = makeUniquePtrCStructWrapper(response, free_container_wait_response); +- +- if (ret != 0) { +- std::string msg = (response != nullptr && response->errmsg != nullptr) ? response->errmsg : "internal"; +- ERROR("Failed to wait sandbox %s: %s", sandboxId.c_str(), msg.c_str()); +- return; +- } +- +- ControllerExitInfo info; +- auto currentTime = std::chrono::high_resolution_clock::now(); +- auto duration = currentTime.time_since_epoch(); +- info.exitedAt = std::chrono::duration_cast(duration).count(); +- info.exitStatus = response->exit_code; +- cb->OnSandboxExit(info); +- }).detach(); +- ++ // ShimController will use sandbox_on_exit callback of supervisor in lower container level ++ // to notify the sandbox exit event + return true; + } + +diff --git a/src/daemon/sandbox/sandbox_ops.cc b/src/daemon/sandbox/sandbox_ops.cc +index 005063c0..b7fb40bf 100644 +--- a/src/daemon/sandbox/sandbox_ops.cc ++++ b/src/daemon/sandbox/sandbox_ops.cc +@@ -18,6 +18,7 @@ + #include + + #include "controller_manager.h" ++#include "sandbox_manager.h" + #include "namespace.h" + #include "utils.h" + +@@ -175,3 +176,25 @@ int sandbox_purge_exec(const container_config_v2_common_config *config, const ch + { + return do_sandbox_purge(config, exec_id); + } ++ ++int sandbox_on_sandbox_exit(const char *sandbox_id, int exit_code) ++{ ++ if (nullptr == sandbox_id) { ++ ERROR("Invalid parameter: sandbox_id"); ++ return -1; ++ } ++ ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(sandbox_id); ++ if (nullptr == sandbox) { ++ ERROR("Sandbox %s not found", sandbox_id); ++ return -1; ++ } ++ ++ sandbox::ControllerExitInfo info; ++ auto currentTime = std::chrono::high_resolution_clock::now(); ++ auto duration = currentTime.time_since_epoch(); ++ info.exitedAt = std::chrono::duration_cast(duration).count(); ++ info.exitStatus = exit_code; ++ sandbox->OnSandboxExit(info); ++ return 0; ++} +diff --git a/src/daemon/sandbox/sandbox_ops.h b/src/daemon/sandbox/sandbox_ops.h +index bef884fb..8189efd6 100644 +--- a/src/daemon/sandbox/sandbox_ops.h ++++ b/src/daemon/sandbox/sandbox_ops.h +@@ -36,6 +36,8 @@ int sandbox_purge_container(const container_config_v2_common_config *config); + + int sandbox_purge_exec(const container_config_v2_common_config *config, const char *exec_id); + ++int sandbox_on_sandbox_exit(const char *sandbox_id, int exit_code); ++ + #ifdef __cplusplus + } + #endif +-- +2.34.1 + diff --git a/0029-refactor-cgroup-module.patch b/0029-refactor-cgroup-module.patch new file mode 100644 index 0000000..f5e853d --- /dev/null +++ b/0029-refactor-cgroup-module.patch @@ -0,0 +1,2538 @@ +From c26604ff3150babae729890c549f2784212073a1 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Wed, 20 Mar 2024 15:53:56 +0800 +Subject: [PATCH 29/43] refactor cgroup module + +Signed-off-by: zhongtao +--- + src/cmd/isulad/main.c | 6 + + src/daemon/common/CMakeLists.txt | 4 + + src/daemon/common/cgroup.c | 625 ---------------- + src/daemon/common/cgroup/CMakeLists.txt | 5 + + src/daemon/common/cgroup/cgroup.c | 136 ++++ + src/daemon/common/cgroup/cgroup.h | 48 ++ + src/daemon/common/cgroup/cgroup_common.c | 131 ++++ + .../{cgroup.h => cgroup/cgroup_common.h} | 62 +- + src/daemon/common/{ => cgroup}/cgroup_v1.c | 699 +++++++++++++++--- + src/daemon/common/cgroup/cgroup_v1.h | 30 + + src/daemon/common/{ => cgroup}/cgroup_v2.c | 162 ++-- + src/daemon/common/cgroup/cgroup_v2.h | 30 + + src/daemon/common/sysinfo.c | 33 +- + src/daemon/common/sysinfo.h | 4 +- + .../v1/v1_cri_pod_sandbox_manager_service.cc | 13 +- + .../cri_pod_sandbox_manager_service.cc | 13 +- + src/daemon/executor/container_cb/execution.c | 13 +- + 17 files changed, 1124 insertions(+), 890 deletions(-) + delete mode 100644 src/daemon/common/cgroup.c + create mode 100644 src/daemon/common/cgroup/CMakeLists.txt + create mode 100644 src/daemon/common/cgroup/cgroup.c + create mode 100644 src/daemon/common/cgroup/cgroup.h + create mode 100644 src/daemon/common/cgroup/cgroup_common.c + rename src/daemon/common/{cgroup.h => cgroup/cgroup_common.h} (60%) + rename src/daemon/common/{ => cgroup}/cgroup_v1.c (52%) + create mode 100644 src/daemon/common/cgroup/cgroup_v1.h + rename src/daemon/common/{ => cgroup}/cgroup_v2.c (88%) + create mode 100644 src/daemon/common/cgroup/cgroup_v2.h + +diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c +index 7c0c072e..9fa87bdb 100644 +--- a/src/cmd/isulad/main.c ++++ b/src/cmd/isulad/main.c +@@ -82,6 +82,7 @@ + #include "network_api.h" + #endif + #include "id_name_manager.h" ++#include "cgroup.h" + + sem_t g_daemon_shutdown_sem; + sem_t g_daemon_wait_shutdown_sem; +@@ -1706,6 +1707,11 @@ static int pre_init_daemon(int argc, char **argv) + goto out; + } + ++ if (cgroup_ops_init() != 0) { ++ ERROR("Failed to init cgroup"); ++ goto out; ++ } ++ + if (server_conf_parse_save(argc, (const char **)argv)) { + ERROR("%s", g_isulad_errmsg ? g_isulad_errmsg : "Failed to parse and save server conf"); + goto out; +diff --git a/src/daemon/common/CMakeLists.txt b/src/daemon/common/CMakeLists.txt +index d634507b..e88578dd 100644 +--- a/src/daemon/common/CMakeLists.txt ++++ b/src/daemon/common/CMakeLists.txt +@@ -9,16 +9,20 @@ if (GRPC_CONNECTOR) + add_subdirectory(cri) + endif() + ++add_subdirectory(cgroup) ++ + set(local_daemon_common_srcs ${daemon_common_top_srcs}) + + set(DAEMON_COMMON_SRCS + ${COMMON_CRI_SRCS} ++ ${COMMON_CGROUP_SRCS} + ${local_daemon_common_srcs} + PARENT_SCOPE + ) + + set(DAEMON_COMMON_INCS + ${COMMON_CRI_INCS} ++ ${COMMON_CGROUP_INCS} + ${CMAKE_CURRENT_SOURCE_DIR} + PARENT_SCOPE + ) +diff --git a/src/daemon/common/cgroup.c b/src/daemon/common/cgroup.c +deleted file mode 100644 +index 3c58f7fa..00000000 +--- a/src/daemon/common/cgroup.c ++++ /dev/null +@@ -1,625 +0,0 @@ +-/****************************************************************************** +- * Copyright (c) Huawei Technologies Co., Ltd. 2017-2023. All rights reserved. +- * iSulad licensed under the Mulan PSL v2. +- * You can use this software according to the terms and conditions of the Mulan PSL v2. +- * You may obtain a copy of Mulan PSL v2 at: +- * http://license.coscl.org.cn/MulanPSL2 +- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +- * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +- * PURPOSE. +- * See the Mulan PSL v2 for more details. +- * Author: zhangxiaoyu +- * Create: 2023-03-29 +- * Description: provide cgroup functions +- ******************************************************************************/ +-#include "cgroup.h" +- +-#include +-#include +-#include +-#include +-#include +-#include +-#include +- +-#include +- +-#include "err_msg.h" +-#include "utils.h" +-#include "utils_array.h" +-#include "sysinfo.h" +- +-#ifndef CGROUP_SUPER_MAGIC +-#define CGROUP_SUPER_MAGIC 0x27e0eb +-#endif +- +-static cgroup_layer_t *new_cgroup_layer(size_t len) +-{ +- cgroup_layer_t *layers = NULL; +- +- if (len == 0) { +- return NULL; +- } +- +- layers = (cgroup_layer_t *)util_common_calloc_s(sizeof(cgroup_layer_t)); +- if (layers == NULL) { +- ERROR("Out of memory"); +- return NULL; +- } +- +- layers->items = (cgroup_layers_item **)util_smart_calloc_s(sizeof(cgroup_layers_item *), len); +- if (layers->items == NULL) { +- ERROR("Out of memory"); +- free(layers); +- return NULL; +- } +- +- layers->len = 0; +- layers->cap = len; +- +- return layers; +-} +- +-static int add_cgroup_layer(cgroup_layer_t *layers, char **clist, char *mountpoint) +-{ +-#define CGROUP_LAYER_MAX_CAPABILITY 1024 +- size_t new_size; +- cgroup_layers_item *newh = NULL; +- cgroup_layers_item **tmp = NULL; +- +- if (layers->len >= CGROUP_LAYER_MAX_CAPABILITY) { +- ERROR("Too many cgroup layers"); +- return -1; +- } +- +- newh = util_common_calloc_s(sizeof(cgroup_layers_item)); +- if (newh == NULL) { +- ERROR("Out of memory"); +- return -1; +- } +- newh->controllers = clist; +- newh->mountpoint = mountpoint; +- +- if (layers->len < layers->cap) { +- goto out; +- } +- +- if (layers->cap > CGROUP_LAYER_MAX_CAPABILITY / 2) { +- new_size = CGROUP_LAYER_MAX_CAPABILITY; +- } else { +- new_size = layers->cap * 2; +- } +- +- if (util_mem_realloc((void **)&tmp, new_size * sizeof(cgroup_layers_item *), +- layers->items, layers->cap * sizeof(cgroup_layers_item *)) != 0) { +- ERROR("Failed to realloc memory"); +- free(newh); +- return -1; +- } +- +- layers->items = tmp; +- tmp = NULL; +- layers->cap = new_size; +- +-out: +- layers->items[layers->len] = newh; +- layers->len++; +- return 0; +-} +- +-void common_free_cgroup_layer(cgroup_layer_t *layers) +-{ +- size_t i; +- +- if (layers == NULL) { +- return; +- } +- +- for (i = 0; i < layers->len && layers->items[i]; i++) { +- free(layers->items[i]->mountpoint); +- layers->items[i]->mountpoint = NULL; +- util_free_array(layers->items[i]->controllers); +- layers->items[i]->controllers = NULL; +- free(layers->items[i]); +- layers->items[i] = NULL; +- } +- +- free(layers->items); +- layers->items = NULL; +- layers->len = 0; +- layers->cap = 0; +- +- free(layers); +-} +- +-static int append_subsystem_to_list(char ***klist, char ***nlist, const char *ptoken) +-{ +- int ret = 0; +- +- if (strncmp(ptoken, "name=", strlen("name=")) == 0) { +- ret = util_array_append(nlist, ptoken); +- if (ret != 0) { +- ERROR("Failed to append string"); +- return -1; +- } +- } else { +- ret = util_array_append(klist, ptoken); +- if (ret != 0) { +- ERROR("Failed to append string"); +- return -1; +- } +- } +- +- return 0; +-} +- +-static int get_cgroup_subsystems(char ***klist, char ***nlist) +-{ +- int ret = 0; +- size_t length = 0; +- FILE *fp = NULL; +- char *pline = NULL; +- +- fp = util_fopen("/proc/self/cgroup", "r"); +- if (fp == NULL) { +- return -1; +- } +- +- while (getline(&pline, &length, fp) != -1) { +- char *pos = NULL; +- char *pos2 = NULL; +- char *ptoken = NULL; +- char *psave = NULL; +- pos = strchr(pline, ':'); +- if (pos == NULL) { +- ERROR("Invalid cgroup entry: must contain at least two colons: %s", pline); +- ret = -1; +- goto out; +- } +- pos++; +- pos2 = strchr(pos, ':'); +- if (pos2 == NULL) { +- ERROR("Invalid cgroup entry: must contain at least two colons: %s", pline); +- ret = -1; +- goto out; +- } +- *pos2 = '\0'; +- +- if ((pos2 - pos) == 0) { +- INFO("Not supported cgroup entry: %s", pline); +- continue; +- } +- +- for (ptoken = strtok_r(pos, ",", &psave); ptoken; ptoken = strtok_r(NULL, ",", &psave)) { +- if (append_subsystem_to_list(klist, nlist, ptoken)) { +- goto out; +- } +- } +- } +- +-out: +- free(pline); +- fclose(fp); +- if (ret != 0) { +- util_free_array(*klist); +- *klist = NULL; +- util_free_array(*nlist); +- *nlist = NULL; +- } +- return ret; +-} +- +-static int append_controller(const char **klist, const char **nlist, char ***clist, const char *entry) +-{ +- int ret = 0; +- char *dup_entry = NULL; +- +- if (util_array_contain(klist, entry) && util_array_contain(nlist, entry)) { +- ERROR("Refusing to use ambiguous controller \"%s\"", entry); +- ERROR("It is both a named and kernel subsystem"); +- return -1; +- } +- +- if (strncmp(entry, "name=", 5) == 0) { +- dup_entry = util_strdup_s(entry); +- } else if (util_array_contain(klist, entry)) { +- dup_entry = util_strdup_s(entry); +- } else { +- dup_entry = util_string_append(entry, "name="); +- } +- if (dup_entry == NULL) { +- ERROR("Out of memory"); +- return -1; +- } +- +- ret = util_array_append(clist, dup_entry); +- if (ret != 0) { +- ERROR("Failed to append array"); +- } +- +- free(dup_entry); +- return ret; +-} +- +-static inline bool is_cgroup_mountpoint(const char *mp) +-{ +- return strncmp(mp, "/sys/fs/cgroup/", strlen("/sys/fs/cgroup/")) == 0; +-} +- +-static char **cgroup_get_controllers(const char **klist, const char **nlist, const char *line) +-{ +- int index; +- char *dup = NULL; +- char *pos2 = NULL; +- char *tok = NULL; +- const char *pos = line; +- char *psave = NULL; +- char *sep = ","; +- char **pret = NULL; +- +- // line example +- // 108 99 0:55 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime - tmpfs tmpfs rw,mode=755 +- for (index = 0; index < 4; index++) { +- pos = strchr(pos, ' '); +- if (pos == NULL) { +- ERROR("Invalid mountinfo format \"%s\"", line); +- return NULL; +- } +- pos++; +- } +- +- if (!is_cgroup_mountpoint(pos)) { +- return NULL; +- } +- +- pos += strlen("/sys/fs/cgroup/"); +- pos2 = strchr(pos, ' '); +- if (pos2 == NULL) { +- ERROR("Invalid mountinfo format \"%s\"", line); +- return NULL; +- } +- +- *pos2 = '\0'; +- dup = util_strdup_s(pos); +- *pos2 = ' '; +- +- for (tok = strtok_r(dup, sep, &psave); tok; tok = strtok_r(NULL, sep, &psave)) { +- if (append_controller(klist, nlist, &pret, tok)) { +- ERROR("Failed to append controller"); +- util_free_array(pret); +- pret = NULL; +- break; +- } +- } +- +- free(dup); +- +- return pret; +-} +- +-int cgroup_get_mountpoint_and_root(char *pline, char **mountpoint, char **root) +-{ +- int index; +- char *posmp = NULL; +- char *posrt = NULL; +- char *pos = pline; +- +- // find root +- // line example +- // 108 99 0:55 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime - tmpfs tmpfs rw,mode=755 +- for (index = 0; index < 3; index++) { +- pos = strchr(pos, ' '); +- if (pos == NULL) { +- return -1; +- } +- pos++; +- } +- posrt = pos; +- +- // find mountpoint +- pos = strchr(pos, ' '); +- if (pos == NULL) { +- return -1; +- } +- +- *pos = '\0'; +- if (root != NULL) { +- *root = util_strdup_s(posrt); +- } +- +- pos++; +- posmp = pos; +- +- if (!is_cgroup_mountpoint(posmp)) { +- return -1; +- } +- +- pos = strchr(pos + strlen("/sys/fs/cgroup/"), ' '); +- if (pos == NULL) { +- return -1; +- } +- *pos = '\0'; +- +- if (mountpoint != NULL) { +- *mountpoint = util_strdup_s(posmp); +- } +- +- return 0; +-} +- +-static bool lists_intersect(const char **controllers, const char **list) +-{ +- int index; +- +- if (controllers == NULL || list == NULL) { +- return false; +- } +- +- for (index = 0; controllers[index]; index++) { +- if (util_array_contain(list, controllers[index])) { +- return true; +- } +- } +- +- return false; +-} +- +-static bool controller_list_is_dup(const cgroup_layer_t *llist, const char **clist) +-{ +- size_t index; +- +- if (llist == NULL) { +- return false; +- } +- +- for (index = 0; index < llist->len && llist->items[index]; index++) { +- if (lists_intersect((const char **)llist->items[index]->controllers, (const char **)clist)) { +- return true; +- } +- } +- +- return false; +-} +- +-cgroup_layer_t *common_cgroup_layers_find(void) +-{ +- int nret; +- int ret = 0; +- FILE *fp = NULL; +- size_t length = 0; +- const size_t cgroup_layer_item_num = 10; +- char *pline = NULL; +- char **klist = NULL; +- char **nlist = NULL; +- cgroup_layer_t *layers = NULL; +- +- layers = new_cgroup_layer(cgroup_layer_item_num); +- if (layers == NULL) { +- ERROR("Failed to new cgroup layer"); +- return NULL; +- } +- +- ret = get_cgroup_subsystems(&klist, &nlist); +- if (ret != 0) { +- ERROR("Failed to retrieve available legacy cgroup controllers\n"); +- goto out; +- } +- +- fp = util_fopen("/proc/self/mountinfo", "r"); +- if (fp == NULL) { +- ERROR("Failed to open \"/proc/self/mountinfo\"\n"); +- ret = -1; +- goto out; +- } +- +- while (getline(&pline, &length, fp) != -1) { +- char *mountpoint = NULL; +- char **clist = NULL; +- int mret; +- +- clist = cgroup_get_controllers((const char **)klist, (const char **)nlist, pline); +- if (clist == NULL) { +- goto list_out; +- } +- +- if (controller_list_is_dup(layers, (const char **)clist)) { +- goto list_out; +- } +- +- mret = cgroup_get_mountpoint_and_root(pline, &mountpoint, NULL); +- if (mret != 0 || mountpoint == NULL) { +- ERROR("Failed parsing mountpoint from \"%s\"\n", pline); +- goto list_out; +- } +- +- nret = add_cgroup_layer(layers, clist, mountpoint); +- if (nret != 0) { +- ERROR("Failed to add hierarchies"); +- goto list_out; +- } +- +- continue; +-list_out: +- util_free_array(clist); +- free(mountpoint); +- } +-out: +- util_free_array(klist); +- util_free_array(nlist); +- if (fp != NULL) { +- fclose(fp); +- } +- free(pline); +- +- if (ret != 0) { +- common_free_cgroup_layer(layers); +- return NULL; +- } +- +- return layers; +-} +- +-char *common_find_cgroup_subsystem_mountpoint(const cgroup_layer_t *layers, const char *subsystem) +-{ +- size_t i; +- +- for (i = 0; i < layers->len && layers->items[i]; i++) { +- char **cit = NULL; +- +- for (cit = layers->items[i]->controllers; cit && *cit; cit++) { +- if (strcmp(*cit, subsystem) == 0) { +- return layers->items[i]->mountpoint; +- } +- } +- } +- return NULL; +-} +- +-/* find cgroup mountpoint and root */ +-int common_find_cgroup_mnt_and_root(const char *subsystem, char **mountpoint, char **root) +-{ +- int ret = 0; +- FILE *fp = NULL; +- size_t length = 0; +- char *pline = NULL; +- +- if (subsystem == NULL) { +- ERROR("Empty subsystem"); +- return -1; +- } +- +- fp = util_fopen("/proc/self/mountinfo", "r"); +- if (fp == NULL) { +- ERROR("Failed to open \"/proc/self/mountinfo\"\n"); +- ret = -1; +- goto free_out; +- } +- +- while (getline(&pline, &length, fp) != -1) { +- char *dup = NULL; +- char *p = NULL; +- char *tok = NULL; +- char *mp = NULL; +- char *rt = NULL; +- char *saveptr = NULL; +- char *sep = ","; +- int mret; +- +- mret = cgroup_get_mountpoint_and_root(pline, &mp, &rt); +- if (mret != 0 || mp == NULL || rt == NULL) { +- goto mp_out; +- } +- +- p = mp; +- p += strlen("/sys/fs/cgroup/"); +- dup = util_strdup_s(p); +- if (dup == NULL) { +- ERROR("Out of memory"); +- free(mp); +- ret = -1; +- goto free_out; +- } +- +- for (tok = strtok_r(dup, sep, &saveptr); tok; tok = strtok_r(NULL, sep, &saveptr)) { +- if (strcmp(tok, subsystem) != 0) { +- continue; +- } +- if (mountpoint != NULL) { +- *mountpoint = mp; +- } else { +- free(mp); +- } +- if (root != NULL) { +- *root = rt; +- } else { +- free(rt); +- } +- free(dup); +- goto free_out; +- } +- free(dup); +-mp_out: +- free(mp); +- free(rt); +- continue; +- } +-free_out: +- if (fp != NULL) { +- fclose(fp); +- } +- free(pline); +- return ret; +-} +- +-int common_get_cgroup_version(void) +-{ +- struct statfs fs = { 0 }; +- +- if (statfs(CGROUP_MOUNTPOINT, &fs) != 0) { +- SYSERROR("failed to statfs %s", CGROUP_MOUNTPOINT); +- return -1; +- } +- +- if (fs.f_type == CGROUP2_SUPER_MAGIC) { +- return CGROUP_VERSION_2; +- } +- +- return CGROUP_VERSION_1; +-} +- +-static int get_value_ull(const char *content, void *result) +-{ +- uint64_t ull_result = 0; +- +- if (util_safe_uint64(content, &ull_result) != 0) { +- ERROR("Failed to convert %s to uint64", content); +- return -1; +- } +- +- *(uint64_t *)result = ull_result; +- return 0; +-} +- +-int get_match_value_ull(const char *content, const char *match, void *result) +-{ +- __isula_auto_free char *llu_string = NULL; +- __isula_auto_free char *match_with_space = NULL; +- __isula_auto_array_t char **lines = NULL; +- char **worker = NULL; +- +- if (match == NULL) { +- return get_value_ull(content, result); +- } +- +- // match full string +- match_with_space = util_string_append(" ", match); +- if (match_with_space == NULL) { +- ERROR("Failed to append string"); +- return -1; +- } +- +- lines = util_string_split(content, '\n'); +- if (lines == NULL) { +- ERROR("Failed to split content %s", content); +- return -1; +- } +- +- for (worker = lines; worker && *worker; worker++) { +- if (util_has_prefix(*worker, match_with_space)) { +- break; +- } +- } +- if (*worker == NULL) { +- ERROR("Cannot find match string %s", match); +- return -1; +- } +- +- llu_string = util_sub_string(*worker, strlen(match_with_space), strlen(*worker) - strlen(match_with_space)); +- if (llu_string == NULL) { +- ERROR("Failed to sub string"); +- return -1; +- } +- llu_string = util_trim_space(llu_string); +- +- return get_value_ull(llu_string, result); +-} +\ No newline at end of file +diff --git a/src/daemon/common/cgroup/CMakeLists.txt b/src/daemon/common/cgroup/CMakeLists.txt +new file mode 100644 +index 00000000..e8c1726c +--- /dev/null ++++ b/src/daemon/common/cgroup/CMakeLists.txt +@@ -0,0 +1,5 @@ ++# get current directory sources files ++aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_common_cgroup_srcs) ++ ++set(COMMON_CGROUP_SRCS ${local_common_cgroup_srcs} PARENT_SCOPE) ++set(COMMON_CGROUP_INCS ${CMAKE_CURRENT_SOURCE_DIR} PARENT_SCOPE) +diff --git a/src/daemon/common/cgroup/cgroup.c b/src/daemon/common/cgroup/cgroup.c +new file mode 100644 +index 00000000..837b514a +--- /dev/null ++++ b/src/daemon/common/cgroup/cgroup.c +@@ -0,0 +1,136 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2017-2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhangxiaoyu ++ * Create: 2023-03-29 ++ * Description: provide cgroup functions ++ ******************************************************************************/ ++#include "cgroup.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++#include "err_msg.h" ++#include "utils.h" ++#include "utils_array.h" ++#include "sysinfo.h" ++#include "cgroup_v1.h" ++#include "cgroup_v2.h" ++#include "path.h" ++ ++#ifndef CGROUP_SUPER_MAGIC ++#define CGROUP_SUPER_MAGIC 0x27e0eb ++#endif ++ ++static cgroup_ops g_cgroup_ops; ++ ++static int get_cgroup_version_for_init(void) ++{ ++ struct statfs fs = { 0 }; ++ ++ if (statfs(CGROUP_MOUNTPOINT, &fs) != 0) { ++ SYSERROR("failed to statfs %s", CGROUP_MOUNTPOINT); ++ return -1; ++ } ++ ++ if (fs.f_type == CGROUP2_SUPER_MAGIC) { ++ return CGROUP_VERSION_2; ++ } ++ ++ return CGROUP_VERSION_1; ++} ++ ++/* connect client ops init */ ++int cgroup_ops_init(void) ++{ ++ (void)memset(&g_cgroup_ops, 0, sizeof(g_cgroup_ops)); ++ int cgroupVersion = get_cgroup_version_for_init(); ++ if (cgroupVersion < 0) { ++ ERROR("Invalid cgroup version"); ++ return -1; ++ } ++ ++ if (cgroupVersion == CGROUP_VERSION_1) { ++ return cgroup_v1_ops_init(&g_cgroup_ops); ++ } else { ++ return cgroup_v2_ops_init(&g_cgroup_ops); ++ } ++} ++ ++int common_get_cgroup_version(void) ++{ ++ if (g_cgroup_ops.get_cgroup_version == NULL) { ++ ERROR("Unimplemented get_cgroup_version ops"); ++ return -1; ++ } ++ ++ return g_cgroup_ops.get_cgroup_version(); ++} ++ ++int common_get_cgroup_info(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, ++ cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, ++ cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, ++ cgroup_files_info_t *filesinfo, bool quiet) ++{ ++ if (g_cgroup_ops.get_cgroup_info == NULL) { ++ ERROR("Unimplemented get_cgroup_info ops"); ++ return -1; ++ } ++ ++ return g_cgroup_ops.get_cgroup_info(meminfo, cpuinfo, hugetlbinfo, blkioinfo, cpusetinfo, pidsinfo, filesinfo, quiet); ++} ++ ++int common_get_cgroup_metrics(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics) ++{ ++ if (g_cgroup_ops.get_cgroup_metrics == NULL) { ++ ERROR("Unimplemented get_cgroup_metrics ops"); ++ return -1; ++ } ++ ++ return g_cgroup_ops.get_cgroup_metrics(cgroup_path, cgroup_metrics); ++} ++ ++int common_get_cgroup_mnt_and_root_path(const char *subsystem, char **mountpoint, char **root) ++{ ++ if (g_cgroup_ops.get_cgroup_mnt_and_root_path == NULL) { ++ ERROR("Unimplemented get_cgroup_mnt_and_root_path ops"); ++ return -1; ++ } ++ ++ return g_cgroup_ops.get_cgroup_mnt_and_root_path(subsystem, mountpoint, root); ++} ++ ++// only for cgroup v1 ++char *common_get_init_cgroup_path(const char *subsystem) ++{ ++ if (g_cgroup_ops.get_init_cgroup_path == NULL) { ++ ERROR("Unimplemented get_init_cgroup_path ops"); ++ return NULL; ++ } ++ ++ return g_cgroup_ops.get_init_cgroup_path(subsystem); ++} ++ ++char *common_get_own_cgroup_path(const char *subsystem) ++{ ++ if (g_cgroup_ops.get_own_cgroup_path == NULL) { ++ ERROR("Unimplemented get_own_cgroup_path ops"); ++ return NULL; ++ } ++ ++ return g_cgroup_ops.get_own_cgroup_path(subsystem); ++} +\ No newline at end of file +diff --git a/src/daemon/common/cgroup/cgroup.h b/src/daemon/common/cgroup/cgroup.h +new file mode 100644 +index 00000000..1efc3ca6 +--- /dev/null ++++ b/src/daemon/common/cgroup/cgroup.h +@@ -0,0 +1,48 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2017-2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhangxiaoyu ++ * Create: 2023-03-29 ++ * Description: provide cgroup definition ++ ******************************************************************************/ ++#ifndef DAEMON_COMMON_CGROUP_H ++#define DAEMON_COMMON_CGROUP_H ++ ++#include ++#include ++#include ++ ++#include ++ ++#include "cgroup_common.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++int cgroup_ops_init(void); ++ ++int common_get_cgroup_version(void); ++int common_get_cgroup_info(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, ++ cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, ++ cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, ++ cgroup_files_info_t *filesinfo, bool quiet); ++int common_get_cgroup_metrics(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics); ++int common_get_cgroup_mnt_and_root_path(const char *subsystem, char **mountpoint, char **root); ++ ++// only for cgroup v1 ++char *common_get_init_cgroup_path(const char *subsystem); ++char *common_get_own_cgroup_path(const char *subsystem); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif // DAEMON_COMMON_CGROUP_H +diff --git a/src/daemon/common/cgroup/cgroup_common.c b/src/daemon/common/cgroup/cgroup_common.c +new file mode 100644 +index 00000000..0fb9405c +--- /dev/null ++++ b/src/daemon/common/cgroup/cgroup_common.c +@@ -0,0 +1,131 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-03-22 ++ * Description: provide cgroup common func definition ++ ******************************************************************************/ ++#include "cgroup_common.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++#include "err_msg.h" ++#include "utils.h" ++#include "utils_array.h" ++#include "sysinfo.h" ++#include "cgroup_v1.h" ++#include "cgroup_v2.h" ++#include "path.h" ++ ++static int get_value_ull(const char *content, void *result) ++{ ++ uint64_t ull_result = 0; ++ ++ if (util_safe_uint64(content, &ull_result) != 0) { ++ ERROR("Failed to convert %s to uint64", content); ++ return -1; ++ } ++ ++ *(uint64_t *)result = ull_result; ++ return 0; ++} ++ ++int get_match_value_ull(const char *content, const char *match, void *result) ++{ ++ __isula_auto_free char *llu_string = NULL; ++ __isula_auto_free char *match_with_space = NULL; ++ __isula_auto_array_t char **lines = NULL; ++ char **worker = NULL; ++ ++ if (match == NULL) { ++ return get_value_ull(content, result); ++ } ++ ++ // match full string ++ match_with_space = util_string_append(" ", match); ++ if (match_with_space == NULL) { ++ ERROR("Failed to append string"); ++ return -1; ++ } ++ ++ lines = util_string_split(content, '\n'); ++ if (lines == NULL) { ++ ERROR("Failed to split content %s", content); ++ return -1; ++ } ++ ++ for (worker = lines; worker && *worker; worker++) { ++ if (util_has_prefix(*worker, match_with_space)) { ++ break; ++ } ++ } ++ if (*worker == NULL) { ++ ERROR("Cannot find match string %s", match); ++ return -1; ++ } ++ ++ llu_string = util_sub_string(*worker, strlen(match_with_space), strlen(*worker) - strlen(match_with_space)); ++ if (llu_string == NULL) { ++ ERROR("Failed to sub string"); ++ return -1; ++ } ++ llu_string = util_trim_space(llu_string); ++ ++ return get_value_ull(llu_string, result); ++} ++ ++int get_cgroup_value_helper(const char *path, struct cgfile_t *cgfile, void *result) ++{ ++ int nret = 0; ++ char file_path[PATH_MAX] = { 0 }; ++ char real_path[PATH_MAX] = { 0 }; ++ char *content = NULL; ++ ++ if (path == NULL || strlen(path) == 0 || result == NULL) { ++ ERROR("%s: Invalid arguments", cgfile->name); ++ return -1; ++ } ++ ++ nret = snprintf(file_path, sizeof(file_path), "%s/%s", path, cgfile->file); ++ if (nret < 0 || (size_t)nret >= sizeof(file_path)) { ++ ERROR("%s: failed to snprintf", cgfile->name); ++ return -1; ++ } ++ ++ if (util_clean_path(file_path, real_path, sizeof(real_path)) == NULL) { ++ ERROR("%s: failed to clean path %s", cgfile->name, file_path); ++ return -1; ++ } ++ ++ content = util_read_content_from_file(real_path); ++ if (content == NULL) { ++ ERROR("%s: failed to read file %s", cgfile->name, real_path); ++ return -1; ++ } ++ ++ util_trim_newline(content); ++ content = util_trim_space(content); ++ ++ nret = cgfile->get_value(content, cgfile->match, result); ++ if (nret != 0) { ++ ERROR("%s: failed to get value", cgfile->name); ++ } ++ ++ free(content); ++ return nret; ++} +\ No newline at end of file +diff --git a/src/daemon/common/cgroup.h b/src/daemon/common/cgroup/cgroup_common.h +similarity index 60% +rename from src/daemon/common/cgroup.h +rename to src/daemon/common/cgroup/cgroup_common.h +index 251e3a3d..2a0935cb 100644 +--- a/src/daemon/common/cgroup.h ++++ b/src/daemon/common/cgroup/cgroup_common.h +@@ -1,5 +1,5 @@ + /****************************************************************************** +- * Copyright (c) Huawei Technologies Co., Ltd. 2017-2023. All rights reserved. ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * iSulad licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: +@@ -8,12 +8,12 @@ + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. +- * Author: zhangxiaoyu +- * Create: 2023-03-29 +- * Description: provide cgroup definition ++ * Author: zhongtao ++ * Create: 2024-03-22 ++ * Description: provide cgroup common func definition + ******************************************************************************/ +-#ifndef DAEMON_COMMON_CGROUP_H +-#define DAEMON_COMMON_CGROUP_H ++#ifndef DAEMON_COMMON_CGROUP_COMMON_H ++#define DAEMON_COMMON_CGROUP_COMMON_H + + #include + #include +@@ -40,10 +40,6 @@ struct cgfile_t { + + int get_match_value_ull(const char *content, const char *match, void *result); + +-int common_get_cgroup_version(void); +- +-int common_find_cgroup_mnt_and_root(const char *subsystem, char **mountpoint, char **root); +- + static inline void common_cgroup_do_log(bool quiet, bool do_log, const char *msg) + { + if (!quiet && do_log) { +@@ -51,23 +47,7 @@ static inline void common_cgroup_do_log(bool quiet, bool do_log, const char *msg + } + } + +-typedef struct { +- char **controllers; +- char *mountpoint; +-} cgroup_layers_item; +- +-typedef struct { +- cgroup_layers_item **items; +- size_t len; +- size_t cap; +-} cgroup_layer_t; +- +-char *common_find_cgroup_subsystem_mountpoint(const cgroup_layer_t *layers, const char *subsystem); +- +-cgroup_layer_t *common_cgroup_layers_find(void); +- +-void common_free_cgroup_layer(cgroup_layer_t *layers); +- ++int get_cgroup_value_helper(const char *path, struct cgfile_t *cgfile, void *result); + + typedef struct { + bool limit; +@@ -113,17 +93,6 @@ typedef struct { + bool fileslimit; + } cgroup_files_info_t; + +-int common_get_cgroup_info_v1(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, +- cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, +- cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, +- cgroup_files_info_t *filesinfo, bool quiet); +- +-int common_get_cgroup_info_v2(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, +- cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, +- cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, +- cgroup_files_info_t *filesinfo, bool quiet); +- +- + typedef struct { + uint64_t cpu_use_nanos; + } cgroup_cpu_metrics_t; +@@ -147,15 +116,22 @@ typedef struct { + cgroup_pids_metrics_t cgpids_metrics; + } cgroup_metrics_t; + +-int common_get_cgroup_v1_metrics(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics); +-int common_get_cgroup_v2_metrics(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics); ++typedef struct { ++ int (*get_cgroup_version)(void); ++ int (*get_cgroup_info)(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, ++ cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, ++ cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, ++ cgroup_files_info_t *filesinfo, bool quiet); ++ int (*get_cgroup_metrics)(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics); + +-char *common_get_init_cgroup(const char *subsystem); ++ int (*get_cgroup_mnt_and_root_path)(const char *subsystem, char **mountpoint, char **root); + +-char *common_get_own_cgroup(const char *subsystem); ++ char *(*get_init_cgroup_path)(const char *subsystem); ++ char *(*get_own_cgroup_path)(const char *subsystem); ++} cgroup_ops; + + #ifdef __cplusplus + } + #endif + +-#endif // DAEMON_COMMON_CGROUP_H ++#endif // DAEMON_COMMON_CGROUP_COMMON_H +diff --git a/src/daemon/common/cgroup_v1.c b/src/daemon/common/cgroup/cgroup_v1.c +similarity index 52% +rename from src/daemon/common/cgroup_v1.c +rename to src/daemon/common/cgroup/cgroup_v1.c +index e34100bc..51cf7512 100644 +--- a/src/daemon/common/cgroup_v1.c ++++ b/src/daemon/common/cgroup/cgroup_v1.c +@@ -18,13 +18,31 @@ + #include + + #include "utils.h" +-#include "path.h" + #include "sysinfo.h" ++#include "err_msg.h" + + #define CGROUP_HUGETLB_LIMIT "hugetlb.%s.limit_in_bytes" ++#define CGROUP_MOUNT_PATH_PREFIX "/sys/fs/cgroup/" + +-static int get_value_ll(const char *content, const char *match, void *result); +-static int get_value_string(const char *content, const char *match, void *result); ++ ++static int get_value_ll(const char *content, const char *match, void *result) ++{ ++ long long ll_result = 0; ++ ++ if (util_safe_llong(content, &ll_result) != 0) { ++ ERROR("Failed to convert %s to long long", content); ++ return -1; ++ } ++ ++ *(int64_t *)result = (int64_t)ll_result; ++ return 0; ++} ++ ++static int get_value_string(const char *content, const char *match, void *result) ++{ ++ *(char **)result = util_strdup_s(content); ++ return 0; ++} + + typedef enum { + // CPU subsystem +@@ -90,105 +108,579 @@ static struct cgfile_t g_cgroup_v1_files[] = { + [PIDS_CURRENT] = {"pids_current", "pids.current", NULL, get_match_value_ull}, + }; + +-static int get_value_ll(const char *content, const char *match, void *result) ++typedef struct { ++ char **controllers; ++ char *mountpoint; ++} cgroup_layers_item; ++ ++typedef struct { ++ cgroup_layers_item **items; ++ size_t len; ++ size_t cap; ++} cgroup_layer_t; ++ ++static char *common_find_cgroup_subsystem_mountpoint(const cgroup_layer_t *layers, const char *subsystem) + { +- long long ll_result = 0; ++ size_t i; + +- if (util_safe_llong(content, &ll_result) != 0) { +- ERROR("Failed to convert %s to long long", content); ++ for (i = 0; i < layers->len && layers->items[i]; i++) { ++ char **cit = NULL; ++ ++ for (cit = layers->items[i]->controllers; cit && *cit; cit++) { ++ if (strcmp(*cit, subsystem) == 0) { ++ return layers->items[i]->mountpoint; ++ } ++ } ++ } ++ return NULL; ++} ++ ++ ++static cgroup_layer_t *new_cgroup_layer(size_t len) ++{ ++ cgroup_layer_t *layers = NULL; ++ ++ if (len == 0) { ++ return NULL; ++ } ++ ++ layers = (cgroup_layer_t *)util_common_calloc_s(sizeof(cgroup_layer_t)); ++ if (layers == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ layers->items = (cgroup_layers_item **)util_smart_calloc_s(sizeof(cgroup_layers_item *), len); ++ if (layers->items == NULL) { ++ ERROR("Out of memory"); ++ free(layers); ++ return NULL; ++ } ++ ++ layers->len = 0; ++ layers->cap = len; ++ ++ return layers; ++} ++ ++static int add_cgroup_layer(cgroup_layer_t *layers, char **clist, char *mountpoint) ++{ ++#define CGROUP_LAYER_MAX_CAPABILITY 1024 ++ size_t new_size; ++ cgroup_layers_item *newh = NULL; ++ cgroup_layers_item **tmp = NULL; ++ ++ if (layers->len >= CGROUP_LAYER_MAX_CAPABILITY) { ++ ERROR("Too many cgroup layers"); + return -1; + } + +- *(int64_t *)result = (int64_t)ll_result; ++ newh = util_common_calloc_s(sizeof(cgroup_layers_item)); ++ if (newh == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ newh->controllers = clist; ++ newh->mountpoint = mountpoint; ++ ++ if (layers->len < layers->cap) { ++ goto out; ++ } ++ ++ if (layers->cap > CGROUP_LAYER_MAX_CAPABILITY / 2) { ++ new_size = CGROUP_LAYER_MAX_CAPABILITY; ++ } else { ++ new_size = layers->cap * 2; ++ } ++ ++ if (util_mem_realloc((void **)&tmp, new_size * sizeof(cgroup_layers_item *), ++ layers->items, layers->cap * sizeof(cgroup_layers_item *)) != 0) { ++ ERROR("Failed to realloc memory"); ++ free(newh); ++ return -1; ++ } ++ ++ layers->items = tmp; ++ tmp = NULL; ++ layers->cap = new_size; ++ ++out: ++ layers->items[layers->len] = newh; ++ layers->len++; + return 0; + } + +-static int get_value_string(const char *content, const char *match, void *result) ++static void common_free_cgroup_layer(cgroup_layer_t *layers) + { +- *(char **)result = util_strdup_s(content); ++ size_t i; ++ ++ if (layers == NULL) { ++ return; ++ } ++ ++ for (i = 0; i < layers->len && layers->items[i]; i++) { ++ free(layers->items[i]->mountpoint); ++ layers->items[i]->mountpoint = NULL; ++ util_free_array(layers->items[i]->controllers); ++ layers->items[i]->controllers = NULL; ++ free(layers->items[i]); ++ layers->items[i] = NULL; ++ } ++ ++ free(layers->items); ++ layers->items = NULL; ++ layers->len = 0; ++ layers->cap = 0; ++ ++ free(layers); ++} ++ ++static int append_subsystem_to_list(char ***klist, char ***nlist, const char *ptoken) ++{ ++ int ret = 0; ++ ++ if (strncmp(ptoken, "name=", strlen("name=")) == 0) { ++ ret = util_array_append(nlist, ptoken); ++ if (ret != 0) { ++ ERROR("Failed to append string"); ++ return -1; ++ } ++ } else { ++ ret = util_array_append(klist, ptoken); ++ if (ret != 0) { ++ ERROR("Failed to append string"); ++ return -1; ++ } ++ } ++ + return 0; + } + +-static bool check_cgroup_v1_helper(const char *mountpoint, const cgroup_v1_files_index index, const bool quiet) ++static int get_cgroup_subsystems(char ***klist, char ***nlist) + { +- int nret = 0; +- char path[PATH_MAX] = { 0 }; ++ int ret = 0; ++ size_t length = 0; ++ FILE *fp = NULL; ++ char *pline = NULL; + +- if (index >= CGROUP_V1_FILES_INDEX_MAXS) { +- ERROR("Index out of range"); +- return false; ++ fp = util_fopen("/proc/self/cgroup", "r"); ++ if (fp == NULL) { ++ return -1; + } + +- if (mountpoint == NULL) { +- ERROR("%s: invalid arguments", g_cgroup_v1_files[index].name); +- return false; ++ while (getline(&pline, &length, fp) != -1) { ++ char *pos = NULL; ++ char *pos2 = NULL; ++ char *ptoken = NULL; ++ char *psave = NULL; ++ pos = strchr(pline, ':'); ++ if (pos == NULL) { ++ ERROR("Invalid cgroup entry: must contain at least two colons: %s", pline); ++ ret = -1; ++ goto out; ++ } ++ pos++; ++ pos2 = strchr(pos, ':'); ++ if (pos2 == NULL) { ++ ERROR("Invalid cgroup entry: must contain at least two colons: %s", pline); ++ ret = -1; ++ goto out; ++ } ++ *pos2 = '\0'; ++ ++ if ((pos2 - pos) == 0) { ++ INFO("Not supported cgroup entry: %s", pline); ++ continue; ++ } ++ ++ for (ptoken = strtok_r(pos, ",", &psave); ptoken; ptoken = strtok_r(NULL, ",", &psave)) { ++ if (append_subsystem_to_list(klist, nlist, ptoken)) { ++ goto out; ++ } ++ } + } + +- nret = snprintf(path, sizeof(path), "%s/%s", mountpoint, g_cgroup_v1_files[index].file); +- if (nret < 0 || (size_t)nret >= sizeof(path)) { +- ERROR("%s: failed to snprintf", g_cgroup_v1_files[index].name); +- return false; ++out: ++ free(pline); ++ fclose(fp); ++ if (ret != 0) { ++ util_free_array(*klist); ++ *klist = NULL; ++ util_free_array(*nlist); ++ *nlist = NULL; + } ++ return ret; ++} + +- if (util_file_exists(path)) { +- return true; ++static int append_controller(const char **klist, const char **nlist, char ***clist, const char *entry) ++{ ++ int ret = 0; ++ char *dup_entry = NULL; ++ ++ if (util_array_contain(klist, entry) && util_array_contain(nlist, entry)) { ++ ERROR("Refusing to use ambiguous controller \"%s\"", entry); ++ ERROR("It is both a named and kernel subsystem"); ++ return -1; + } + +- if (!quiet) { +- WARN("Your kernel does not support cgroup %s", g_cgroup_v1_files[index].name); ++ if (strncmp(entry, "name=", 5) == 0) { ++ dup_entry = util_strdup_s(entry); ++ } else if (util_array_contain(klist, entry)) { ++ dup_entry = util_strdup_s(entry); ++ } else { ++ dup_entry = util_string_append(entry, "name="); ++ } ++ if (dup_entry == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ ++ ret = util_array_append(clist, dup_entry); ++ if (ret != 0) { ++ ERROR("Failed to append array"); ++ } ++ ++ free(dup_entry); ++ return ret; ++} ++ ++static inline bool is_cgroup_mountpoint(const char *mp) ++{ ++ return strncmp(mp, CGROUP_MOUNT_PATH_PREFIX, strlen(CGROUP_MOUNT_PATH_PREFIX)) == 0; ++} ++ ++static char **cgroup_get_controllers(const char **klist, const char **nlist, const char *line) ++{ ++ int index; ++ char *dup = NULL; ++ char *pos2 = NULL; ++ char *tok = NULL; ++ const char *pos = line; ++ char *psave = NULL; ++ char *sep = ","; ++ char **pret = NULL; ++ ++ // line example ++ // 108 99 0:55 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime - tmpfs tmpfs rw,mode=755 ++ for (index = 0; index < 4; index++) { ++ pos = strchr(pos, ' '); ++ if (pos == NULL) { ++ ERROR("Invalid mountinfo format \"%s\"", line); ++ return NULL; ++ } ++ pos++; ++ } ++ ++ if (!is_cgroup_mountpoint(pos)) { ++ return NULL; ++ } ++ ++ pos += strlen(CGROUP_MOUNT_PATH_PREFIX); ++ pos2 = strchr(pos, ' '); ++ if (pos2 == NULL) { ++ ERROR("Invalid mountinfo format \"%s\"", line); ++ return NULL; ++ } ++ ++ *pos2 = '\0'; ++ dup = util_strdup_s(pos); ++ *pos2 = ' '; ++ ++ for (tok = strtok_r(dup, sep, &psave); tok; tok = strtok_r(NULL, sep, &psave)) { ++ if (append_controller(klist, nlist, &pret, tok)) { ++ ERROR("Failed to append controller"); ++ util_free_array(pret); ++ pret = NULL; ++ break; ++ } ++ } ++ ++ free(dup); ++ ++ return pret; ++} ++ ++static bool lists_intersect(const char **controllers, const char **list) ++{ ++ int index; ++ ++ if (controllers == NULL || list == NULL) { ++ return false; ++ } ++ ++ for (index = 0; controllers[index]; index++) { ++ if (util_array_contain(list, controllers[index])) { ++ return true; ++ } + } + + return false; + } + +-static int get_cgroup_v1_value_helper(const char *path, const cgroup_v1_files_index index, +- void *result) ++static bool controller_list_is_dup(const cgroup_layer_t *llist, const char **clist) + { +- int nret = 0; +- char file_path[PATH_MAX] = { 0 }; +- char real_path[PATH_MAX] = { 0 }; +- char *content = NULL; ++ size_t index; + +- if (index >= CGROUP_V1_FILES_INDEX_MAXS) { +- ERROR("Index out of range"); ++ if (llist == NULL) { + return false; + } + +- if (path == NULL || strlen(path) == 0 || result == NULL) { +- ERROR("%s: Invalid arguments", g_cgroup_v1_files[index].name); ++ for (index = 0; index < llist->len && llist->items[index]; index++) { ++ if (lists_intersect((const char **)llist->items[index]->controllers, (const char **)clist)) { ++ return true; ++ } ++ } ++ ++ return false; ++} ++ ++static int cgroup_get_mountpoint_and_root(char *pline, char **mountpoint, char **root) ++{ ++ int index; ++ char *posmp = NULL; ++ char *posrt = NULL; ++ char *pos = pline; ++ ++ // find root ++ // line example ++ // 108 99 0:55 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime - tmpfs tmpfs rw,mode=755 ++ for (index = 0; index < 3; index++) { ++ pos = strchr(pos, ' '); ++ if (pos == NULL) { ++ return -1; ++ } ++ pos++; ++ } ++ posrt = pos; ++ ++ // find mountpoint ++ pos = strchr(pos, ' '); ++ if (pos == NULL) { + return -1; + } + +- nret = snprintf(file_path, sizeof(file_path), "%s/%s", path, g_cgroup_v1_files[index].file); +- if (nret < 0 || (size_t)nret >= sizeof(file_path)) { +- ERROR("%s: failed to snprintf", g_cgroup_v1_files[index].name); ++ *pos = '\0'; ++ if (root != NULL) { ++ *root = util_strdup_s(posrt); ++ } ++ ++ pos++; ++ posmp = pos; ++ ++ if (!is_cgroup_mountpoint(posmp)) { + return -1; + } + +- if (util_clean_path(file_path, real_path, sizeof(real_path)) == NULL) { +- ERROR("%s: failed to clean path %s", g_cgroup_v1_files[index].name, file_path); ++ pos = strchr(pos + strlen(CGROUP_MOUNT_PATH_PREFIX), ' '); ++ if (pos == NULL) { + return -1; + } ++ *pos = '\0'; ++ ++ if (mountpoint != NULL) { ++ *mountpoint = util_strdup_s(posmp); ++ } ++ ++ return 0; ++} ++ ++/* find cgroup mountpoint and root */ ++static int get_cgroup_mnt_and_root_path_v1(const char *subsystem, char **mountpoint, char **root) ++{ ++ int ret = 0; ++ FILE *fp = NULL; ++ size_t length = 0; ++ char *pline = NULL; + +- content = util_read_content_from_file(real_path); +- if (content == NULL) { +- ERROR("%s: failed to read file %s", g_cgroup_v1_files[index].name, real_path); ++ if (subsystem == NULL) { ++ ERROR("Empty subsystem"); + return -1; + } + +- util_trim_newline(content); +- content = util_trim_space(content); ++ fp = util_fopen("/proc/self/mountinfo", "r"); ++ if (fp == NULL) { ++ ERROR("Failed to open \"/proc/self/mountinfo\"\n"); ++ ret = -1; ++ goto free_out; ++ } ++ ++ while (getline(&pline, &length, fp) != -1) { ++ char *dup = NULL; ++ char *p = NULL; ++ char *tok = NULL; ++ char *mp = NULL; ++ char *rt = NULL; ++ char *saveptr = NULL; ++ char *sep = ","; ++ int mret; ++ ++ mret = cgroup_get_mountpoint_and_root(pline, &mp, &rt); ++ if (mret != 0 || mp == NULL || rt == NULL) { ++ goto mp_out; ++ } ++ ++ p = mp; ++ p += strlen(CGROUP_MOUNT_PATH_PREFIX); ++ dup = util_strdup_s(p); ++ if (dup == NULL) { ++ ERROR("Out of memory"); ++ free(mp); ++ ret = -1; ++ goto free_out; ++ } ++ ++ for (tok = strtok_r(dup, sep, &saveptr); tok; tok = strtok_r(NULL, sep, &saveptr)) { ++ if (strcmp(tok, subsystem) != 0) { ++ continue; ++ } ++ if (mountpoint != NULL) { ++ *mountpoint = mp; ++ } else { ++ free(mp); ++ } ++ if (root != NULL) { ++ *root = rt; ++ } else { ++ free(rt); ++ } ++ free(dup); ++ goto free_out; ++ } ++ free(dup); ++mp_out: ++ free(mp); ++ free(rt); ++ continue; ++ } ++free_out: ++ if (fp != NULL) { ++ fclose(fp); ++ } ++ free(pline); ++ return ret; ++} ++ ++static cgroup_layer_t *common_cgroup_layers_find(void) ++{ ++ int nret; ++ int ret = 0; ++ FILE *fp = NULL; ++ size_t length = 0; ++ const size_t cgroup_layer_item_num = 10; ++ char *pline = NULL; ++ char **klist = NULL; ++ char **nlist = NULL; ++ cgroup_layer_t *layers = NULL; ++ ++ layers = new_cgroup_layer(cgroup_layer_item_num); ++ if (layers == NULL) { ++ ERROR("Failed to new cgroup layer"); ++ return NULL; ++ } ++ ++ ret = get_cgroup_subsystems(&klist, &nlist); ++ if (ret != 0) { ++ ERROR("Failed to retrieve available legacy cgroup controllers\n"); ++ goto out; ++ } ++ ++ fp = util_fopen("/proc/self/mountinfo", "r"); ++ if (fp == NULL) { ++ ERROR("Failed to open \"/proc/self/mountinfo\"\n"); ++ ret = -1; ++ goto out; ++ } ++ ++ while (getline(&pline, &length, fp) != -1) { ++ char *mountpoint = NULL; ++ char **clist = NULL; ++ int mret; ++ ++ clist = cgroup_get_controllers((const char **)klist, (const char **)nlist, pline); ++ if (clist == NULL) { ++ goto list_out; ++ } ++ ++ if (controller_list_is_dup(layers, (const char **)clist)) { ++ goto list_out; ++ } ++ ++ mret = cgroup_get_mountpoint_and_root(pline, &mountpoint, NULL); ++ if (mret != 0 || mountpoint == NULL) { ++ ERROR("Failed parsing mountpoint from \"%s\"\n", pline); ++ goto list_out; ++ } ++ ++ nret = add_cgroup_layer(layers, clist, mountpoint); ++ if (nret != 0) { ++ ERROR("Failed to add hierarchies"); ++ goto list_out; ++ } ++ ++ continue; ++list_out: ++ util_free_array(clist); ++ free(mountpoint); ++ } ++out: ++ util_free_array(klist); ++ util_free_array(nlist); ++ if (fp != NULL) { ++ fclose(fp); ++ } ++ free(pline); ++ ++ if (ret != 0) { ++ common_free_cgroup_layer(layers); ++ return NULL; ++ } ++ ++ return layers; ++} ++ ++static int get_cgroup_v1_value_helper(const char *path, const cgroup_v1_files_index index, void *result) ++{ ++ if (index >= CGROUP_V1_FILES_INDEX_MAXS) { ++ ERROR("Index out of range"); ++ return false; ++ } ++ ++ return get_cgroup_value_helper(path, &g_cgroup_v1_files[index], result); ++} ++ ++static bool check_cgroup_v1_file_exists(const char *mountpoint, const cgroup_v1_files_index index, const bool quiet) ++{ ++ int nret = 0; ++ char path[PATH_MAX] = { 0 }; ++ ++ if (index >= CGROUP_V1_FILES_INDEX_MAXS) { ++ ERROR("Index out of range"); ++ return false; ++ } ++ ++ if (mountpoint == NULL) { ++ ERROR("%s: invalid arguments", g_cgroup_v1_files[index].name); ++ return false; ++ } ++ ++ nret = snprintf(path, sizeof(path), "%s/%s", mountpoint, g_cgroup_v1_files[index].file); ++ if (nret < 0 || (size_t)nret >= sizeof(path)) { ++ ERROR("%s: failed to snprintf", g_cgroup_v1_files[index].name); ++ return false; ++ } + +- nret = g_cgroup_v1_files[index].get_value(content, g_cgroup_v1_files[index].match, result); +- if (nret != 0) { +- ERROR("%s: failed to get value", g_cgroup_v1_files[index].name); ++ if (util_file_exists(path)) { ++ return true; + } + +- free(content); +- return nret; ++ if (!quiet) { ++ WARN("Your kernel does not support cgroup %s", g_cgroup_v1_files[index].name); ++ } ++ ++ return false; + } + +-static void check_cgroup_v1_cpu(const cgroup_layer_t *layers, const bool quiet, cgroup_cpu_info_t *cpuinfo) ++static void get_cgroup_v1_cpu_info(const cgroup_layer_t *layers, const bool quiet, cgroup_cpu_info_t *cpuinfo) + { + char *mountpoint = NULL; + +@@ -198,14 +690,14 @@ static void check_cgroup_v1_cpu(const cgroup_layer_t *layers, const bool quiet, + return; + } + +- cpuinfo->cpu_rt_period = check_cgroup_v1_helper(mountpoint, CPU_RT_PERIOD, quiet); +- cpuinfo->cpu_rt_runtime = check_cgroup_v1_helper(mountpoint, CPU_RT_RUNTIME, quiet); +- cpuinfo->cpu_shares = check_cgroup_v1_helper(mountpoint, CPU_SHARES, quiet); +- cpuinfo->cpu_cfs_period = check_cgroup_v1_helper(mountpoint, CPU_CFS_PERIOD, quiet); +- cpuinfo->cpu_cfs_quota = check_cgroup_v1_helper(mountpoint, CPU_CFS_QUOTA, quiet); ++ cpuinfo->cpu_rt_period = check_cgroup_v1_file_exists(mountpoint, CPU_RT_PERIOD, quiet); ++ cpuinfo->cpu_rt_runtime = check_cgroup_v1_file_exists(mountpoint, CPU_RT_RUNTIME, quiet); ++ cpuinfo->cpu_shares = check_cgroup_v1_file_exists(mountpoint, CPU_SHARES, quiet); ++ cpuinfo->cpu_cfs_period = check_cgroup_v1_file_exists(mountpoint, CPU_CFS_PERIOD, quiet); ++ cpuinfo->cpu_cfs_quota = check_cgroup_v1_file_exists(mountpoint, CPU_CFS_QUOTA, quiet); + } + +-static void check_cgroup_v1_cpuset(const cgroup_layer_t *layers, const bool quiet, cgroup_cpuset_info_t *cpusetinfo) ++static void get_cgroup_v1_cpuset_info(const cgroup_layer_t *layers, const bool quiet, cgroup_cpuset_info_t *cpusetinfo) + { + char *mountpoint = NULL; + +@@ -215,10 +707,10 @@ static void check_cgroup_v1_cpuset(const cgroup_layer_t *layers, const bool quie + return; + } + +- if (!check_cgroup_v1_helper(mountpoint, CPUSET_CPUS, quiet)) { ++ if (!check_cgroup_v1_file_exists(mountpoint, CPUSET_CPUS, quiet)) { + return; + } +- if (!check_cgroup_v1_helper(mountpoint, CPUSET_MEMS, quiet)) { ++ if (!check_cgroup_v1_file_exists(mountpoint, CPUSET_MEMS, quiet)) { + return; + } + +@@ -238,7 +730,7 @@ static void check_cgroup_v1_cpuset(const cgroup_layer_t *layers, const bool quie + cpusetinfo->cpuset = true; + } + +-static void check_cgroup_v1_mem(const cgroup_layer_t *layers, const bool quiet, cgroup_mem_info_t *meminfo) ++static void get_cgroup_v1_mem_info(const cgroup_layer_t *layers, const bool quiet, cgroup_mem_info_t *meminfo) + { + char *mountpoint = NULL; + +@@ -248,15 +740,15 @@ static void check_cgroup_v1_mem(const cgroup_layer_t *layers, const bool quiet, + return; + } + +- meminfo->limit = check_cgroup_v1_helper(mountpoint, MEMORY_LIMIT, quiet); +- meminfo->swap = check_cgroup_v1_helper(mountpoint, MEMORY_SW_LIMIT, quiet); +- meminfo->reservation = check_cgroup_v1_helper(mountpoint, MEMORY_SOFT_LIMIT, quiet); +- meminfo->oomkilldisable = check_cgroup_v1_helper(mountpoint, MEMORY_OOM_CONTROL, quiet); +- meminfo->swappiness = check_cgroup_v1_helper(mountpoint, MEMORY_SWAPPINESS, quiet); +- meminfo->kernel = check_cgroup_v1_helper(mountpoint, MEMORY_KMEM_LIMIT, quiet); ++ meminfo->limit = check_cgroup_v1_file_exists(mountpoint, MEMORY_LIMIT, quiet); ++ meminfo->swap = check_cgroup_v1_file_exists(mountpoint, MEMORY_SW_LIMIT, quiet); ++ meminfo->reservation = check_cgroup_v1_file_exists(mountpoint, MEMORY_SOFT_LIMIT, quiet); ++ meminfo->oomkilldisable = check_cgroup_v1_file_exists(mountpoint, MEMORY_OOM_CONTROL, quiet); ++ meminfo->swappiness = check_cgroup_v1_file_exists(mountpoint, MEMORY_SWAPPINESS, quiet); ++ meminfo->kernel = check_cgroup_v1_file_exists(mountpoint, MEMORY_KMEM_LIMIT, quiet); + } + +-static void check_cgroup_v1_blkio(const cgroup_layer_t *layers, const bool quiet, cgroup_blkio_info_t *blkioinfo) ++static void get_cgroup_v1_blkio_info(const cgroup_layer_t *layers, const bool quiet, cgroup_blkio_info_t *blkioinfo) + { + char *mountpoint = NULL; + +@@ -266,15 +758,15 @@ static void check_cgroup_v1_blkio(const cgroup_layer_t *layers, const bool quiet + return; + } + +- blkioinfo->blkio_weight = check_cgroup_v1_helper(mountpoint, BLKIO_WEIGTH, quiet); +- blkioinfo->blkio_weight_device = check_cgroup_v1_helper(mountpoint, BLKIO_WEIGTH_DEVICE, quiet); +- blkioinfo->blkio_read_bps_device = check_cgroup_v1_helper(mountpoint, BLKIO_READ_BPS, quiet); +- blkioinfo->blkio_write_bps_device = check_cgroup_v1_helper(mountpoint, BLKIO_WRITE_BPS, quiet); +- blkioinfo->blkio_read_iops_device = check_cgroup_v1_helper(mountpoint, BLKIO_READ_IOPS, quiet); +- blkioinfo->blkio_write_iops_device = check_cgroup_v1_helper(mountpoint, BLKIO_WRITE_IOPS, quiet); ++ blkioinfo->blkio_weight = check_cgroup_v1_file_exists(mountpoint, BLKIO_WEIGTH, quiet); ++ blkioinfo->blkio_weight_device = check_cgroup_v1_file_exists(mountpoint, BLKIO_WEIGTH_DEVICE, quiet); ++ blkioinfo->blkio_read_bps_device = check_cgroup_v1_file_exists(mountpoint, BLKIO_READ_BPS, quiet); ++ blkioinfo->blkio_write_bps_device = check_cgroup_v1_file_exists(mountpoint, BLKIO_WRITE_BPS, quiet); ++ blkioinfo->blkio_read_iops_device = check_cgroup_v1_file_exists(mountpoint, BLKIO_READ_IOPS, quiet); ++ blkioinfo->blkio_write_iops_device = check_cgroup_v1_file_exists(mountpoint, BLKIO_WRITE_IOPS, quiet); + } + +-static void check_cgroup_v1_hugetlb(const cgroup_layer_t *layers, const bool quiet, cgroup_hugetlb_info_t *hugetlbinfo) ++static void get_cgroup_v1_hugetlb_info(const cgroup_layer_t *layers, const bool quiet, cgroup_hugetlb_info_t *hugetlbinfo) + { + int nret; + char *mountpoint = NULL; +@@ -306,7 +798,7 @@ free_out: + free(defaultpagesize); + } + +-static void check_cgroup_v1_pids(const cgroup_layer_t *layers, const bool quiet, cgroup_pids_info_t *pidsinfo) ++static void get_cgroup_v1_pids_info(const cgroup_layer_t *layers, const bool quiet, cgroup_pids_info_t *pidsinfo) + { + char *mountpoint = NULL; + +@@ -319,7 +811,7 @@ static void check_cgroup_v1_pids(const cgroup_layer_t *layers, const bool quiet, + pidsinfo->pidslimit = true; + } + +-static void check_cgroup_v1_files(const cgroup_layer_t *layers, const bool quiet, cgroup_files_info_t *filesinfo) ++static void get_cgroup_v1_files_info(const cgroup_layer_t *layers, const bool quiet, cgroup_files_info_t *filesinfo) + { + char *mountpoint = NULL; + +@@ -332,7 +824,7 @@ static void check_cgroup_v1_files(const cgroup_layer_t *layers, const bool quiet + filesinfo->fileslimit = true; + } + +-int common_get_cgroup_info_v1(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, ++static int get_cgroup_info_v1(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, + cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, + cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, + cgroup_files_info_t *filesinfo, bool quiet) +@@ -345,13 +837,13 @@ int common_get_cgroup_info_v1(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpu + return -1; + } + +- check_cgroup_v1_cpu(layers, quiet, cpuinfo); +- check_cgroup_v1_cpuset(layers, quiet, cpusetinfo); +- check_cgroup_v1_mem(layers, quiet, meminfo); +- check_cgroup_v1_blkio(layers, quiet, blkioinfo); +- check_cgroup_v1_hugetlb(layers, quiet, hugetlbinfo); +- check_cgroup_v1_pids(layers, quiet, pidsinfo); +- check_cgroup_v1_files(layers, quiet, filesinfo); ++ get_cgroup_v1_cpu_info(layers, quiet, cpuinfo); ++ get_cgroup_v1_cpuset_info(layers, quiet, cpusetinfo); ++ get_cgroup_v1_mem_info(layers, quiet, meminfo); ++ get_cgroup_v1_blkio_info(layers, quiet, blkioinfo); ++ get_cgroup_v1_hugetlb_info(layers, quiet, hugetlbinfo); ++ get_cgroup_v1_pids_info(layers, quiet, pidsinfo); ++ get_cgroup_v1_files_info(layers, quiet, filesinfo); + + common_free_cgroup_layer(layers); + +@@ -432,7 +924,7 @@ static void get_cgroup_v1_metrics_pid(const cgroup_layer_t *layers, const char * + get_cgroup_v1_value_helper(path, PIDS_CURRENT, (void *)&cgroup_pids_metrics->pid_current); + } + +-int common_get_cgroup_v1_metrics(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics) ++static int get_cgroup_metrics_v1(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics) + { + cgroup_layer_t *layers = NULL; + +@@ -553,22 +1045,31 @@ static char *common_get_cgroup_path(const char *path, const char *subsystem) + return res; + } + +-char *common_get_init_cgroup(const char *subsystem) ++char *get_init_cgroup_path_v1(const char *subsystem) + { +- if (common_get_cgroup_version() != CGROUP_VERSION_1) { +- ERROR("Not implemented for cgroup v2 hierarchy"); +- return NULL; +- } +- + return common_get_cgroup_path("/proc/1/cgroup", subsystem); + } + +-char *common_get_own_cgroup(const char *subsystem) ++char *get_own_cgroup_v1(const char *subsystem) + { +- if (common_get_cgroup_version() != CGROUP_VERSION_1) { +- ERROR("Not implemented for cgroup v2 hierarchy"); +- return NULL; +- } +- + return common_get_cgroup_path("/proc/self/cgroup", subsystem); + } ++ ++int get_cgroup_version_v1() ++{ ++ return CGROUP_VERSION_1; ++} ++ ++int cgroup_v1_ops_init(cgroup_ops *ops) ++{ ++ if (ops == NULL) { ++ return -1; ++ } ++ ops->get_cgroup_version = get_cgroup_version_v1; ++ ops->get_cgroup_info = get_cgroup_info_v1; ++ ops->get_cgroup_metrics = get_cgroup_metrics_v1; ++ ops->get_cgroup_mnt_and_root_path = get_cgroup_mnt_and_root_path_v1; ++ ops->get_init_cgroup_path = get_init_cgroup_path_v1; ++ ops->get_own_cgroup_path = get_own_cgroup_v1; ++ return 0; ++} +\ No newline at end of file +diff --git a/src/daemon/common/cgroup/cgroup_v1.h b/src/daemon/common/cgroup/cgroup_v1.h +new file mode 100644 +index 00000000..c2696725 +--- /dev/null ++++ b/src/daemon/common/cgroup/cgroup_v1.h +@@ -0,0 +1,30 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-02-19 ++ * Description: provide cgroup v1 definition ++ ******************************************************************************/ ++#ifndef DAEMON_COMMON_CGROUP_CGROUP_V1_H ++#define DAEMON_COMMON_CGROUP_CGROUP_V1_H ++ ++#include "cgroup.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++int cgroup_v1_ops_init(cgroup_ops *ops); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif // DAEMON_COMMON_CGROUP_CGROUP_V1_H +diff --git a/src/daemon/common/cgroup_v2.c b/src/daemon/common/cgroup/cgroup_v2.c +similarity index 88% +rename from src/daemon/common/cgroup_v2.c +rename to src/daemon/common/cgroup/cgroup_v2.c +index 25509bda..65cf90d8 100644 +--- a/src/daemon/common/cgroup_v2.c ++++ b/src/daemon/common/cgroup/cgroup_v2.c +@@ -105,48 +105,12 @@ static struct cgfile_t g_cgroup_v2_files[] = { + + static int get_cgroup_v2_value_helper(const char *path, const cgroup_v2_files_index index, void *result) + { +- int nret = 0; +- char file_path[PATH_MAX] = { 0 }; +- char real_path[PATH_MAX] = { 0 }; +- char *content = NULL; +- + if (index >= CGROUP_V2_FILES_INDEX_MAXS) { + ERROR("Index out of range"); + return false; + } + +- if (path == NULL || strlen(path) == 0 || result == NULL) { +- ERROR("%s: Invalid arguments", g_cgroup_v2_files[index].name); +- return -1; +- } +- +- nret = snprintf(file_path, sizeof(file_path), "%s/%s", path, g_cgroup_v2_files[index].file); +- if (nret < 0 || (size_t)nret >= sizeof(file_path)) { +- ERROR("%s: failed to snprintf", g_cgroup_v2_files[index].name); +- return -1; +- } +- +- if (util_clean_path(file_path, real_path, sizeof(real_path)) == NULL) { +- ERROR("%s: failed to clean path %s", g_cgroup_v2_files[index].name, file_path); +- return -1; +- } +- +- content = util_read_content_from_file(real_path); +- if (content == NULL) { +- ERROR("%s: failed to read file %s", g_cgroup_v2_files[index].name, real_path); +- return -1; +- } +- +- util_trim_newline(content); +- content = util_trim_space(content); +- +- nret = g_cgroup_v2_files[index].get_value(content, g_cgroup_v2_files[index].match, result); +- if (nret != 0) { +- ERROR("%s: failed to get value", g_cgroup_v2_files[index].name); +- } +- +- free(content); +- return nret; ++ return get_cgroup_value_helper(path, &g_cgroup_v2_files[index], result); + } + + static void get_cgroup_v2_metrics_cpu(const char *cgroup_path, cgroup_cpu_metrics_t *cgroup_cpu_metrics) +@@ -306,28 +270,18 @@ static bool cgroup_v2_enabled(const char *mountpoint, const char *name) + return util_file_exists(path); + } + +-int common_get_cgroup_info_v2(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, +- cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, +- cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, +- cgroup_files_info_t *filesinfo, bool quiet) ++static void get_cgroup_v2_cpu_info(const bool quiet, cgroup_cpu_info_t *cpuinfo) + { +- int ret = 0; +- int nret = 0; +- __isula_auto_free char *size = NULL; +- char path[PATH_MAX] = { 0 }; +- +- if (make_sure_cgroup2_isulad_path_exist() != 0) { +- return -1; +- } +- +- // cpu cgroup + cpuinfo->cpu_shares = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPU_WEIGHT); + common_cgroup_do_log(quiet, !(cpuinfo->cpu_shares), "Your kernel does not support cgroup2 cpu weight"); + + cpuinfo->cpu_cfs_period = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPU_MAX); + cpuinfo->cpu_cfs_quota = cpuinfo->cpu_cfs_period; + common_cgroup_do_log(quiet, !(cpuinfo->cpu_cfs_period), "Your kernel does not support cgroup2 cpu max"); ++} + ++static int get_cgroup_v2_cpuset_info(const bool quiet, cgroup_cpuset_info_t *cpusetinfo) ++{ + cpusetinfo->cpuset = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPUSET_CPUS_EFFECTIVE) && + cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPUSET_CPUS) && + cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_CPUSET_MEMS_EFFECTIVE) && +@@ -343,8 +297,23 @@ int common_get_cgroup_info_v2(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpu + cpusetinfo->cpus = util_trim_space(cpusetinfo->cpus); + cpusetinfo->mems = util_trim_space(cpusetinfo->mems); + } ++ return 0; ++} ++ ++static void get_cgroup_v2_mem_info(const bool quiet, cgroup_mem_info_t *meminfo) ++{ ++ meminfo->limit = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_MEMORY_MAX); ++ common_cgroup_do_log(quiet, !(meminfo->limit), "Your kernel does not support cgroup2 memory max"); ++ ++ meminfo->reservation = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_MEMORY_LOW); ++ common_cgroup_do_log(quiet, !(meminfo->reservation), "Your kernel does not support cgroup2 memory low"); + +- // io cgroup ++ meminfo->swap = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_MEMORY_SWAP_MAX); ++ common_cgroup_do_log(quiet, !(meminfo->swap), "Your kernel does not support cgroup2 memory swap max"); ++} ++ ++static void get_cgroup_v2_blkio_info(const bool quiet, cgroup_blkio_info_t *blkioinfo) ++{ + blkioinfo->blkio_weight = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_IO_BFQ_WEIGHT) || + cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_IO_WEIGHT); + blkioinfo->blkio_weight_device = blkioinfo->blkio_weight; +@@ -355,22 +324,14 @@ int common_get_cgroup_info_v2(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpu + blkioinfo->blkio_read_iops_device = blkioinfo->blkio_read_bps_device; + blkioinfo->blkio_write_iops_device = blkioinfo->blkio_read_bps_device; + common_cgroup_do_log(quiet, !(blkioinfo->blkio_read_bps_device), "Your kernel does not support cgroup2 io max"); ++} + +- // memory cgroup +- meminfo->limit = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_MEMORY_MAX); +- common_cgroup_do_log(quiet, !(meminfo->limit), "Your kernel does not support cgroup2 memory max"); +- +- meminfo->reservation = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_MEMORY_LOW); +- common_cgroup_do_log(quiet, !(meminfo->reservation), "Your kernel does not support cgroup2 memory low"); +- +- meminfo->swap = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_MEMORY_SWAP_MAX); +- common_cgroup_do_log(quiet, !(meminfo->swap), "Your kernel does not support cgroup2 memory swap max"); +- +- // pids cgroup +- pidsinfo->pidslimit = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_PIDS_MAX); +- common_cgroup_do_log(quiet, !(pidsinfo->pidslimit), "Your kernel does not support cgroup2 pids max"); ++static int get_cgroup_v2_hugetlb_info(const bool quiet, cgroup_hugetlb_info_t *hugetlbinfo) ++{ ++ __isula_auto_free char *size = NULL; ++ int nret = 0; ++ char path[PATH_MAX] = { 0 }; + +- // hugetlb cgroup + size = get_default_huge_page_size(); + if (size != NULL) { + nret = snprintf(path, sizeof(path), CGROUP2_HUGETLB_MAX, size); +@@ -383,15 +344,55 @@ int common_get_cgroup_info_v2(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpu + } else { + WARN("Your kernel does not support cgroup2 hugetlb limit"); + } ++ return 0; ++} ++ ++static void get_cgroup_v2_pids_info(const bool quiet, cgroup_pids_info_t *pidsinfo) ++{ ++ pidsinfo->pidslimit = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_PIDS_MAX); ++ common_cgroup_do_log(quiet, !(pidsinfo->pidslimit), "Your kernel does not support cgroup2 pids max"); ++ ++} + +- // files cgroup ++static void get_cgroup_v2_files_info(const bool quiet, cgroup_files_info_t *filesinfo) ++{ + filesinfo->fileslimit = cgroup_v2_enabled(CGROUP_ISULAD_PATH, CGROUP2_FILES_LIMIT); + common_cgroup_do_log(quiet, !(filesinfo->fileslimit), "Your kernel does not support cgroup2 files limit"); + +- return ret; + } + +-int common_get_cgroup_v2_metrics(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics) ++static int get_cgroup_info_v2(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, ++ cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, ++ cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, ++ cgroup_files_info_t *filesinfo, bool quiet) ++{ ++ int ret = 0; ++ if (make_sure_cgroup2_isulad_path_exist() != 0) { ++ return -1; ++ } ++ ++ get_cgroup_v2_cpu_info(quiet, cpuinfo); ++ ++ ret = get_cgroup_v2_cpuset_info(quiet, cpusetinfo); ++ if (ret != 0) { ++ return ret; ++ } ++ ++ get_cgroup_v2_mem_info(quiet, meminfo); ++ get_cgroup_v2_blkio_info(quiet, blkioinfo); ++ ++ ret = get_cgroup_v2_hugetlb_info(quiet, hugetlbinfo); ++ if (ret != 0) { ++ return ret; ++ } ++ ++ get_cgroup_v2_pids_info(quiet, pidsinfo); ++ get_cgroup_v2_files_info(quiet, filesinfo); ++ ++ return 0; ++} ++ ++static int get_cgroup_metrics_v2(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics) + { + if (cgroup_path == NULL || strlen(cgroup_path) == 0 || cgroup_metrics == NULL) { + ERROR("Invalid arguments"); +@@ -404,3 +405,26 @@ int common_get_cgroup_v2_metrics(const char *cgroup_path, cgroup_metrics_t *cgro + + return 0; + } ++ ++static int get_cgroup_mnt_and_root_v2(const char *subsystem, char **mountpoint, char **root) ++{ ++ *mountpoint = util_strdup_s(CGROUP_ISULAD_PATH); ++ return 0; ++} ++ ++int get_cgroup_version_v2() ++{ ++ return CGROUP_VERSION_2; ++} ++ ++int cgroup_v2_ops_init(cgroup_ops *ops) ++{ ++ if (ops == NULL) { ++ return -1; ++ } ++ ops->get_cgroup_version = get_cgroup_version_v2; ++ ops->get_cgroup_info = get_cgroup_info_v2; ++ ops->get_cgroup_metrics = get_cgroup_metrics_v2; ++ ops->get_cgroup_mnt_and_root_path = get_cgroup_mnt_and_root_v2; ++ return 0; ++} +\ No newline at end of file +diff --git a/src/daemon/common/cgroup/cgroup_v2.h b/src/daemon/common/cgroup/cgroup_v2.h +new file mode 100644 +index 00000000..0e8e4818 +--- /dev/null ++++ b/src/daemon/common/cgroup/cgroup_v2.h +@@ -0,0 +1,30 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-02-19 ++ * Description: provide cgroup v2 definition ++ ******************************************************************************/ ++#ifndef DAEMON_COMMON_CGROUP_CGROUP_V2_H ++#define DAEMON_COMMON_CGROUP_CGROUP_V2_H ++ ++#include "cgroup.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++int cgroup_v2_ops_init(cgroup_ops *ops); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif // DAEMON_COMMON_CGROUP_CGROUP_V2_H +diff --git a/src/daemon/common/sysinfo.c b/src/daemon/common/sysinfo.c +index a082f717..e369c3e3 100644 +--- a/src/daemon/common/sysinfo.c ++++ b/src/daemon/common/sysinfo.c +@@ -188,15 +188,10 @@ static char **get_huge_page_sizes() + struct dirent *info_archivo = NULL; + int cgroup_version = 0; + +- cgroup_version = common_get_cgroup_version(); +- if (cgroup_version == CGROUP_VERSION_2) { +- hugetlbmp = util_strdup_s(CGROUP_ISULAD_PATH); +- } else { +- ret = common_find_cgroup_mnt_and_root("hugetlb", &hugetlbmp, NULL); +- if (ret != 0 || hugetlbmp == NULL) { +- ERROR("Hugetlb cgroup not supported"); +- return NULL; +- } ++ ret = common_get_cgroup_mnt_and_root_path("hugetlb", &hugetlbmp, NULL); ++ if (ret != 0 || hugetlbmp == NULL) { ++ ERROR("Hugetlb cgroup not supported"); ++ return NULL; + } + + dir = opendir(hugetlbmp); +@@ -211,6 +206,7 @@ static char **get_huge_page_sizes() + char *pos = NULL; + char *dot2 = NULL; + ++ cgroup_version = common_get_cgroup_version(); + if (cgroup_version == CGROUP_VERSION_2) { + if (!is_hugetlb_max(info_archivo->d_name)) { + continue; +@@ -375,7 +371,6 @@ void free_sysinfo(sysinfo_t *sysinfo) + /* get sys info */ + sysinfo_t *get_sys_info(bool quiet) + { +- int cgroup_version = 0; + sysinfo_t *sysinfo = NULL; + int ret = 0; + +@@ -388,21 +383,9 @@ sysinfo_t *get_sys_info(bool quiet) + sysinfo->ncpus = get_nprocs(); + sysinfo->ncpus_conf = get_nprocs_conf(); + +- cgroup_version = common_get_cgroup_version(); +- if (cgroup_version < 0) { +- ret = -1; +- goto out; +- } +- +- if (cgroup_version == CGROUP_VERSION_1) { +- ret = common_get_cgroup_info_v1(&sysinfo->cgmeminfo, &sysinfo->cgcpuinfo, &sysinfo->hugetlbinfo, ++ ret = common_get_cgroup_info(&sysinfo->cgmeminfo, &sysinfo->cgcpuinfo, &sysinfo->hugetlbinfo, + &sysinfo->blkioinfo, &sysinfo->cpusetinfo, &sysinfo->pidsinfo, + &sysinfo->filesinfo, quiet); +- } else { +- ret = common_get_cgroup_info_v2(&sysinfo->cgmeminfo, &sysinfo->cgcpuinfo, &sysinfo->hugetlbinfo, +- &sysinfo->blkioinfo, &sysinfo->cpusetinfo, &sysinfo->pidsinfo, +- &sysinfo->filesinfo, quiet); +- } + if (ret != 0) { + goto out; + } +@@ -563,7 +546,7 @@ free_out: + return minfos; + } + +-char *sysinfo_cgroup_controller_cpurt_mnt_path(void) ++char *sysinfo_get_cpurt_mnt_path(void) + { + int nret = 0; + __isula_auto_free char *mnt = NULL; +@@ -583,7 +566,7 @@ char *sysinfo_cgroup_controller_cpurt_mnt_path(void) + return NULL; + } + +- nret = common_find_cgroup_mnt_and_root("cpu", &mnt, &root); ++ nret = common_get_cgroup_mnt_and_root_path("cpu", &mnt, &root); + if (nret != 0 || mnt == NULL || root == NULL) { + ERROR("Can not find cgroup mnt and root path for subsystem 'cpu'"); + isulad_set_error_message("Can not find cgroup mnt and root path for subsystem 'cpu'"); +diff --git a/src/daemon/common/sysinfo.h b/src/daemon/common/sysinfo.h +index cb44d1c5..6142487b 100644 +--- a/src/daemon/common/sysinfo.h ++++ b/src/daemon/common/sysinfo.h +@@ -95,13 +95,13 @@ mountinfo_t *find_mount_info(mountinfo_t **minfos, const char *dir); + + void free_mounts_info(mountinfo_t **minfos); + +-char *sysinfo_cgroup_controller_cpurt_mnt_path(void); +- + // define auto free function callback for sysinfo_t + define_auto_cleanup_callback(free_sysinfo, sysinfo_t) + // define auto free macro for sysinfo_t + #define __isula_auto_sysinfo_t auto_cleanup_tag(free_sysinfo) + ++char *sysinfo_get_cpurt_mnt_path(void); ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +index 3bdc3af8..f125e714 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +@@ -872,18 +872,7 @@ void PodSandboxManagerService::GetPodSandboxCgroupMetrics(const std::string &cgr + return; + } + +- auto cgroupVersion = common_get_cgroup_version(); +- if (cgroupVersion < 0) { +- error.Errorf("Invalid cgroup version"); +- return; +- } +- +- if (cgroupVersion == CGROUP_VERSION_1) { +- nret = common_get_cgroup_v1_metrics(cgroupParent.c_str(), &cgroupMetrics); +- } else { +- nret = common_get_cgroup_v2_metrics(cgroupParent.c_str(), &cgroupMetrics); +- } +- ++ nret = common_get_cgroup_metrics(cgroupParent.c_str(), &cgroupMetrics); + if (nret != 0) { + error.Errorf("Failed to get cgroup metrics"); + } +diff --git a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +index 49a7ca54..af6b5fff 100644 +--- a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +@@ -1328,18 +1328,7 @@ void PodSandboxManagerService::GetPodSandboxCgroupMetrics(const container_inspec + return; + } + +- auto cgroupVersion = common_get_cgroup_version(); +- if (cgroupVersion < 0) { +- error.Errorf("Invalid cgroup version"); +- return; +- } +- +- if (cgroupVersion == CGROUP_VERSION_1) { +- nret = common_get_cgroup_v1_metrics(cgroupParent, &cgroupMetrics); +- } else { +- nret = common_get_cgroup_v2_metrics(cgroupParent, &cgroupMetrics); +- } +- ++ nret = common_get_cgroup_metrics(cgroupParent, &cgroupMetrics); + if (nret != 0) { + error.Errorf("Failed to get cgroup metrics"); + } +diff --git a/src/daemon/executor/container_cb/execution.c b/src/daemon/executor/container_cb/execution.c +index 7ed8e837..88c6b354 100644 +--- a/src/daemon/executor/container_cb/execution.c ++++ b/src/daemon/executor/container_cb/execution.c +@@ -413,6 +413,13 @@ static int cpurt_controller_init(const char *id, const host_config *host_spec) + char *dirpath = NULL; + int64_t cpu_rt_period = 0; + int64_t cpu_rt_runtime = 0; ++ int cgroup_version = 0; ++ ++ // cgroup v2 is not support cpurt ++ cgroup_version = common_get_cgroup_version(); ++ if (cgroup_version == CGROUP_VERSION_2) { ++ return 0; ++ } + + cgroups_path = merge_container_cgroups_path(id, host_spec); + if (cgroups_path == NULL || strcmp(cgroups_path, "/") == 0 || strcmp(cgroups_path, ".") == 0) { +@@ -433,13 +440,13 @@ static int cpurt_controller_init(const char *id, const host_config *host_spec) + // should iSulad set cpu.rt_runtime_us and cpu.rt_period_us for the parent path? + // in fact, even if system.slice is used, + // cpu.rt_runtime_us and cpu.rt_period_us might still needed to be set manually +- __isula_auto_free char *init_cgroup = common_get_init_cgroup("cpu"); ++ __isula_auto_free char *init_cgroup = common_get_init_cgroup_path("cpu"); + if (init_cgroup == NULL) { + ERROR("Failed to get init cgroup"); + return -1; + } + // make sure that the own cgroup path for cpu existed +- __isula_auto_free char *own_cgroup = common_get_own_cgroup("cpu"); ++ __isula_auto_free char *own_cgroup = common_get_own_cgroup_path("cpu"); + if (own_cgroup == NULL) { + ERROR("Failed to get own cgroup"); + return -1; +@@ -453,7 +460,7 @@ static int cpurt_controller_init(const char *id, const host_config *host_spec) + cgroups_path = new_cgroups_path; + } + +- mnt_root = sysinfo_cgroup_controller_cpurt_mnt_path(); ++ mnt_root = sysinfo_get_cpurt_mnt_path(); + if (mnt_root == NULL) { + ERROR("Failed to get cpu rt controller mnt root path"); + return -1; +-- +2.34.1 + diff --git a/0030-adaptor-unit-test-for-cgroup-module.patch b/0030-adaptor-unit-test-for-cgroup-module.patch new file mode 100644 index 0000000..66fd4dd --- /dev/null +++ b/0030-adaptor-unit-test-for-cgroup-module.patch @@ -0,0 +1,351 @@ +From 59e7ea0f16e83e0bdbc39bdc41d1ade8d3db885e Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Thu, 22 Feb 2024 09:52:30 +0800 +Subject: [PATCH 30/43] adaptor unit test for cgroup module + +Signed-off-by: zhongtao +--- + src/daemon/modules/image/CMakeLists.txt | 6 ++++-- + test/cgroup/cpu/CMakeLists.txt | 7 +++++-- + test/cgroup/cpu/cgroup_cpu_ut.cc | 12 ++++++++++-- + test/image/oci/oci_config_merge/CMakeLists.txt | 7 +++++-- + test/image/oci/registry/CMakeLists.txt | 7 +++++-- + test/network/network_ns/CMakeLists.txt | 7 +++++-- + test/runtime/isula/CMakeLists.txt | 7 +++++-- + test/runtime/lcr/CMakeLists.txt | 7 +++++-- + .../execute/execution_extend/CMakeLists.txt | 6 +++++- + test/specs/specs/CMakeLists.txt | 7 +++++-- + test/specs/specs_extend/CMakeLists.txt | 7 +++++-- + test/specs/verify/CMakeLists.txt | 7 +++++-- + test/volume/CMakeLists.txt | 7 +++++-- + 13 files changed, 69 insertions(+), 25 deletions(-) + +diff --git a/src/daemon/modules/image/CMakeLists.txt b/src/daemon/modules/image/CMakeLists.txt +index f8bc5840..d8b78ce1 100644 +--- a/src/daemon/modules/image/CMakeLists.txt ++++ b/src/daemon/modules/image/CMakeLists.txt +@@ -68,8 +68,9 @@ set(LIB_ISULAD_IMG_SRCS + ${CMAKE_SOURCE_DIR}/src/utils/buffer/buffer.c + ${CMAKE_SOURCE_DIR}/src/daemon/common/err_msg.c + ${CMAKE_SOURCE_DIR}/src/daemon/common/sysinfo.c +- ${CMAKE_SOURCE_DIR}/src/daemon/common/cgroup.c +- ${CMAKE_SOURCE_DIR}/src/daemon/common/cgroup_v1.c ++ ${CMAKE_SOURCE_DIR}/src/daemon/common/cgroup/cgroup.c ++ ${CMAKE_SOURCE_DIR}/src/daemon/common/cgroup/cgroup_v1.c ++ ${CMAKE_SOURCE_DIR}/src/daemon/common/cgroup/cgroup_v2.c + ${CMAKE_SOURCE_DIR}/src/utils/tar/util_gzip.c + ${CMAKE_SOURCE_DIR}/src/daemon/config/isulad_config.c + ${CMAKE_SOURCE_DIR}/src/daemon/config/daemon_arguments.c +@@ -102,6 +103,7 @@ target_include_directories(${LIB_ISULAD_IMG} PUBLIC + ${CMAKE_SOURCE_DIR}/src/utils/tar + ${CMAKE_SOURCE_DIR}/src/daemon/config + ${CMAKE_SOURCE_DIR}/src/daemon/common ++ ${CMAKE_SOURCE_DIR}/src/daemon/common/cgroup + ${CMAKE_SOURCE_DIR}/src/daemon/modules/spec/ + ${CMAKE_SOURCE_DIR}/src/utils/cutils + ${CMAKE_SOURCE_DIR}/src/utils/cutils/map +diff --git a/test/cgroup/cpu/CMakeLists.txt b/test/cgroup/cpu/CMakeLists.txt +index 32fe0a23..30bfc417 100644 +--- a/test/cgroup/cpu/CMakeLists.txt ++++ b/test/cgroup/cpu/CMakeLists.txt +@@ -6,8 +6,10 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/path.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/sysinfo.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v2.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_common.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/command_parser.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/daemon_arguments.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/isulad_config.c +@@ -20,6 +22,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/common + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup + ${CMAKE_BINARY_DIR}/conf + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd +diff --git a/test/cgroup/cpu/cgroup_cpu_ut.cc b/test/cgroup/cpu/cgroup_cpu_ut.cc +index eaee90c0..6e6e04f4 100644 +--- a/test/cgroup/cpu/cgroup_cpu_ut.cc ++++ b/test/cgroup/cpu/cgroup_cpu_ut.cc +@@ -69,13 +69,21 @@ TEST(CgroupCpuUnitTest, test_common_find_cgroup_mnt_and_root) + { + char *mnt = NULL; + char *root = NULL; +- ASSERT_EQ(common_find_cgroup_mnt_and_root(nullptr, &mnt, &root), -1); ++ ++ int ret = cgroup_ops_init(); ++ ASSERT_EQ(ret, 0); ++ ++ ASSERT_EQ(common_get_cgroup_mnt_and_root_path(nullptr, &mnt, &root), -1); + } + + TEST(CgroupCpuUnitTest, test_sysinfo_cgroup_controller_cpurt_mnt_path) + { + MOCK_SET(util_common_calloc_s, nullptr); + ASSERT_EQ(get_sys_info(true), nullptr); +- ASSERT_EQ(sysinfo_cgroup_controller_cpurt_mnt_path(), nullptr); ++ ++ int ret = cgroup_ops_init(); ++ ASSERT_EQ(ret, 0); ++ ++ ASSERT_EQ(sysinfo_get_cpurt_mnt_path(), nullptr); + MOCK_CLEAR(util_common_calloc_s); + } +diff --git a/test/image/oci/oci_config_merge/CMakeLists.txt b/test/image/oci/oci_config_merge/CMakeLists.txt +index 90809080..d13ec738 100644 +--- a/test/image/oci/oci_config_merge/CMakeLists.txt ++++ b/test/image/oci/oci_config_merge/CMakeLists.txt +@@ -25,8 +25,10 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/err_msg.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/config/isulad_config.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/sysinfo.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup/cgroup.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup/cgroup_v2.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup/cgroup_common.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/cmd/command_parser.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/config/daemon_arguments.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/image/oci/oci_ut_common.cc +@@ -55,6 +57,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/utils/cutils/map + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/volume + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/api + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/runtime/engines +diff --git a/test/image/oci/registry/CMakeLists.txt b/test/image/oci/registry/CMakeLists.txt +index 77a7907e..5b5bc3f5 100644 +--- a/test/image/oci/registry/CMakeLists.txt ++++ b/test/image/oci/registry/CMakeLists.txt +@@ -26,8 +26,10 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/image_store/image_type.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/registry_type.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/sysinfo.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup/cgroup.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup/cgroup_v2.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup/cgroup_common.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/image_store/image_store.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/registry/registry.c +@@ -58,6 +60,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/api + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/image_store +diff --git a/test/network/network_ns/CMakeLists.txt b/test/network/network_ns/CMakeLists.txt +index 50520427..71b8039d 100644 +--- a/test/network/network_ns/CMakeLists.txt ++++ b/test/network/network_ns/CMakeLists.txt +@@ -30,8 +30,10 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/daemon_arguments.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/sysinfo.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v2.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_common.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/command_parser.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/image/oci/oci_ut_common.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/image_mock.cc +@@ -55,6 +57,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/options + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/api + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/volume + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/runtime +diff --git a/test/runtime/isula/CMakeLists.txt b/test/runtime/isula/CMakeLists.txt +index cc1178b8..c1f0a5cc 100644 +--- a/test/runtime/isula/CMakeLists.txt ++++ b/test/runtime/isula/CMakeLists.txt +@@ -21,8 +21,10 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/mainloop.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/sysinfo.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v2.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_common.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/command_parser.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/daemon_arguments.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/image/oci/oci_ut_common.cc +@@ -51,6 +53,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/container + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/container/restart_manager + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/container/health_check +diff --git a/test/runtime/lcr/CMakeLists.txt b/test/runtime/lcr/CMakeLists.txt +index 424a6101..c3b93d67 100644 +--- a/test/runtime/lcr/CMakeLists.txt ++++ b/test/runtime/lcr/CMakeLists.txt +@@ -18,8 +18,10 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map/rb_tree.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/sysinfo.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v2.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_common.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/command_parser.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/daemon_arguments.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/image/oci/oci_ut_common.cc +@@ -39,6 +41,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/api + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/container + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/container/restart_manager +diff --git a/test/services/execution/execute/execution_extend/CMakeLists.txt b/test/services/execution/execute/execution_extend/CMakeLists.txt +index 68e0f443..f0875fd7 100644 +--- a/test/services/execution/execute/execution_extend/CMakeLists.txt ++++ b/test/services/execution/execute/execution_extend/CMakeLists.txt +@@ -21,7 +21,10 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/mainloop.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/filters.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common/err_msg.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common/cgroup.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common/cgroup/cgroup.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common/cgroup/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common/cgroup/cgroup_v2.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common/cgroup/cgroup_common.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/events_sender/event_sender.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/console/console.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/utils.c +@@ -59,6 +62,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/console + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common/cgroup + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/api + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image +diff --git a/test/specs/specs/CMakeLists.txt b/test/specs/specs/CMakeLists.txt +index 508123fa..45f688f9 100644 +--- a/test/specs/specs/CMakeLists.txt ++++ b/test/specs/specs/CMakeLists.txt +@@ -28,8 +28,10 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/spec/specs_security.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/sysinfo.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v2.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_common.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/command_parser.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/daemon_arguments.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/image/oci/oci_ut_common.cc +@@ -54,6 +56,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/api + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/volume + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/runtime +diff --git a/test/specs/specs_extend/CMakeLists.txt b/test/specs/specs_extend/CMakeLists.txt +index bf4b378e..1b737089 100644 +--- a/test/specs/specs_extend/CMakeLists.txt ++++ b/test/specs/specs_extend/CMakeLists.txt +@@ -28,8 +28,10 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/spec/specs_security.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/sysinfo.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v2.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_common.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/command_parser.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/daemon_arguments.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/image/oci/oci_ut_common.cc +@@ -54,6 +56,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/volume + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/api + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/container +diff --git a/test/specs/verify/CMakeLists.txt b/test/specs/verify/CMakeLists.txt +index 0e60a39e..b0602127 100644 +--- a/test/specs/verify/CMakeLists.txt ++++ b/test/specs/verify/CMakeLists.txt +@@ -23,8 +23,10 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map/rb_tree.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/sysinfo.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v2.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_common.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/spec/verify.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/image/oci/oci_ut_common.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/containers_store_mock.cc +@@ -50,6 +52,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/api + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/volume + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/runtime +diff --git a/test/volume/CMakeLists.txt b/test/volume/CMakeLists.txt +index cc309352..27d07330 100644 +--- a/test/volume/CMakeLists.txt ++++ b/test/volume/CMakeLists.txt +@@ -20,8 +20,10 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/utils/sha256/sha256.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/err_msg.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/sysinfo.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup/cgroup.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup/cgroup_v2.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup/cgroup_common.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/cmd/command_parser.c + volume_ut.cc) + +@@ -34,6 +36,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/utils/http + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/modules/api + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/modules/volume + ${CMAKE_BINARY_DIR}/conf + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/utils/sha256 +-- +2.34.1 + diff --git a/0031-cgroup-v2-does-not-support-isulad-setting-cpu_rt-opt.patch b/0031-cgroup-v2-does-not-support-isulad-setting-cpu_rt-opt.patch new file mode 100644 index 0000000..ecca55a --- /dev/null +++ b/0031-cgroup-v2-does-not-support-isulad-setting-cpu_rt-opt.patch @@ -0,0 +1,71 @@ +From 8e11a1eea62cb8061f1613379ff83bd9a721fa50 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Wed, 31 Jan 2024 18:10:46 +0800 +Subject: [PATCH 31/43] cgroup v2 does not support isulad setting cpu_rt + options + +Signed-off-by: zhongtao +--- + src/cmd/isulad/isulad_commands.c | 32 ++++++++++++++++++++++++++++++++ + 1 file changed, 32 insertions(+) + +diff --git a/src/cmd/isulad/isulad_commands.c b/src/cmd/isulad/isulad_commands.c +index 5fb55689..619e36d1 100644 +--- a/src/cmd/isulad/isulad_commands.c ++++ b/src/cmd/isulad/isulad_commands.c +@@ -34,6 +34,7 @@ + #include "utils_verify.h" + #include "opt_ulimit.h" + #include "opt_log.h" ++#include "sysinfo.h" + + const char isulad_desc[] = "GLOBAL OPTIONS:"; + const char isulad_usage[] = "[global options]"; +@@ -411,6 +412,33 @@ out: + return ret; + } + ++static int check_args_cpu_rt(const struct service_arguments *args) ++{ ++ int ret = 0; ++ __isula_auto_sysinfo_t sysinfo_t *sysinfo = NULL; ++ ++ sysinfo = get_sys_info(true); ++ if (sysinfo == NULL) { ++ COMMAND_ERROR("Failed to get system info"); ++ ERROR("Failed to get system info"); ++ return -1; ++ } ++ ++ if (!(sysinfo->cgcpuinfo.cpu_rt_period) && args->json_confs->cpu_rt_period != 0) { ++ COMMAND_ERROR("Invalid --cpu-rt-period: Your kernel does not support cgroup rt period"); ++ ERROR("Invalid --cpu-rt-period: Your kernel does not support cgroup rt period"); ++ return -1; ++ } ++ ++ if (!(sysinfo->cgcpuinfo.cpu_rt_runtime) && args->json_confs->cpu_rt_runtime != 0) { ++ COMMAND_ERROR("Invalid --cpu-rt-runtime: Your kernel does not support cgroup rt runtime"); ++ ERROR("Invalid --cpu-rt-runtime: Your kernel does not support cgroup rt runtime"); ++ return -1; ++ } ++ ++ return ret; ++} ++ + int check_args(struct service_arguments *args) + { + int ret = 0; +@@ -471,6 +499,10 @@ int check_args(struct service_arguments *args) + goto out; + } + ++ if (check_args_cpu_rt(args) != 0) { ++ ret = -1; ++ goto out; ++ } + out: + return ret; + } +-- +2.34.1 + diff --git a/0032-add-test-that-isulad-cannot-set-cpu_rt-parameters-wh.patch b/0032-add-test-that-isulad-cannot-set-cpu_rt-parameters-wh.patch new file mode 100644 index 0000000..1568edb --- /dev/null +++ b/0032-add-test-that-isulad-cannot-set-cpu_rt-parameters-wh.patch @@ -0,0 +1,30 @@ +From 1ab0f4608fb749b50aa6f8d8188db23aa8a6e1ac Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Thu, 1 Feb 2024 10:48:45 +0800 +Subject: [PATCH 32/43] add test that isulad cannot set cpu_rt parameters when + adding cgroup v2 + +Signed-off-by: zhongtao +--- + CI/test_cases/container_cases/cpu_rt.sh | 5 ++++- + 1 file changed, 4 insertions(+), 1 deletion(-) + +diff --git a/CI/test_cases/container_cases/cpu_rt.sh b/CI/test_cases/container_cases/cpu_rt.sh +index bdc43a5e..23d3baed 100755 +--- a/CI/test_cases/container_cases/cpu_rt.sh ++++ b/CI/test_cases/container_cases/cpu_rt.sh +@@ -106,7 +106,10 @@ function test_kernel_without_cpurt() + + msg_info "${test} starting..." + +- start_isulad_without_valgrind --cpu-rt-period 1000000 --cpu-rt-runtime 950000 ++ isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 2>&1 | grep 'Your kernel does not support cgroup rt period' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - kernel does not support cpu-rt, but start isulad with cpu-rt success" && ((ret++)) ++ ++ start_isulad_without_valgrind + + isula pull ${image} + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${image}" && return ${FAILURE} +-- +2.34.1 + diff --git a/0033-fix-sandbox-container-bool-value-uninitialized.patch b/0033-fix-sandbox-container-bool-value-uninitialized.patch new file mode 100644 index 0000000..2f88424 --- /dev/null +++ b/0033-fix-sandbox-container-bool-value-uninitialized.patch @@ -0,0 +1,26 @@ +From f62df3dedbbe11bb56e6da7dd610c573fd3ed828 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Mon, 25 Mar 2024 10:01:56 +0800 +Subject: [PATCH 33/43] fix sandbox container bool value uninitialized + +Signed-off-by: jikai +--- + src/daemon/modules/service/service_container.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c +index 7b34cc7f..a8090d5a 100644 +--- a/src/daemon/modules/service/service_container.c ++++ b/src/daemon/modules/service/service_container.c +@@ -750,7 +750,7 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo + oci_runtime_spec *oci_spec = NULL; + rt_create_params_t create_params = { 0 }; + rt_start_params_t start_params = { 0 }; +- bool sandbox_container; ++ bool sandbox_container = false; + + nret = snprintf(bundle, sizeof(bundle), "%s/%s", cont->root_path, id); + if (nret < 0 || (size_t)nret >= sizeof(bundle)) { +-- +2.34.1 + diff --git a/0034-bugfix-for-cpurt.sh.patch b/0034-bugfix-for-cpurt.sh.patch new file mode 100644 index 0000000..d8b52bb --- /dev/null +++ b/0034-bugfix-for-cpurt.sh.patch @@ -0,0 +1,47 @@ +From 411483ad9b2a0c50190f9b56779d41889c895014 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Wed, 27 Mar 2024 10:29:11 +0800 +Subject: [PATCH 34/43] bugfix for cpurt.sh + +Signed-off-by: zhongtao +--- + CI/test_cases/container_cases/cpu_rt.sh | 9 ++++----- + 1 file changed, 4 insertions(+), 5 deletions(-) + +diff --git a/CI/test_cases/container_cases/cpu_rt.sh b/CI/test_cases/container_cases/cpu_rt.sh +index 23d3baed..64dcd81f 100755 +--- a/CI/test_cases/container_cases/cpu_rt.sh ++++ b/CI/test_cases/container_cases/cpu_rt.sh +@@ -109,7 +109,7 @@ function test_kernel_without_cpurt() + isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 2>&1 | grep 'Your kernel does not support cgroup rt period' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - kernel does not support cpu-rt, but start isulad with cpu-rt success" && ((ret++)) + +- start_isulad_without_valgrind ++ start_isulad_with_valgrind + + isula pull ${image} + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${image}" && return ${FAILURE} +@@ -194,6 +194,9 @@ function do_test() + test_cpurt_isulad_abnormal $runtime || ((ret++)) + test_isula_update_normal $runtime || ((ret++)) + test_isula_update_abnormal $runtime || ((ret++)) ++ stop_isulad_without_valgrind ++ # set cpu-rt to the initial state ++ start_isulad_with_valgrind --cpu-rt-period 1000000 --cpu-rt-runtime 0 + else + test_kernel_without_cpurt $runtime || ((ans++)) + fi +@@ -211,10 +214,6 @@ do + + do_test $element || ((ans++)) + +- stop_isulad_without_valgrind +- # set cpu-rt to the initial state +- start_isulad_with_valgrind --cpu-rt-period 1000000 --cpu-rt-runtime 0 +- + isula rm -f $(isula ps -aq) + done + +-- +2.34.1 + diff --git a/0035-monitor-cgroup-oom-killed-event-and-update-to-cri-of.patch b/0035-monitor-cgroup-oom-killed-event-and-update-to-cri-of.patch new file mode 100644 index 0000000..85f7f62 --- /dev/null +++ b/0035-monitor-cgroup-oom-killed-event-and-update-to-cri-of.patch @@ -0,0 +1,868 @@ +From 947cf87a87ec49409ae509e5142b8134454d1547 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Thu, 28 Mar 2024 12:51:09 +0000 +Subject: [PATCH 35/43] monitor cgroup oom killed event and update to cri of + container + +Signed-off-by: jikai +--- + src/daemon/common/cgroup/cgroup.c | 91 +++++++++- + src/daemon/common/cgroup/cgroup.h | 5 + + src/daemon/common/cgroup/cgroup_common.h | 13 ++ + src/daemon/common/cgroup/cgroup_v1.c | 160 ++++++++++++++++++ + src/daemon/common/cgroup/cgroup_v2.c | 138 ++++++++++++++- + .../v1/v1_cri_container_manager_service.cc | 3 + + src/daemon/modules/api/container_api.h | 5 +- + .../container/container_events_handler.c | 12 +- + .../modules/container/container_state.c | 15 ++ + .../modules/container/restore/restore.c | 10 +- + .../modules/container/supervisor/supervisor.c | 54 +++++- + src/daemon/modules/events/collector.c | 7 +- + .../modules/service/service_container.c | 11 +- + 13 files changed, 498 insertions(+), 26 deletions(-) + +diff --git a/src/daemon/common/cgroup/cgroup.c b/src/daemon/common/cgroup/cgroup.c +index 837b514a..d3f1445a 100644 +--- a/src/daemon/common/cgroup/cgroup.c ++++ b/src/daemon/common/cgroup/cgroup.c +@@ -133,4 +133,93 @@ char *common_get_own_cgroup_path(const char *subsystem) + } + + return g_cgroup_ops.get_own_cgroup_path(subsystem); +-} +\ No newline at end of file ++} ++ ++char *common_convert_cgroup_path(const char *cgroup_path) ++{ ++ char *token = NULL; ++ char result[PATH_MAX + 1] = {0}; ++ __isula_auto_array_t char **arr = NULL; ++ ++ if (cgroup_path == NULL) { ++ ERROR("Invalid NULL cgroup path"); ++ return NULL; ++ } ++ ++ // for cgroup fs cgroup path, return directly ++ if (!util_has_suffix(cgroup_path, ".slice")) { ++ return util_strdup_s(cgroup_path); ++ } ++ ++ // for systemd cgroup, cgroup_path should have the form slice:prefix:id, ++ // convert it to a true path, such as from test-a.slice:isulad:id ++ // to test.slice/test-a.slice/isulad-id.scope ++ arr = util_string_split_n(cgroup_path, ':', 3); ++ if (arr == NULL || util_array_len((const char **)arr) != 3) { ++ ERROR("Invalid systemd cgroup parent"); ++ return NULL; ++ } ++ ++ token = strchr(arr[0], '-'); ++ while (token != NULL) { ++ *token = '\0'; ++ if (strlen(arr[0]) > PATH_MAX || strlen(result) + 1 + strlen(".slice") > ++ PATH_MAX - strlen(arr[0])) { ++ ERROR("Invalid systemd cgroup parent: exceeds max length of path"); ++ *token = '-'; ++ return NULL; ++ } ++ if (result[0] != '\0') { ++ strcat(result, "/"); ++ } ++ strcat(result, arr[0]); ++ strcat(result, ".slice"); ++ *token = '-'; ++ token = strchr(token + 1, '-'); ++ } ++ ++ // Add /arr[0]/arr[1]-arr[2].scope, 3 include two slashes and one dash ++ if (strlen(cgroup_path) > PATH_MAX || strlen(result) + 3 + strlen(".scope") > ++ PATH_MAX - strlen(arr[0] - strlen(arr[1]) - strlen(arr[2]))) { ++ ERROR("Invalid systemd cgroup parent: exceeds max length of path"); ++ return NULL; ++ } ++ ++ (void)strcat(result, "/"); ++ (void)strcat(result, arr[0]); ++ (void)strcat(result, "/"); ++ (void)strcat(result, arr[1]); ++ (void)strcat(result, "-"); ++ (void)strcat(result, arr[2]); ++ (void)strcat(result, ".scope"); ++ ++ return util_strdup_s(result); ++} ++ ++cgroup_oom_handler_info_t *common_get_cgroup_oom_handler(int fd, const char *name, const char *cgroup_path, const char *exit_fifo) ++{ ++ if (g_cgroup_ops.get_cgroup_oom_handler == NULL) { ++ ERROR("Unimplmented get_cgroup_oom_handler op"); ++ return NULL; ++ } ++ ++ return g_cgroup_ops.get_cgroup_oom_handler(fd, name, cgroup_path, exit_fifo); ++} ++ ++void common_free_cgroup_oom_handler_info(cgroup_oom_handler_info_t *info) ++{ ++ if (info == NULL) { ++ return; ++ } ++ ++ if (info->oom_event_fd >= 0) { ++ close(info->oom_event_fd); ++ } ++ if (info->cgroup_file_fd >= 0) { ++ close(info->cgroup_file_fd); ++ } ++ ++ free(info->name); ++ free(info->cgroup_memory_event_path); ++ free(info); ++} +diff --git a/src/daemon/common/cgroup/cgroup.h b/src/daemon/common/cgroup/cgroup.h +index 1efc3ca6..8c76d99d 100644 +--- a/src/daemon/common/cgroup/cgroup.h ++++ b/src/daemon/common/cgroup/cgroup.h +@@ -41,6 +41,11 @@ int common_get_cgroup_mnt_and_root_path(const char *subsystem, char **mountpoint + char *common_get_init_cgroup_path(const char *subsystem); + char *common_get_own_cgroup_path(const char *subsystem); + ++char *common_convert_cgroup_path(const char *cgroup_path); ++ ++cgroup_oom_handler_info_t *common_get_cgroup_oom_handler(int fd, const char *name, const char *cgroup_path, const char *exit_fifo); ++void common_free_cgroup_oom_handler_info(cgroup_oom_handler_info_t *info); ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/common/cgroup/cgroup_common.h b/src/daemon/common/cgroup/cgroup_common.h +index 2a0935cb..e3912bf0 100644 +--- a/src/daemon/common/cgroup/cgroup_common.h ++++ b/src/daemon/common/cgroup/cgroup_common.h +@@ -116,6 +116,17 @@ typedef struct { + cgroup_pids_metrics_t cgpids_metrics; + } cgroup_metrics_t; + ++#define CGROUP_OOM_HANDLE_CONTINUE false ++#define CGROUP_OOM_HANDLE_CLOSE true ++ ++typedef struct _cgroup_oom_handler_info_t { ++ int oom_event_fd; ++ int cgroup_file_fd; ++ char *name; ++ char *cgroup_memory_event_path; ++ bool (*oom_event_handler)(int, void *); ++} cgroup_oom_handler_info_t; ++ + typedef struct { + int (*get_cgroup_version)(void); + int (*get_cgroup_info)(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, +@@ -128,6 +139,8 @@ typedef struct { + + char *(*get_init_cgroup_path)(const char *subsystem); + char *(*get_own_cgroup_path)(const char *subsystem); ++ ++ cgroup_oom_handler_info_t *(*get_cgroup_oom_handler)(int fd, const char *name, const char *cgroup_path, const char *exit_fifo); + } cgroup_ops; + + #ifdef __cplusplus +diff --git a/src/daemon/common/cgroup/cgroup_v1.c b/src/daemon/common/cgroup/cgroup_v1.c +index 51cf7512..41f3110a 100644 +--- a/src/daemon/common/cgroup/cgroup_v1.c ++++ b/src/daemon/common/cgroup/cgroup_v1.c +@@ -12,14 +12,20 @@ + * Create: 2023-03-29 + * Description: provide cgroup v1 functions + ******************************************************************************/ ++#ifndef _GNU_SOURCE ++#define _GNU_SOURCE ++#endif ++ + #include "cgroup.h" + + #include + #include ++#include + + #include "utils.h" + #include "sysinfo.h" + #include "err_msg.h" ++#include "events_sender_api.h" + + #define CGROUP_HUGETLB_LIMIT "hugetlb.%s.limit_in_bytes" + #define CGROUP_MOUNT_PATH_PREFIX "/sys/fs/cgroup/" +@@ -1045,6 +1051,159 @@ static char *common_get_cgroup_path(const char *path, const char *subsystem) + return res; + } + ++static bool oom_cb_cgroup_v1(int fd, void *cbdata) ++{ ++ cgroup_oom_handler_info_t *info = (cgroup_oom_handler_info_t *)cbdata; ++ /* Try to read cgroup.event_control and known if the cgroup was removed ++ * if the cgroup was removed and only one event received, ++ * we know that it is a cgroup removal event rather than an oom event ++ */ ++ bool cgroup_removed = false; ++ if (info == NULL) { ++ ERROR("Invalide callback data"); ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ if (access(info->cgroup_memory_event_path, F_OK) < 0) { ++ DEBUG("Cgroup event path was removed"); ++ cgroup_removed = true; ++ } ++ ++ uint64_t event_count; ++ ssize_t num_read = util_read_nointr(fd, &event_count, sizeof(uint64_t)); ++ if (num_read < 0) { ++ ERROR("Failed to read oom event from eventfd"); ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ if (num_read == 0) { ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ if (num_read != sizeof(uint64_t)) { ++ ERROR("Failed to read full oom event from eventfd"); ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ if (event_count == 0) { ++ ERROR("Unexpected event count when reading for oom event"); ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ if (event_count == 1 && cgroup_removed) { ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ INFO("OOM event detected"); ++ (void)isulad_monitor_send_container_event(info->name, OOM, -1, 0, NULL, NULL); ++ ++ return CGROUP_OOM_HANDLE_CLOSE; ++} ++ ++static char *get_memory_cgroup_path_v1(const char *cgroup_path) ++{ ++ int nret = 0; ++ __isula_auto_free char *converted_cgroup_path = NULL; ++ __isula_auto_free char *mnt = NULL; ++ __isula_auto_free char *root = NULL; ++ char fpath[PATH_MAX] = { 0 }; ++ ++ converted_cgroup_path = common_convert_cgroup_path(cgroup_path); ++ if (converted_cgroup_path == NULL) { ++ ERROR("Failed to transfer cgroup path"); ++ return NULL; ++ } ++ ++ nret = get_cgroup_mnt_and_root_path_v1("memory", &mnt, &root); ++ if (nret != 0 || mnt == NULL || root == NULL) { ++ ERROR("Can not find cgroup mnt and root path for subsystem 'memory'"); ++ return NULL; ++ } ++ ++ // When iSulad is run inside docker, the root is based of the host cgroup. ++ // Replace root to "/" ++ if (strncmp(root, "/docker/", strlen("/docker/")) == 0) { ++ root[1] = '\0'; ++ } ++ ++ nret = snprintf(fpath, sizeof(fpath), "%s/%s", mnt, root); ++ if (nret < 0 || (size_t)nret >= sizeof(fpath)) { ++ ERROR("Failed to print string"); ++ return NULL; ++ } ++ ++ return util_path_join(fpath, converted_cgroup_path); ++} ++ ++static cgroup_oom_handler_info_t *get_cgroup_oom_handler_v1(int fd, const char *name, const char *cgroup_path, const char *exit_fifo) ++{ ++ __isula_auto_free char *memory_cgroup_path = NULL; ++ __isula_auto_free char *memory_cgroup_oom_control_path = NULL; ++ __isula_auto_free char *data = NULL; ++ __isula_auto_close int cgroup_event_control_fd = -1; ++ if (name == NULL || cgroup_path == NULL || exit_fifo == NULL) { ++ ERROR("Invalid arguments"); ++ return NULL; ++ } ++ ++ cgroup_oom_handler_info_t *info = util_common_calloc_s(sizeof(cgroup_oom_handler_info_t)); ++ if (info == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ info->name = util_strdup_s(name); ++ info->cgroup_file_fd = -1; ++ info->oom_event_fd = -1; ++ info->oom_event_handler = oom_cb_cgroup_v1; ++ ++ memory_cgroup_path = get_memory_cgroup_path_v1(cgroup_path); ++ if (memory_cgroup_path == NULL) { ++ ERROR("Failed to get memory cgroup path"); ++ goto cleanup; ++ } ++ ++ info->cgroup_memory_event_path = util_path_join(memory_cgroup_path, "cgroup.event_control"); ++ if (info->cgroup_memory_event_path == NULL) { ++ ERROR("Failed to join memory cgroup file path"); ++ goto cleanup; ++ } ++ ++ cgroup_event_control_fd = util_open(info->cgroup_memory_event_path, O_WRONLY | O_CLOEXEC, 0); ++ if (cgroup_event_control_fd < 0) { ++ ERROR("Failed to open %s", info->cgroup_memory_event_path); ++ goto cleanup; ++ } ++ ++ memory_cgroup_oom_control_path = util_path_join(memory_cgroup_path, "memory.oom_control"); ++ if (memory_cgroup_oom_control_path == NULL) { ++ ERROR("Failed to join memory cgroup file path"); ++ goto cleanup; ++ } ++ ++ info->cgroup_file_fd = util_open(memory_cgroup_oom_control_path, O_RDONLY | O_CLOEXEC, 0); ++ if (info->cgroup_file_fd < 0) { ++ ERROR("Failed to open %s", memory_cgroup_oom_control_path); ++ goto cleanup; ++ } ++ ++ info->oom_event_fd = eventfd(0, EFD_CLOEXEC); ++ if (info->oom_event_fd < 0) { ++ ERROR("Failed to create oom eventfd"); ++ goto cleanup; ++ } ++ ++ if (asprintf(&data, "%d %d", info->oom_event_fd, info->cgroup_file_fd) < 0 || ++ util_write_nointr(cgroup_event_control_fd, data, strlen(data)) < 0) { ++ ERROR("Failed to write to cgroup.event_control"); ++ goto cleanup; ++ } ++ ++ return info; ++cleanup: ++ common_free_cgroup_oom_handler_info(info); ++ return NULL; ++} ++ + char *get_init_cgroup_path_v1(const char *subsystem) + { + return common_get_cgroup_path("/proc/1/cgroup", subsystem); +@@ -1071,5 +1230,6 @@ int cgroup_v1_ops_init(cgroup_ops *ops) + ops->get_cgroup_mnt_and_root_path = get_cgroup_mnt_and_root_path_v1; + ops->get_init_cgroup_path = get_init_cgroup_path_v1; + ops->get_own_cgroup_path = get_own_cgroup_v1; ++ ops->get_cgroup_oom_handler = get_cgroup_oom_handler_v1; + return 0; + } +\ No newline at end of file +diff --git a/src/daemon/common/cgroup/cgroup_v2.c b/src/daemon/common/cgroup/cgroup_v2.c +index 65cf90d8..a36258f0 100644 +--- a/src/daemon/common/cgroup/cgroup_v2.c ++++ b/src/daemon/common/cgroup/cgroup_v2.c +@@ -17,12 +17,14 @@ + #include + #include + #include ++#include + + #include + + #include "utils.h" + #include "path.h" + #include "sysinfo.h" ++#include "events_sender_api.h" + + // Cgroup V2 Item Definition + #define CGROUP2_CPU_WEIGHT "cpu.weight" +@@ -408,10 +410,143 @@ static int get_cgroup_metrics_v2(const char *cgroup_path, cgroup_metrics_t *cgro + + static int get_cgroup_mnt_and_root_v2(const char *subsystem, char **mountpoint, char **root) + { +- *mountpoint = util_strdup_s(CGROUP_ISULAD_PATH); ++ if (mountpoint != NULL) { ++ *mountpoint = util_strdup_s(CGROUP_ISULAD_PATH); ++ } + return 0; + } + ++static bool oom_cb_cgroup_v2(int fd, void *cbdata) ++{ ++ const size_t events_size = sizeof(struct inotify_event) + NAME_MAX + 1; ++ char events[events_size]; ++ cgroup_oom_handler_info_t *info = (cgroup_oom_handler_info_t *)cbdata; ++ ++ if (info == NULL) { ++ ERROR("Invalid callback data"); ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ ssize_t num_read = util_read_nointr(fd, &events, events_size); ++ if (num_read < 0) { ++ ERROR("Failed to read oom event from eventfd in v2"); ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ if (((struct inotify_event *)events)->mask & ( IN_DELETE | IN_DELETE_SELF)) { ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ __isula_auto_file FILE *fp = fopen(info->cgroup_memory_event_path, "re"); ++ if (fp == NULL) { ++ ERROR("Failed to open cgroups file: %s", info->cgroup_memory_event_path); ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ __isula_auto_free char *line = NULL; ++ size_t len = 0; ++ ssize_t read; ++ while ((read = getline(&line, &len, fp)) != -1) { ++ int count; ++ const char *oom_str = "oom "; ++ const char *oom_kill_str = "oom_kill "; ++ const int oom_len = strlen(oom_str), oom_kill_len = strlen(oom_kill_str); ++ ++ if (read >= oom_kill_len + 2 && memcmp(line, oom_kill_str, oom_kill_len) == 0) { ++ len = oom_kill_len; ++ } else if (read >= oom_len + 2 && memcmp(line, oom_str, oom_len) == 0) { ++ len = oom_len; ++ } else { ++ continue; ++ } ++ ++ // to make use of util_safe_int, it requires it ends with '\0' ++ line[strcspn(line, "\n")] = '\0'; ++ if (util_safe_int(&line[len], &count) < 0) { ++ ERROR("Failed to parse: %s", &line[len]); ++ continue; ++ } ++ ++ if (count == 0) { ++ continue; ++ } ++ ++ INFO("OOM event detected in cgroup v2"); ++ (void)isulad_monitor_send_container_event(info->name, OOM, -1, 0, NULL, NULL); ++ ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ return CGROUP_OOM_HANDLE_CONTINUE; ++} ++ ++static char *get_real_cgroup_path_v2(const char *cgroup_path) ++{ ++ __isula_auto_free char *converted_cgroup_path = NULL; ++ converted_cgroup_path = common_convert_cgroup_path(cgroup_path); ++ if (converted_cgroup_path == NULL) { ++ ERROR("Failed to convert cgroup path"); ++ return NULL; ++ } ++ ++ return util_path_join(CGROUP_MOUNTPOINT, converted_cgroup_path); ++} ++ ++cgroup_oom_handler_info_t *get_cgroup_oom_handler_v2(int fd, const char *name, const char *cgroup_path, const char *exit_fifo) ++{ ++ __isula_auto_free char *real_cgroup_path = NULL; ++ if (name == NULL || cgroup_path == NULL || exit_fifo == NULL) { ++ ERROR("Invalid arguments"); ++ return NULL; ++ } ++ ++ cgroup_oom_handler_info_t *info = util_common_calloc_s(sizeof(cgroup_oom_handler_info_t)); ++ if (info == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ info->name = util_strdup_s(name); ++ info->oom_event_fd = -1; ++ info->cgroup_file_fd = -1; ++ info->oom_event_handler = oom_cb_cgroup_v2; ++ ++ real_cgroup_path = get_real_cgroup_path_v2(cgroup_path); ++ if (real_cgroup_path == NULL) { ++ ERROR("Failed to transfer cgroup path: %s", cgroup_path); ++ goto cleanup; ++ } ++ ++ info->cgroup_memory_event_path = util_path_join(real_cgroup_path, "memory.events"); ++ if (info->cgroup_memory_event_path == NULL) { ++ ERROR("Failed to join path"); ++ goto cleanup; ++ } ++ ++ if ((info->oom_event_fd = inotify_init()) < 0) { ++ ERROR("Failed to init inotify fd"); ++ goto cleanup; ++ } ++ ++ if (inotify_add_watch(info->oom_event_fd, info->cgroup_memory_event_path, IN_MODIFY) < 0) { ++ ERROR("Failed to watch inotify fd for %s", info->cgroup_memory_event_path); ++ goto cleanup; ++ } ++ ++ // watch exit fifo for container exit, so we can close the inotify fd ++ // because inotify cannot watch cgroup file delete event ++ if (inotify_add_watch(info->oom_event_fd, exit_fifo, IN_DELETE | IN_DELETE_SELF) < 0) { ++ ERROR("Failed to watch inotify fd for %s", exit_fifo); ++ goto cleanup; ++ } ++ ++ return info; ++ ++cleanup: ++ common_free_cgroup_oom_handler_info(info); ++ return NULL; ++} ++ + int get_cgroup_version_v2() + { + return CGROUP_VERSION_2; +@@ -426,5 +561,6 @@ int cgroup_v2_ops_init(cgroup_ops *ops) + ops->get_cgroup_info = get_cgroup_info_v2; + ops->get_cgroup_metrics = get_cgroup_metrics_v2; + ops->get_cgroup_mnt_and_root_path = get_cgroup_mnt_and_root_v2; ++ ops->get_cgroup_oom_handler = get_cgroup_oom_handler_v2; + return 0; + } +\ No newline at end of file +diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +index 47a33c2c..cac5c0ba 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +@@ -1055,6 +1055,9 @@ void ContainerManagerService::UpdateBaseStatusFromInspect( + } else { // Case 3 + state = runtime::v1::CONTAINER_CREATED; + } ++ if (inspect->state->oom_killed) { ++ reason = "OOMKilled"; ++ } + if (inspect->state->error != nullptr) { + message = inspect->state->error; + } +diff --git a/src/daemon/modules/api/container_api.h b/src/daemon/modules/api/container_api.h +index 43d66d64..830fd696 100644 +--- a/src/daemon/modules/api/container_api.h ++++ b/src/daemon/modules/api/container_api.h +@@ -221,6 +221,8 @@ void container_state_set_restarting(container_state_t *s, int exit_code); + void container_state_set_paused(container_state_t *s); + void container_state_reset_paused(container_state_t *s); + ++void container_state_set_oom_killed(container_state_t *s); ++ + void container_state_set_dead(container_state_t *s); + + void container_state_increase_restart_count(container_state_t *s); +@@ -269,8 +271,7 @@ bool container_is_valid_state_string(const char *state); + + void container_update_health_monitor(const char *container_id); + +-extern int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_info, const char *name, +- const char *runtime, bool sandbox_container); ++extern int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const pid_ppid_info_t *pid_info, const container_t *cont); + + extern char *container_exit_fifo_create(const char *cont_state_path); + +diff --git a/src/daemon/modules/container/container_events_handler.c b/src/daemon/modules/container/container_events_handler.c +index b84f1ad5..109a628c 100644 +--- a/src/daemon/modules/container/container_events_handler.c ++++ b/src/daemon/modules/container/container_events_handler.c +@@ -114,7 +114,7 @@ static int container_state_changed(container_t *cont, const struct isulad_events + bool has_been_manually_stopped = false; + + /* only handle Exit event */ +- if (events->type != EVENTS_TYPE_STOPPED1) { ++ if (events->type != EVENTS_TYPE_STOPPED1 && events->type != EVENTS_TYPE_OOM) { + return 0; + } + +@@ -187,6 +187,16 @@ static int container_state_changed(container_t *cont, const struct isulad_events + } + + break; ++ ++ case EVENTS_TYPE_OOM: { ++ container_lock(cont); ++ container_state_set_oom_killed(cont->state); ++ if (container_state_to_disk(cont)) { ++ WARN("Failed to save container \"%s\" to disk", id); ++ } ++ container_unlock(cont); ++ break; ++ } + default: + /* ignore garbage */ + break; +diff --git a/src/daemon/modules/container/container_state.c b/src/daemon/modules/container/container_state.c +index f31959fa..452a2b26 100644 +--- a/src/daemon/modules/container/container_state.c ++++ b/src/daemon/modules/container/container_state.c +@@ -154,6 +154,7 @@ void container_state_set_running(container_state_t *s, const pid_ppid_info_t *pi + state->paused = false; + } + state->exit_code = 0; ++ state->oom_killed = false; + + if (pid_info != NULL) { + state->pid = pid_info->pid; +@@ -222,6 +223,19 @@ void container_state_set_paused(container_state_t *s) + container_state_unlock(s); + } + ++void container_state_set_oom_killed(container_state_t *s) ++{ ++ if (s == NULL || s->state == NULL) { ++ return; ++ } ++ ++ container_state_lock(s); ++ ++ s->state->oom_killed = true; ++ ++ container_state_unlock(s); ++} ++ + /* state reset paused */ + void container_state_reset_paused(container_state_t *s) + { +@@ -573,6 +587,7 @@ container_inspect_state *container_state_to_inspect_state(container_state_t *s) + state->running = s->state->running; + state->paused = s->state->paused; + state->restarting = s->state->restarting; ++ state->oom_killed = s->state->oom_killed; + state->pid = s->state->pid; + + state->exit_code = s->state->exit_code; +diff --git a/src/daemon/modules/container/restore/restore.c b/src/daemon/modules/container/restore/restore.c +index 76868e28..52f68d21 100644 +--- a/src/daemon/modules/container/restore/restore.c ++++ b/src/daemon/modules/container/restore/restore.c +@@ -24,6 +24,7 @@ + #include + #include + #include ++#include + + #include "isulad_config.h" + +@@ -44,6 +45,8 @@ + #include "utils_file.h" + #include "utils_timestamp.h" + #include "id_name_manager.h" ++#include "cgroup.h" ++#include "specs_api.h" + + /* restore supervisor */ + static int restore_supervisor(const container_t *cont) +@@ -55,9 +58,7 @@ static int restore_supervisor(const container_t *cont) + char *exit_fifo = NULL; + char *id = cont->common_config->id; + char *statepath = cont->state_path; +- char *runtime = cont->runtime; + pid_ppid_info_t pid_info = { 0 }; +- bool sandbox_container = false; + + nret = snprintf(container_state, sizeof(container_state), "%s/%s", statepath, id); + if (nret < 0 || (size_t)nret >= sizeof(container_state)) { +@@ -91,11 +92,8 @@ static int restore_supervisor(const container_t *cont) + pid_info.ppid = cont->state->state->p_pid; + pid_info.start_time = cont->state->state->start_time; + pid_info.pstart_time = cont->state->state->p_start_time; +-#ifdef ENABLE_CRI_API_V1 +- sandbox_container = is_sandbox_container(cont->common_config->sandbox_info); +-#endif + +- if (container_supervisor_add_exit_monitor(exit_fifo_fd, &pid_info, id, runtime, sandbox_container)) { ++ if (container_supervisor_add_exit_monitor(exit_fifo_fd, exit_fifo, &pid_info, cont)) { + ERROR("Failed to add exit monitor to supervisor"); + ret = -1; + goto out; +diff --git a/src/daemon/modules/container/supervisor/supervisor.c b/src/daemon/modules/container/supervisor/supervisor.c +index 63289283..1b7da383 100644 +--- a/src/daemon/modules/container/supervisor/supervisor.c ++++ b/src/daemon/modules/container/supervisor/supervisor.c +@@ -41,6 +41,8 @@ + #ifdef ENABLE_CRI_API_V1 + #include "sandbox_ops.h" + #endif ++#include "cgroup.h" ++#include "specs_api.h" + + pthread_mutex_t g_supervisor_lock = PTHREAD_MUTEX_INITIALIZER; + struct epoll_descr g_supervisor_descr; +@@ -269,24 +271,52 @@ static int supervisor_exit_cb(int fd, uint32_t events, void *cbdata, struct epol + return EPOLL_LOOP_HANDLE_CONTINUE; + } + ++static int oom_handle_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr) ++{ ++ cgroup_oom_handler_info_t *oom_handler_info = (cgroup_oom_handler_info_t *)cbdata; ++ bool close_oom_handler = CGROUP_OOM_HANDLE_CLOSE; ++ // supervisor only handle one oom event, so we remove the handler directly ++ if (oom_handler_info != NULL && oom_handler_info->oom_event_handler != NULL) { ++ close_oom_handler = oom_handler_info->oom_event_handler(fd, oom_handler_info); ++ } ++ ++ if (close_oom_handler == CGROUP_OOM_HANDLE_CLOSE) { ++ supervisor_handler_lock(); ++ epoll_loop_del_handler(&g_supervisor_descr, fd); ++ supervisor_handler_unlock(); ++ ++ common_free_cgroup_oom_handler_info(oom_handler_info); ++ } ++ ++ return EPOLL_LOOP_HANDLE_CONTINUE; ++} ++ + /* supervisor add exit monitor */ +-int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_info, const char *name, +- const char *runtime, bool sandbox_container) ++int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const pid_ppid_info_t *pid_info, const container_t *cont) + { + int ret = 0; + struct supervisor_handler_data *data = NULL; ++ cgroup_oom_handler_info_t *oom_handler_info = NULL; ++ __isula_auto_free char *cgroup_path = NULL; + + if (fd < 0) { + ERROR("Invalid exit fifo fd"); + return -1; + } + +- if (pid_info == NULL || name == NULL || runtime == NULL) { ++ if (pid_info == NULL || cont == NULL || cont->common_config == NULL) { + ERROR("Invalid input arguments"); + close(fd); + return -1; + } + ++ cgroup_path = merge_container_cgroups_path(cont->common_config->id, cont->hostconfig); ++ if (cgroup_path == NULL) { ++ ERROR("Failed to get cgroup path"); ++ close(fd); ++ return -1; ++ } ++ + data = util_common_calloc_s(sizeof(struct supervisor_handler_data)); + if (data == NULL) { + ERROR("Memory out"); +@@ -295,15 +325,26 @@ int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_inf + } + + data->fd = fd; +- data->name = util_strdup_s(name); +- data->runtime = util_strdup_s(runtime); +- data->is_sandbox_container = sandbox_container; ++ data->name = util_strdup_s(cont->common_config->id); ++ data->runtime = util_strdup_s(cont->runtime); ++#ifdef ENABLE_CRI_API_V1 ++ data->is_sandbox_container = is_sandbox_container(cont->common_config->sandbox_info); ++#endif + data->pid_info.pid = pid_info->pid; + data->pid_info.start_time = pid_info->start_time; + data->pid_info.ppid = pid_info->ppid; + data->pid_info.pstart_time = pid_info->pstart_time; ++ oom_handler_info = common_get_cgroup_oom_handler(fd, cont->common_config->id, cgroup_path, exit_fifo); + + supervisor_handler_lock(); ++ if (oom_handler_info != NULL) { ++ ret = epoll_loop_add_handler(&g_supervisor_descr, oom_handler_info->oom_event_fd, oom_handle_cb, oom_handler_info); ++ if (ret != 0) { ++ ERROR("Failed to add handler for oom event"); ++ goto err; ++ } ++ } ++ + ret = epoll_loop_add_handler(&g_supervisor_descr, fd, supervisor_exit_cb, data); + if (ret != 0) { + ERROR("Failed to add handler for exit fifo"); +@@ -314,6 +355,7 @@ int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_inf + + err: + supervisor_handler_data_free(data); ++ common_free_cgroup_oom_handler_info(oom_handler_info); + out: + supervisor_handler_unlock(); + return ret; +diff --git a/src/daemon/modules/events/collector.c b/src/daemon/modules/events/collector.c +index fb4a7fea..af688742 100644 +--- a/src/daemon/modules/events/collector.c ++++ b/src/daemon/modules/events/collector.c +@@ -133,6 +133,9 @@ static container_events_type_t lcrsta2Evetype(int value) + case THAWED: + et = EVENTS_TYPE_THAWED; + break; ++ case OOM: ++ et = EVENTS_TYPE_OOM; ++ break; + default: + et = EVENTS_TYPE_EXIT; + break; +@@ -822,8 +825,8 @@ static int post_event_to_events_hander(const struct isulad_events_format *events + return -1; + } + +- /* only post STOPPED event to events_hander */ +- if (events->type != EVENTS_TYPE_STOPPED1) { ++ /* only post STOPPED event and OOM event to events_hander */ ++ if (events->type != EVENTS_TYPE_STOPPED1 && events->type != EVENTS_TYPE_OOM) { + return 0; + } + +diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c +index a8090d5a..eb7ce4f4 100644 +--- a/src/daemon/modules/service/service_container.c ++++ b/src/daemon/modules/service/service_container.c +@@ -275,14 +275,13 @@ static void clean_resources_on_failure(const container_t *cont, const char *engi + return; + } + +-static int do_post_start_on_success(const char *id, const char *runtime, bool sandbox_container, +- const char *pidfile, int exit_fifo_fd, +- const pid_ppid_info_t *pid_info) ++static int do_post_start_on_success(container_t *cont, int exit_fifo_fd, ++ const char *exit_fifo, const pid_ppid_info_t *pid_info) + { + int ret = 0; + + // exit_fifo_fd was closed in container_supervisor_add_exit_monitor +- if (container_supervisor_add_exit_monitor(exit_fifo_fd, pid_info, id, runtime, sandbox_container)) { ++ if (container_supervisor_add_exit_monitor(exit_fifo_fd, exit_fifo, pid_info, cont)) { + ERROR("Failed to add exit monitor to supervisor"); + ret = -1; + } +@@ -750,7 +749,6 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo + oci_runtime_spec *oci_spec = NULL; + rt_create_params_t create_params = { 0 }; + rt_start_params_t start_params = { 0 }; +- bool sandbox_container = false; + + nret = snprintf(bundle, sizeof(bundle), "%s/%s", cont->root_path, id); + if (nret < 0 || (size_t)nret >= sizeof(bundle)) { +@@ -899,7 +897,6 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo + if (cont->common_config->sandbox_info != NULL) { + create_params.task_addr = cont->common_config->sandbox_info->task_address; + } +- sandbox_container = is_sandbox_container(cont->common_config->sandbox_info); + #endif + + if (runtime_create(id, runtime, &create_params) != 0) { +@@ -924,7 +921,7 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo + + ret = runtime_start(id, runtime, &start_params, pid_info); + if (ret == 0) { +- if (do_post_start_on_success(id, runtime, sandbox_container, pidfile, exit_fifo_fd, pid_info) != 0) { ++ if (do_post_start_on_success(cont, exit_fifo_fd, exit_fifo, pid_info) != 0) { + ERROR("Failed to do post start on runtime start success"); + ret = -1; + goto clean_resources; +-- +2.34.1 + diff --git a/0036-add-ci-cases-for-oomkilled-monitor.patch b/0036-add-ci-cases-for-oomkilled-monitor.patch new file mode 100644 index 0000000..8f4f67e --- /dev/null +++ b/0036-add-ci-cases-for-oomkilled-monitor.patch @@ -0,0 +1,279 @@ +From 0111a575f829b946068dcb11286f0d84363cfc3d Mon Sep 17 00:00:00 2001 +From: jikai +Date: Thu, 28 Mar 2024 12:51:53 +0000 +Subject: [PATCH 36/43] add ci cases for oomkilled monitor + +Signed-off-by: jikai +--- + CI/test_cases/container_cases/inspect.sh | 14 ++++++++ + test/cgroup/cpu/CMakeLists.txt | 2 ++ + .../image/oci/oci_config_merge/CMakeLists.txt | 1 + + test/image/oci/registry/CMakeLists.txt | 1 + + test/mocks/sender_mock.cc | 34 +++++++++++++++++++ + test/mocks/sender_mock.h | 31 +++++++++++++++++ + test/network/network_ns/CMakeLists.txt | 1 + + test/runtime/isula/CMakeLists.txt | 1 + + test/runtime/lcr/CMakeLists.txt | 1 + + test/specs/specs/CMakeLists.txt | 1 + + test/specs/specs_extend/CMakeLists.txt | 1 + + test/specs/verify/CMakeLists.txt | 1 + + test/volume/CMakeLists.txt | 3 +- + 13 files changed, 91 insertions(+), 1 deletion(-) + create mode 100644 test/mocks/sender_mock.cc + create mode 100644 test/mocks/sender_mock.h + +diff --git a/CI/test_cases/container_cases/inspect.sh b/CI/test_cases/container_cases/inspect.sh +index cde9ea1f..b4f4a785 100755 +--- a/CI/test_cases/container_cases/inspect.sh ++++ b/CI/test_cases/container_cases/inspect.sh +@@ -27,6 +27,7 @@ function test_inspect_spec() + { + local ret=0 + local image="busybox" ++ local ubuntu_image="ubuntu" + local test="container inspect test => (${FUNCNAME[@]})" + + msg_info "${test} starting..." +@@ -37,6 +38,12 @@ function test_inspect_spec() + isula images | grep busybox + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++)) + ++ isula pull ${ubuntu_image} ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${ubuntu_image}" && return ${FAILURE} ++ ++ isula images | grep ubuntu ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${ubuntu_image}" && ((ret++)) ++ + containername=test_inspect + + isula create --name $containername --ipc host --pid host --uts host --restart=on-failure:10 --hook-spec ${test_data_path}/test-hookspec.json --cpu-shares 100 --memory 5MB --memory-reservation 4MB --cpu-period 1000000 --cpu-quota 200000 --cpuset-cpus 1 --cpuset-mems 0 --kernel-memory 50M --pids-limit=10000 --volume /home:/root --env a=1 $image /bin/sh ls +@@ -139,6 +146,13 @@ function test_inspect_spec() + + isula rm -f $containername + ++ isula run -it -m 4m --name $containername $ubuntu_image perl -e 'for ($i = 0; $i < 100000000; $i++) { $a .= " " x 1024 }' ++ ++ isula inspect -f "{{json .State.OOMKilled}} {{.Name}}" $containername 2>&1 | sed -n '1p' | grep "true" ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container with image: ${ubuntu_image}" && ((ret++)) ++ ++ isula rm -f $containername ++ + msg_info "${test} finished with return ${ret}..." + return ${ret} + } +diff --git a/test/cgroup/cpu/CMakeLists.txt b/test/cgroup/cpu/CMakeLists.txt +index 30bfc417..9c3cfa12 100644 +--- a/test/cgroup/cpu/CMakeLists.txt ++++ b/test/cgroup/cpu/CMakeLists.txt +@@ -13,6 +13,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/command_parser.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/daemon_arguments.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/isulad_config.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc + cgroup_cpu_ut.cc) + + target_include_directories(${EXE} PUBLIC +@@ -23,6 +24,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/api + ${CMAKE_BINARY_DIR}/conf + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd +diff --git a/test/image/oci/oci_config_merge/CMakeLists.txt b/test/image/oci/oci_config_merge/CMakeLists.txt +index d13ec738..ffd3999d 100644 +--- a/test/image/oci/oci_config_merge/CMakeLists.txt ++++ b/test/image/oci/oci_config_merge/CMakeLists.txt +@@ -35,6 +35,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/containers_store_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/namespace_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/container_unix_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/sender_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/spec/parse_volume.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/spec/specs.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/spec/parse_volume.c +diff --git a/test/image/oci/registry/CMakeLists.txt b/test/image/oci/registry/CMakeLists.txt +index 5b5bc3f5..6166c2d0 100644 +--- a/test/image/oci/registry/CMakeLists.txt ++++ b/test/image/oci/registry/CMakeLists.txt +@@ -44,6 +44,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../mocks/storage_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../mocks/oci_image_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../mocks/http_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../mocks/sender_mock.cc + registry_ut.cc) + + target_include_directories(${EXE} PUBLIC +diff --git a/test/mocks/sender_mock.cc b/test/mocks/sender_mock.cc +new file mode 100644 +index 00000000..26028d7f +--- /dev/null ++++ b/test/mocks/sender_mock.cc +@@ -0,0 +1,34 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-29 ++ * Description: provide collector mock ++ ******************************************************************************/ ++ ++#include "sender_mock.h" ++ ++namespace { ++MockEventSender *g_sender_mock = nullptr; ++} ++ ++void MockEventSender_SetMock(MockEventSender *mock) ++{ ++ g_sender_mock = mock; ++} ++ ++int isulad_monitor_send_container_event(const char *name, runtime_state_t state, int pid, int exit_code, ++ const char *args, const char *extra_annations) ++{ ++ if (g_sender_mock != nullptr) { ++ return g_sender_mock->IsuladMonitorEventSendContainerEvent(name, state, pid, exit_code, args, extra_annations); ++ } ++ return 0; ++} +diff --git a/test/mocks/sender_mock.h b/test/mocks/sender_mock.h +new file mode 100644 +index 00000000..f4fe75f0 +--- /dev/null ++++ b/test/mocks/sender_mock.h +@@ -0,0 +1,31 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-30 ++ * Description: provide sender mock ++ ******************************************************************************/ ++ ++#ifndef _ISULAD_TEST_MOCKS_SENDER_MOCK_H ++#define _ISULAD_TEST_MOCKS_SENDER_MOCK_H ++ ++#include ++#include "events_sender_api.h" ++ ++class MockEventSender { ++public: ++ MOCK_METHOD6(IsuladMonitorEventSendContainerEvent, int(const char *name, runtime_state_t state, int pid, int exit_code, ++ const char *args, const char *extra_annations)); ++}; ++ ++void MockEventSender_SetMock(MockEventSender *mock); ++ ++#endif ++ +diff --git a/test/network/network_ns/CMakeLists.txt b/test/network/network_ns/CMakeLists.txt +index 71b8039d..6f3f36a0 100644 +--- a/test/network/network_ns/CMakeLists.txt ++++ b/test/network/network_ns/CMakeLists.txt +@@ -43,6 +43,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/selinux_label_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/isulad_config_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc + network_ns_ut.cc) + + target_include_directories(${EXE} PUBLIC +diff --git a/test/runtime/isula/CMakeLists.txt b/test/runtime/isula/CMakeLists.txt +index c1f0a5cc..15636623 100644 +--- a/test/runtime/isula/CMakeLists.txt ++++ b/test/runtime/isula/CMakeLists.txt +@@ -31,6 +31,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/engine_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/isulad_config_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/runtime/isula/isula_rt_ops.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc + isula_rt_ops_ut.cc) + + target_include_directories(${EXE} PUBLIC +diff --git a/test/runtime/lcr/CMakeLists.txt b/test/runtime/lcr/CMakeLists.txt +index c3b93d67..5b2ed11a 100644 +--- a/test/runtime/lcr/CMakeLists.txt ++++ b/test/runtime/lcr/CMakeLists.txt +@@ -29,6 +29,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/namespace_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/container_unix_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/engine_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/isulad_config_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c + lcr_rt_ops_ut.cc) +diff --git a/test/specs/specs/CMakeLists.txt b/test/specs/specs/CMakeLists.txt +index 45f688f9..12c11f51 100644 +--- a/test/specs/specs/CMakeLists.txt ++++ b/test/specs/specs/CMakeLists.txt +@@ -43,6 +43,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/isulad_config_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/image_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc + specs_ut.cc) + + target_include_directories(${EXE} PUBLIC +diff --git a/test/specs/specs_extend/CMakeLists.txt b/test/specs/specs_extend/CMakeLists.txt +index 1b737089..2fd37e1c 100644 +--- a/test/specs/specs_extend/CMakeLists.txt ++++ b/test/specs/specs_extend/CMakeLists.txt +@@ -43,6 +43,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/isulad_config_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/image_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc + specs_extend_ut.cc) + + target_include_directories(${EXE} PUBLIC +diff --git a/test/specs/verify/CMakeLists.txt b/test/specs/verify/CMakeLists.txt +index b0602127..7f000cd1 100644 +--- a/test/specs/verify/CMakeLists.txt ++++ b/test/specs/verify/CMakeLists.txt +@@ -38,6 +38,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/image_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc + verify_ut.cc) + + target_include_directories(${EXE} PUBLIC +diff --git a/test/volume/CMakeLists.txt b/test/volume/CMakeLists.txt +index 27d07330..1f9dac03 100644 +--- a/test/volume/CMakeLists.txt ++++ b/test/volume/CMakeLists.txt +@@ -24,6 +24,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup/cgroup_v1.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup/cgroup_v2.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup/cgroup_common.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../test/mocks/sender_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/cmd/command_parser.c + volume_ut.cc) + +@@ -43,6 +44,6 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/utils/console + ) + +-target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) ++target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) + set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +-- +2.34.1 + diff --git a/0037-add-cgroup-v2-doc.patch b/0037-add-cgroup-v2-doc.patch new file mode 100644 index 0000000..bfd7530 --- /dev/null +++ b/0037-add-cgroup-v2-doc.patch @@ -0,0 +1,252 @@ +From 2cbce684b973bf4e250f41d750253b5b8abde32d Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Tue, 20 Feb 2024 19:22:11 +0800 +Subject: [PATCH 37/43] add cgroup v2 doc + +Signed-off-by: zhongtao +--- + docs/design/README_zh.md | 2 + + .../detailed/Container/cgroup_v2_design_zh.md | 193 ++++++++++++++++++ + docs/images/cgroup_v2_module.svg | 16 ++ + 3 files changed, 211 insertions(+) + create mode 100644 docs/design/detailed/Container/cgroup_v2_design_zh.md + create mode 100644 docs/images/cgroup_v2_module.svg + +diff --git a/docs/design/README_zh.md b/docs/design/README_zh.md +index f2c187a1..b7ec3ddb 100644 +--- a/docs/design/README_zh.md ++++ b/docs/design/README_zh.md +@@ -18,6 +18,8 @@ + + - 查看 restart 模块的设计文档： [restart_manager_design](./detailed/Container/restart_manager_design.md)。 + ++- 查看 cgroup v2 的设计文档： [cgroup_v2_design](./detailed/Container/cgroup_v2_design_zh.md)。 ++ + ## CRI + + - 查看 CRI的启动程序的重构文档： [cri_cni_refactor](./detailed/CRI/cri_cni_refactor_zh.md) 。 +diff --git a/docs/design/detailed/Container/cgroup_v2_design_zh.md b/docs/design/detailed/Container/cgroup_v2_design_zh.md +new file mode 100644 +index 00000000..e1ce81d0 +--- /dev/null ++++ b/docs/design/detailed/Container/cgroup_v2_design_zh.md +@@ -0,0 +1,193 @@ ++| Author | zhongtao | ++| ------ | --------------------- | ++| Date | 2024-02-19 | ++| Email | zhongtao17@huawei.com | ++# 方案目标 ++ ++cgroup是linux中用于限制进程组资源的功能。cgroup目前包括两个版本，cgroup v1和cgroup v2。cgroup v2的目标是取代cgroup v1，出于兼容性的考虑，cgroup v1并没有在内核中删除，并且大概率会长期存在。该需求的目的为使得iSulad支持cgroup v2. ++ ++## 与cgroup v1差异 ++无论是cgroup v1还是cgroup v2，iSulad提供给用户使用的接口都是一致的。不过由于有部分cgroup v1支持的功能在cgroup v2中被去掉了或者实现方式有所变化，因此部分接口在cgroup v2中不可用或者含义发生变化。iSulad支持限制如下资源： ++ ++|资源|功能|和cgroup v1的差异| ++|---|---|---| ++|devices|限制对应的设备是否可以在容器中访问以及访问权限|devcies子系统不再使用往cgroup文件里写值的方式进行限制，而是采用ebpf的方式进行限制| ++|memory|限制容器的内存资源|不支持swappiness，不支持kmem相关参数，不支持oom_control| ++|cpu/cpuset|限制容器的cpu资源|不支持rt_*相关(实时线程)的限制| ++|blkio/io|限制容器的块设备io|不仅限制块设备的IO，也能限制buffer IO| ++|hugetlb|限制大页内存的使用|无差异| ++|pids|限制容器使用的pid|无差异| ++|files|限制容器使用的fd|无差异| ++|freeze|暂停容器|无差异| ++ ++## 使用方式 ++ ++使用的示例如下： ++ ++1. 以限制内存资源为例，假设我们需要限制单个容器最多使用10M内存，则可以在运行容器时加上-m 10m参数进行限制： ++ ++ ```sh ++ [root@openEuler iSulad]# isula run -tid -m 10m busybox sh ++ 000c0c6eb609179062b19a3d2de4d7c38a42c887f55e2a7759ed9df851277163 ++ ``` ++ ++ -m 10m表示限制容器内最多只能使用10m内存，可以通过isula stats命令查看资源的限制情况： ++ ++ ```sh ++ [root@openEuler iSulad]# isula stats --no-stream 000c0c6eb6 ++ CONTAINER CPU % MEM USAGE / LIMIT MEM % BLOCK I / O PIDS ++ 000c0c6eb609 0.00 104.00 KiB / 10.00 MiB 1.02 0.00 B / 0.00 B 1 ++ ``` ++ ++ 可以动态更新资源的限制： ++ ```sh ++ [root@openEuler iSulad]# isula update -m 20m 000c0c6eb6 ++ 000c0c6eb6 ++ [root@openEuler iSulad]# isula stats --no-stream 000c0c6eb6 ++ CONTAINER CPU % MEM USAGE / LIMIT MEM % BLOCK I / O PIDS ++ 000c0c6eb609 0.00 104.00 KiB / 20.00 MiB 0.51 0.00 B / 0.00 B 1 ++ ``` ++ ++2. 假设我们要将设备/dev/sda挂载到容器中成为/dev/sdx并限制为只读设备，则可以这么配置： ++ ++```sh ++ [root@openEuler iSulad]# isula run -ti --rm --device=/dev/sda:/dev/sdx:wm busybox fdisk /dev/sdx ++ fdisk: can't open '/dev/sdx' ++ [root@openEuler iSulad]# ++``` ++ ++挂载设备到容器的语法为`--device=$host:$container:rwm $host`指定设备在主机上的绝对路径，$container指定设备在容器内的绝对路径，r表示可读，w表示可写，m表示可以创建node 上述命令中rwm三个参数缺少r参数，也就是说允许写和创建node但是不允许读(即只读)。 ++ ++3. 使用cri的PodSandboxStats接口与ContainerStats接口获取容器的资源使用状况： ++ ++```sh ++[root@openEuler ~]# crictl statsp c3 ++ POD POD ID CPU % MEM ++ test-sandbox c32556d3bb139 0.00 196.6kB ++[root@openEuler ~]# crictl statsp --output json c3 ++...... ++ "linux": { ++ "cpu": { ++ "timestamp": "1708499622485777700", ++ "usageCoreNanoSeconds": { ++ "value": "180973" ++ }, ++ "usageNanoCores": null ++ }, ++ "memory": { ++ "timestamp": "1708499622485777700", ++ "workingSetBytes": { ++ "value": "196608" ++ }, ++ "availableBytes": { ++ "value": "0" ++ }, ++ "usageBytes": { ++ "value": "4386816" ++ }, ++ "rssBytes": { ++ "value": "176128" ++ }, ++ "pageFaults": { ++ "value": "1193" ++ }, ++ "majorPageFaults": { ++ "value": "6" ++ } ++ }, ++ "network": null, ++ "process": { ++ "timestamp": "1708499622485777700", ++ "processCount": { ++ "value": "2" ++ } ++ }, ++..... ++ ++[root@openEuler ~]# crictl stats 01 ++CONTAINER CPU % MEM DISK INODES ++01a726f61c5c3 0.01 3.801MB 16.4kB 8 ++[root@openEuler ~]# ++``` ++ ++# 总体设计 ++ ++在原有只支持cgroup v1的基础上，对cgroup模块进行了重构，重构后的架构图如下： ++ ++![cgroup_v2_module](../../../images/cgroup_v2_module.svg) ++ ++主要功能为以下两种： ++ ++1. iSulad在资源控制参数设置和更新过程中，负责参数的合法校验，用于拦截非法请求，真正的cgroup操作由容器运行时完成。 ++ ++2. iSulad在获得sandbox资源使用信息时，直接读取sandbox cgroup文件信息。 ++ ++# 接口描述 ++由于无论是cgroup v1还是cgroup v2，iSulad提供给用户使用的接口都是一致的。 ++无新增接口。 ++ ++```c ++int verify_container_settings(const oci_runtime_spec *container, const sysinfo_t *sysinfo); ++ ++int verify_host_config_settings(host_config *hostconfig, const sysinfo_t *sysinfo, bool update); ++ ++ ++typedef struct { ++ int (*get_cgroup_info)(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, ++ cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, ++ cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, ++ cgroup_files_info_t *filesinfo, bool quiet); ++ int (*get_cgroup_metrics)(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics); ++ ++ int (*common_find_cgroup_mnt_and_root)(const char *subsystem, char **mountpoint, char **root); ++ ++ char *(*sysinfo_cgroup_controller_cpurt_mnt_path)(void); ++} cgroup_ops; ++ ++int cgroup_v2_ops_init(cgroup_ops *ops) ++{ ++ if (ops == NULL) { ++ return -1; ++ } ++ ops->get_cgroup_info = common_get_cgroup_info_v2; ++ ops->get_cgroup_metrics = common_get_cgroup_v2_metrics; ++ ops->common_find_cgroup_mnt_and_root = common_find_cgroup_v2_mnt_and_root; ++ return 0; ++} ++``` ++ ++# 详细设计 ++ ++```mermaid ++sequenceDiagram ++ participant isula ++ participant kubelet ++ participant isulad ++ participant runc ++ participant cgroup ++ ++ isula->>isulad: request ++ kubelet->>isulad:request ++ alt run/create/update ++ isulad->>isulad:verify request option ++ isulad->>runc:run/create/update request ++ runc ->> cgroup:write cgroup file ++ else stats ++ par container stats ++ isulad->>runc:container stats request ++ runc ->> cgroup:read cgroup file ++ runc ->> isulad:container stats info ++ and sandbox stats ++ isulad ->> cgroup: read cgroup file ++ end ++ end ++ isulad ->> isula:response ++ isulad ->> kubelet:response ++``` ++ ++# 使用限制 ++ ++1. 只支持cgroup 挂载点在/sys/fs/cgroup ++2. cgroup v1与cgrooup v2混用场景不支持 ++3. 该需求只涉及cgroup v2对runc容器运行时的支持 ++ +diff --git a/docs/images/cgroup_v2_module.svg b/docs/images/cgroup_v2_module.svg +new file mode 100644 +index 00000000..59e7939b +--- /dev/null ++++ b/docs/images/cgroup_v2_module.svg +@@ -0,0 +1,16 @@ ++ ++ ++ ++ ++ ++ ++ CLICRICRI moduleverify modulecgroup moduleisulad-> main.ccgroup_ops_initcgroup_v1 modulecgroup_v2 moduleruntime moduleruncisula modulecommon cgroup apicgroup fileread/writeread/write +\ No newline at end of file +-- +2.34.1 + diff --git a/0038-add-modify-for-cgroup-v2-ci-test.patch b/0038-add-modify-for-cgroup-v2-ci-test.patch new file mode 100644 index 0000000..843652a --- /dev/null +++ b/0038-add-modify-for-cgroup-v2-ci-test.patch @@ -0,0 +1,608 @@ +From 33b26f27dd897574d73ce8654620a13edbeb947e Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Tue, 2 Apr 2024 02:31:58 +1400 +Subject: [PATCH 38/69] add modify for cgroup v2 ci test + +Signed-off-by: zhongtao +--- + CI/test_cases/manual_cases/cgroupv2.sh | 276 +++++++++++++++---------- + 1 file changed, 165 insertions(+), 111 deletions(-) + +diff --git a/CI/test_cases/manual_cases/cgroupv2.sh b/CI/test_cases/manual_cases/cgroupv2.sh +index f8982f08..8e431688 100755 +--- a/CI/test_cases/manual_cases/cgroupv2.sh ++++ b/CI/test_cases/manual_cases/cgroupv2.sh +@@ -31,59 +31,59 @@ function test_cgroup2_cpu() + + if [[ -f /sys/fs/cgroup/isulad/cpu.weight ]];then + # min value +- isula run -ti --rm --cpu-shares 2 busybox cat /sys/fs/cgroup/cpu.weight | grep ^1$'\r' ++ isula run --runtime $1 -ti --rm --cpu-shares 2 busybox cat /sys/fs/cgroup/cpu.weight | grep ^1$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.weight min value failed" && ((ret++)) + + # max value +- isula run -ti --rm --cpu-shares 262144 busybox cat /sys/fs/cgroup/cpu.weight | grep ^10000$'\r' ++ isula run --runtime $1 -ti --rm --cpu-shares 262144 busybox cat /sys/fs/cgroup/cpu.weight | grep ^10000$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.weight max value failed" && ((ret++)) + + # invalid value +- isula run -ti --rm --cpu-shares -1 busybox echo hello ++ isula run --runtime $1 -ti --rm --cpu-shares -1 busybox echo hello + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.weight -1 failed" && ((ret++)) + + # default value +- isula run -ti --rm --cpu-shares 0 busybox cat /sys/fs/cgroup/cpu.weight | grep ^100$'\r' ++ isula run --runtime $1 -ti --rm --cpu-shares 0 busybox cat /sys/fs/cgroup/cpu.weight | grep ^100$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.weight default value failed" && ((ret++)) + fi + + if [[ -f /sys/fs/cgroup/isulad/cpu.max ]];then + # normal value +- isula run -ti --rm --cpu-quota 50000 --cpu-period 12345 busybox cat /sys/fs/cgroup/cpu.max | grep ^"50000 12345"$'\r' ++ isula run --runtime $1 -ti --rm --cpu-quota 50000 --cpu-period 12345 busybox cat /sys/fs/cgroup/cpu.max | grep ^"50000 12345"$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.max normal value failed" && ((ret++)) + + # invalid min period +- isula run -ti --rm --cpu-quota 50000 --cpu-period 999 busybox echo hello ++ isula run --runtime $1 -ti --rm --cpu-quota 50000 --cpu-period 999 busybox echo hello + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.max invalid min period failed" && ((ret++)) + + # invalid max period +- isula run -ti --rm --cpu-quota 50000 --cpu-period 1000001 busybox echo hello ++ isula run --runtime $1 -ti --rm --cpu-quota 50000 --cpu-period 1000001 busybox echo hello + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.max invalid max period failed" && ((ret++)) + + # invalid quota +- isula run -ti --rm --cpu-quota 999 --cpu-period 1000000 busybox echo hello ++ isula run --runtime $1 -ti --rm --cpu-quota 999 --cpu-period 1000000 busybox echo hello + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.max invalid quota failed" && ((ret++)) + + # default 0 quota +- isula run -ti --rm --cpu-quota 0 --cpu-period 1000000 busybox cat /sys/fs/cgroup/cpu.max | grep ^"max 1000000"$'\r' ++ isula run --runtime $1 -ti --rm --cpu-quota 0 --cpu-period 1000000 busybox cat /sys/fs/cgroup/cpu.max | grep ^"max 1000000"$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.max default 0 quota failed" && ((ret++)) + + # default -1 quota +- isula run -ti --rm --cpu-quota -1 --cpu-period 1000000 busybox cat /sys/fs/cgroup/cpu.max | grep ^"max 1000000"$'\r' ++ isula run --runtime $1 -ti --rm --cpu-quota -1 --cpu-period 1000000 busybox cat /sys/fs/cgroup/cpu.max | grep ^"max 1000000"$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.max default -1 quota failed" && ((ret++)) + + # cpus 1 +- isula run -ti --rm --cpus 1 busybox cat /sys/fs/cgroup/cpu.max | grep ^"100000 100000"$'\r' ++ isula run --runtime $1 -ti --rm --cpus 1 busybox cat /sys/fs/cgroup/cpu.max | grep ^"100000 100000"$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.max cpus 1 failed" && ((ret++)) + + # cpus 0 +- isula run -ti --rm --cpus 0 busybox cat /sys/fs/cgroup/cpu.max | grep ^"max 100000"$'\r' ++ isula run --runtime $1 -ti --rm --cpus 0 busybox cat /sys/fs/cgroup/cpu.max | grep ^"max 100000"$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.max cpus 0 failed" && ((ret++)) + fi + + if [[ -f /sys/fs/cgroup/isulad/cpuset.cpus.effective ]];then + # normal value +- isula run -tid -n cpuset --cpuset-cpus 0 --cpuset-mems 0 busybox sh ++ isula run --runtime $1 -tid -n cpuset --cpuset-cpus 0 --cpuset-mems 0 busybox sh + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpuset run container failed" && ((ret++)) + + isula exec -ti cpuset cat /sys/fs/cgroup/cpuset.cpus | grep ^0$'\r' +@@ -96,19 +96,19 @@ function test_cgroup2_cpu() + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpuset remove container failed" && ((ret++)) + + # invalid cpus -1 value +- isula run -tid -n cpuset --cpuset-cpus -1 busybox sh ++ isula run --runtime $1 -tid -n cpuset --cpuset-cpus -1 busybox sh + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpuset cpus invalid -1 failed" && ((ret++)) + + # invalid cpus 100000 value +- isula run -tid -n cpuset --cpuset-cpus 100000 busybox sh ++ isula run --runtime $1 -tid -n cpuset --cpuset-cpus 100000 busybox sh + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpuset cpus invalid 100000 failed" && ((ret++)) + + # invalid mems -1 value +- isula run -tid -n cpuset --cpuset-mems -1 busybox sh ++ isula run --runtime $1 -tid -n cpuset --cpuset-mems -1 busybox sh + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpuset mems invalid -1 failed" && ((ret++)) + + # invalid mems 100000 value +- isula run -tid -n cpuset --cpuset-mems 100000 busybox sh ++ isula run --runtime $1 -tid -n cpuset --cpuset-mems 100000 busybox sh + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpuset mems invalid 100000 failed" && ((ret++)) + fi + +@@ -121,33 +121,38 @@ function test_cgroup2_io() + + if [[ -f "/sys/fs/cgroup/isulad/io.bfq.weight" ]];then + # min value +- isula run -ti --rm --blkio-weight 10 busybox cat "/sys/fs/cgroup/io.bfq.weight" | grep 1$'\r' +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.bfq.weight min value failed" && ((ret++)) ++ if [ $1 == "lcr" ]; then ++ isula run --runtime $1 -ti --rm --blkio-weight 10 busybox cat "/sys/fs/cgroup/io.bfq.weight" | grep 1$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.bfq.weight min value failed" && ((ret++)) ++ else ++ isula run --runtime $1 -ti --rm --blkio-weight 10 busybox cat "/sys/fs/cgroup/io.bfq.weight" | grep 10$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.bfq.weight min value failed" && ((ret++)) ++ fi + + # max value +- isula run -ti --rm --blkio-weight 1000 busybox cat "/sys/fs/cgroup/io.bfq.weight" | grep 1000$'\r' ++ isula run --runtime $1 -ti --rm --blkio-weight 1000 busybox cat "/sys/fs/cgroup/io.bfq.weight" | grep 1000$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.bfq.weight max value failed" && ((ret++)) + + # default value +- isula run -ti --rm --blkio-weight 0 busybox cat "/sys/fs/cgroup/io.bfq.weight" | grep 100$'\r' ++ isula run --runtime $1 -ti --rm --blkio-weight 0 busybox cat "/sys/fs/cgroup/io.bfq.weight" | grep 100$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.bfq.weight default value failed" && ((ret++)) + + # invalid value +- isula run -ti --rm --blkio-weight -1 busybox echo hello ++ isula run --runtime $1 -ti --rm --blkio-weight -1 busybox echo hello + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.bfq.weight -1 failed" && ((ret++)) + fi + + if [[ -f "/sys/fs/cgroup/isulad/io.bfq.weight_device" ]];then + # min value +- isula run -ti --rm --blkio-weight-device /dev/null:10 busybox cat "/sys/fs/cgroup/io.bfq.weight_device" | grep ^"1:3 10"$'\r' ++ isula run --runtime $1 -ti --rm --blkio-weight-device /dev/null:10 busybox cat "/sys/fs/cgroup/io.bfq.weight_device" | grep ^"1:3 10"$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.bfq.weight_device max value failed" && ((ret++)) + + # max value +- isula run -ti --rm --blkio-weight-device /dev/null:1000 busybox cat "/sys/fs/cgroup/io.bfq.weight_device" | grep ^"1:3 10000"$'\r' ++ isula run --runtime $1 -ti --rm --blkio-weight-device /dev/null:1000 busybox cat "/sys/fs/cgroup/io.bfq.weight_device" | grep ^"1:3 10000"$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.bfq.weight_device max value failed" && ((ret++)) + + # disable weight device +- isula run -tid -n weight_device --rm --blkio-weight-device /dev/null:0 busybox sh ++ isula run --runtime $1 -tid -n weight_device --rm --blkio-weight-device /dev/null:0 busybox sh + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.bfq.weight_device failed" && ((ret++)) + + isula exec -ti weight_device cat "/sys/fs/cgroup/io.bfq.weight_device" | grep "1:3" +@@ -159,33 +164,43 @@ function test_cgroup2_io() + + if [[ -f "/sys/fs/cgroup/isulad/io.weight" ]];then + # min value +- isula run -ti --rm --blkio-weight 10 busybox cat "/sys/fs/cgroup/io.weight" | grep ^"default 1"$'\r' +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight min value failed" && ((ret++)) ++ if [ $1 == "lcr" ]; then ++ isula run --runtime $1 -ti --rm --blkio-weight 10 busybox cat "/sys/fs/cgroup/io.weight" | grep ^"default 1"$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight min value failed" && ((ret++)) ++ else ++ isula run --runtime $1 -ti --rm --blkio-weight 10 busybox cat "/sys/fs/cgroup/io.bfq.weight" | grep ^"default 10"$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight min value failed" && ((ret++)) ++ fi + + # max value +- isula run -ti --rm --blkio-weight 1000 busybox cat "/sys/fs/cgroup/io.weight" | grep ^"default 10000"$'\r' +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight max value failed" && ((ret++)) ++ if [ $1 == "lcr" ]; then ++ isula run --runtime $1 -ti --rm --blkio-weight 1000 busybox cat "/sys/fs/cgroup/io.weight" | grep ^"default 10000"$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight max value failed" && ((ret++)) ++ else ++ isula run --runtime $1 -ti --rm --blkio-weight 1000 busybox cat "/sys/fs/cgroup/io.bfq.weight" | grep ^"default 1000"$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight max value failed" && ((ret++)) ++ fi + + # default value +- isula run -ti --rm --blkio-weight 0 busybox cat "/sys/fs/cgroup/io.weight" | grep ^"default 100"$'\r' ++ isula run --runtime $1 -ti --rm --blkio-weight 0 busybox cat "/sys/fs/cgroup/io.weight" | grep ^"default 100"$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight default value failed" && ((ret++)) + + # invalid value +- isula run -ti --rm --blkio-weight -1 busybox echo hello ++ isula run --runtime $1 -ti --rm --blkio-weight -1 busybox echo hello + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight -1 failed" && ((ret++)) + fi + + if [[ -f "/sys/fs/cgroup/isulad/io.weight_device" ]];then + # min value +- isula run -ti --rm --blkio-weight-device /dev/null:10 busybox cat "/sys/fs/cgroup/io.weight_device" | grep ^"1:3 10"$'\r' ++ isula run --runtime $1 -ti --rm --blkio-weight-device /dev/null:10 busybox cat "/sys/fs/cgroup/io.weight_device" | grep ^"1:3 10"$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight max value failed" && ((ret++)) + + # max value +- isula run -ti --rm --blkio-weight-device /dev/null:1000 busybox cat "/sys/fs/cgroup/io.weight_device" | grep ^"1:3 10000"$'\r' ++ isula run --runtime $1 -ti --rm --blkio-weight-device /dev/null:1000 busybox cat "/sys/fs/cgroup/io.weight_device" | grep ^"1:3 10000"$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight max value failed" && ((ret++)) + + # disable weight device +- isula run -tid -n weight_device --rm --blkio-weight-device /dev/null:0 busybox sh ++ isula run --runtime $1 -tid -n weight_device --rm --blkio-weight-device /dev/null:0 busybox sh + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight failed" && ((ret++)) + + isula exec -ti weight_device cat "/sys/fs/cgroup/io.weight_device" | grep ^"1:3"$'\r' +@@ -197,16 +212,22 @@ function test_cgroup2_io() + + if [[ -f /sys/fs/cgroup/isulad/io.max ]];then + # normal value +- isula run -ti --rm --device-read-bps /dev/null:1g --device-read-iops /dev/null:1000 --device-write-bps /dev/null:2g --device-write-iops /dev/null:2000 busybox cat /sys/fs/cgroup/io.max | grep ^"1:3 rbps=1073741824 wbps=2147483648 riops=1000 wiops=2000"$'\r' ++ isula run --runtime $1 -ti --rm --device-read-bps /dev/null:1g --device-read-iops /dev/null:1000 --device-write-bps /dev/null:2g --device-write-iops /dev/null:2000 busybox cat /sys/fs/cgroup/io.max | grep ^"1:3 rbps=1073741824 wbps=2147483648 riops=1000 wiops=2000"$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.max failed" && ((ret++)) + + # invalid +- isula run -ti --rm --device-read-bps /dev/null:-1 busybox echo hello ++ isula run --runtime $1 -ti --rm --device-read-bps /dev/null:-1 busybox echo hello + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.max -1 failed" && ((ret++)) + +- # 0 is no limit +- isula run -ti --rm --device-read-bps /dev/null:0 --device-read-iops /dev/null:0 --device-write-bps /dev/null:0 --device-write-iops /dev/null:0 busybox echo hello +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.max 0 failed" && ((ret++)) ++ if [ $1 == "lcr" ]; then ++ # 0 is no limit ++ isula run --runtime $1 -ti --rm --device-read-bps /dev/null:0 --device-read-iops /dev/null:0 --device-write-bps /dev/null:0 --device-write-iops /dev/null:0 busybox echo hello ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.max 0 failed" && ((ret++)) ++ else ++ # 0 is limit ++ isula run --runtime $1 -ti --rm --device-read-bps /dev/null:0 --device-read-iops /dev/null:0 --device-write-bps /dev/null:0 --device-write-iops /dev/null:0 busybox echo hello ++ [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.max 0 success" && ((ret++)) ++ fi + fi + + return ${ret} +@@ -218,51 +239,51 @@ function test_cgroup2_memory() + + if [[ -f /sys/fs/cgroup/isulad/memory.max ]];then + # normal value +- isula run -ti --rm -m 10m busybox cat /sys/fs/cgroup/memory.max | grep ^10485760$'\r' ++ isula run --runtime $1 -ti --rm -m 10m busybox cat /sys/fs/cgroup/memory.max | grep ^10485760$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.max run container failed" && ((ret++)) + + # 0 is max +- isula run -ti --rm -m 0 busybox cat /sys/fs/cgroup/memory.max | grep ^max$'\r' ++ isula run --runtime $1 -ti --rm -m 0 busybox cat /sys/fs/cgroup/memory.max | grep ^max$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.max 0 failed" && ((ret++)) + + # invalid +- isula run -ti --rm -m -1 busybox echo hello ++ isula run --runtime $1 -ti --rm -m -1 busybox echo hello + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.max -1 failed" && ((ret++)) + fi + + if [[ -f /sys/fs/cgroup/isulad/memory.low ]];then + # normal value +- isula run -ti --rm --memory-reservation 10m busybox cat /sys/fs/cgroup/memory.low | grep ^10485760$'\r' ++ isula run --runtime $1 -ti --rm --memory-reservation 10m busybox cat /sys/fs/cgroup/memory.low | grep ^10485760$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.low normal value failed" && ((ret++)) + + # -1 is invalid +- isula run -ti --rm --memory-reservation -1 busybox echo hello ++ isula run --runtime $1 -ti --rm --memory-reservation -1 busybox echo hello + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.low invalid failed" && ((ret++)) + + # 0 +- isula run -ti --rm --memory-reservation 0 busybox cat /sys/fs/cgroup/memory.low | grep ^0$'\r' ++ isula run --runtime $1 -ti --rm --memory-reservation 0 busybox cat /sys/fs/cgroup/memory.low | grep ^0$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.low 0 failed" && ((ret++)) + fi + + if [[ -f /sys/fs/cgroup/isulad/memory.swap.max ]];then + # normal value +- isula run -ti --rm --memory 10m --memory-swap 20m busybox cat /sys/fs/cgroup/memory.swap.max | grep ^10485760$'\r' ++ isula run --runtime $1 -ti --rm --memory 10m --memory-swap 20m busybox cat /sys/fs/cgroup/memory.swap.max | grep ^10485760$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.swap.max normal value failed" && ((ret++)) + + # invalid +- isula run -ti --rm --memory 10m --memory-swap 5m busybox echo hello ++ isula run --runtime $1 -ti --rm --memory 10m --memory-swap 5m busybox echo hello + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.swap.max invalid failed" && ((ret++)) + + # 0 is the same as memory +- isula run -ti --rm --memory 10m --memory-swap 0 busybox cat /sys/fs/cgroup/memory.swap.max | grep ^10485760$'\r' ++ isula run --runtime $1 -ti --rm --memory 10m --memory-swap 0 busybox cat /sys/fs/cgroup/memory.swap.max | grep ^10485760$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.swap.max 0 failed" && ((ret++)) + + # -1 is max +- isula run -ti --rm --memory 10m --memory-swap -1 busybox cat /sys/fs/cgroup/memory.swap.max | grep ^max$'\r' ++ isula run --runtime $1 -ti --rm --memory 10m --memory-swap -1 busybox cat /sys/fs/cgroup/memory.swap.max | grep ^max$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.swap.max -1 failed" && ((ret++)) + + # disable swap +- isula run -ti --rm --memory 10m --memory-swap 10m busybox cat /sys/fs/cgroup/memory.swap.max | grep ^0$'\r' ++ isula run --runtime $1 -ti --rm --memory 100m --memory-swap 100m busybox cat /sys/fs/cgroup/memory.swap.max | grep ^0$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.swap.max disable swap failed" && ((ret++)) + fi + +@@ -275,15 +296,15 @@ function test_cgroup2_pids() + + if [[ -f /sys/fs/cgroup/isulad/pids.max ]];then + # normal value +- isula run -ti --rm --pids-limit 123456 busybox cat /sys/fs/cgroup/pids.max | grep ^123456$'\r' ++ isula run --runtime $1 -ti --rm --pids-limit 123456 busybox cat /sys/fs/cgroup/pids.max | grep ^123456$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 pids.max run container failed" && ((ret++)) + + # -1 is max +- isula run -ti --rm --pids-limit -1 busybox cat /sys/fs/cgroup/pids.max | grep ^max$'\r' ++ isula run --runtime $1 -ti --rm --pids-limit -1 busybox cat /sys/fs/cgroup/pids.max | grep ^max$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 pids.max run container failed" && ((ret++)) + + # 0 is max +- isula run -ti --rm --pids-limit 0 busybox cat /sys/fs/cgroup/pids.max | grep ^max$'\r' ++ isula run --runtime $1 -ti --rm --pids-limit 0 busybox cat /sys/fs/cgroup/pids.max | grep ^max$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 pids.max run container failed" && ((ret++)) + fi + +@@ -295,7 +316,7 @@ function test_cgroup2_hugetlb() + local ret=0 + + if [[ -f /sys/fs/cgroup/isulad/hugetlb.2MB.max ]];then +- isula run -ti --rm --hugetlb-limit 2M:32M busybox cat /sys/fs/cgroup/hugetlb.2MB.max | grep ^33554432$'\r' ++ isula run --runtime $1 -ti --rm --hugetlb-limit 2M:32M busybox cat /sys/fs/cgroup/hugetlb.2MB.max | grep ^33554432$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 hugetlb.2M.max run container failed" && ((ret++)) + fi + +@@ -307,7 +328,7 @@ function test_cgroup2_freeze() + local ret=0 + + if [[ -f /sys/fs/cgroup/isulad/cgroup.freeze ]];then +- isula run -tid -n freeze busybox sh ++ isula run --runtime $1 -tid -n freeze busybox sh + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 freeze run container failed" && ((ret++)) + + isula pause freeze +@@ -335,15 +356,15 @@ function test_cgroup2_files() + + if [[ -f /sys/fs/cgroup/isulad/files.limit ]];then + # normal value +- isula run -ti --rm --files-limit 123 busybox cat /sys/fs/cgroup/files.limit | grep ^123$'\r' ++ isula run --runtime $1 -ti --rm --files-limit 123 busybox cat /sys/fs/cgroup/files.limit | grep ^123$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 files.limit run container failed" && ((ret++)) + + # -1 is max +- isula run -ti --rm --files-limit -1 busybox cat /sys/fs/cgroup/files.limit | grep ^max$'\r' ++ isula run --runtime $1 -ti --rm --files-limit -1 busybox cat /sys/fs/cgroup/files.limit | grep ^max$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 files.limit run container failed" && ((ret++)) + + # 0 is max +- isula run -ti --rm --files-limit 0 busybox cat /sys/fs/cgroup/files.limit | grep ^max$'\r' ++ isula run --runtime $1 -ti --rm --files-limit 0 busybox cat /sys/fs/cgroup/files.limit | grep ^max$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 files.limit run container failed" && ((ret++)) + fi + +@@ -405,8 +426,13 @@ function test_cgroup2_cpu_update() + isula update --cpu-quota 0 --cpu-period 1000000 $cgroup2_update + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update cpu.max 0 quota failed" && ((ret++)) + +- isula exec -ti $cgroup2_update cat /sys/fs/cgroup/cpu.max | grep ^"max 1000000"$'\r' +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update cpu.max 0 quota value not right" && ((ret++)) ++ if [ $1 == "lcr" ]; then ++ isula exec -ti $cgroup2_update cat /sys/fs/cgroup/cpu.max | grep ^"max 1000000"$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update cpu.max 0 quota value not right" && ((ret++)) ++ else ++ isula exec -ti $cgroup2_update cat /sys/fs/cgroup/cpu.max | grep ^"50000 1000000"$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update cpu.max 0 quota value not right" && ((ret++)) ++ fi + + # default -1 quota + isula update --cpu-quota -1 --cpu-period 1000000 $cgroup2_update +@@ -416,7 +442,7 @@ function test_cgroup2_cpu_update() + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update cpu.max -1 quota value not right" && ((ret++)) + + # cpus 1 +- isula run -tid -n cpu_update busybox sh ++ isula run --runtime $1 -tid -n cpu_update busybox sh + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 run cpu_update failed" && ((ret++)) + + isula update --cpus 1 cpu_update +@@ -476,8 +502,13 @@ function test_cgroup2_io_update() + isula update --blkio-weight 10 $cgroup2_update + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.bfq.weight min value failed" && ((ret++)) + +- isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.bfq.weight" | grep 1$'\r' +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.bfq.weight min value not right" && ((ret++)) ++ if [ $1 == "lcr" ]; then ++ isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.bfq.weight" | grep 1$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.bfq.weight min value not right" && ((ret++)) ++ else ++ isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.bfq.weight" | grep 10$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.bfq.weight min value not right" && ((ret++)) ++ fi + + # max value + isula update --blkio-weight 1000 $cgroup2_update +@@ -503,22 +534,38 @@ function test_cgroup2_io_update() + isula update --blkio-weight 10 $cgroup2_update + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight min value failed" && ((ret++)) + +- isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.weight" | grep ^"default 1"$'\r' +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight min value not right" && ((ret++)) ++ if [ $1 == "lcr" ]; then ++ isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.weight" | grep ^"default 1"$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight min value not right" && ((ret++)) ++ else ++ isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.bfq.weight" | grep ^"default 10"$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight min value not right" && ((ret++)) ++ fi + + # max value + isula update --blkio-weight 1000 $cgroup2_update + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight max value failed" && ((ret++)) + +- isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.weight" | grep ^"default 10000"$'\r' +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight max value not right" && ((ret++)) ++ if [ $1 == "lcr" ]; then ++ isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.weight" | grep ^"default 10000"$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight max value not right" && ((ret++)) ++ else ++ isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.bfq.weight" | grep ^"default 1000"$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight max value not right" && ((ret++)) ++ fi + + # 0 means value not change + isula update --blkio-weight 0 $cgroup2_update + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight 0 failed" && ((ret++)) + +- isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.weight" | grep ^"default 10000"$'\r' +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight 0 not right" && ((ret++)) ++ ++ if [ $1 == "lcr" ]; then ++ isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.weight" | grep ^"default 10000"$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight max value not right" && ((ret++)) ++ else ++ isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.bfq.weight" | grep ^"default 1000"$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight max value not right" && ((ret++)) ++ fi + + # invalid value + isula update --blkio-weight -1 $cgroup2_update echo hello +@@ -591,12 +638,13 @@ function test_cgroup2_memory_update() + isula exec -ti $cgroup2_update cat /sys/fs/cgroup/memory.swap.max | grep ^10485760$'\r' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update memory.swap.max 0 value not right" && ((ret++)) + +- # -1 is max +- isula update --memory 10m --memory-swap -1 $cgroup2_update +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update memory.swap.max -1 failed" && ((ret++)) +- +- isula exec -ti $cgroup2_update cat /sys/fs/cgroup/memory.swap.max | grep ^max$'\r' +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update memory.swap.max -1 value not right" && ((ret++)) ++ if [ $1 == "lcr" ]; then ++ # -1 is max ++ isula update --memory 10m --memory-swap -1 $cgroup2_update ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update memory.swap.max -1 failed" && ((ret++)) ++ isula exec -ti $cgroup2_update cat /sys/fs/cgroup/memory.swap.max | grep ^max$'\r' ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update memory.swap.max -1 value not right" && ((ret++)) ++ fi + + # disable swap + isula update --memory 10m --memory-swap 10m $cgroup2_update +@@ -613,16 +661,16 @@ function test_cgroup2_unsupported() + { + local ret=0 + +- isula run -ti --rm --cpu-rt-period 1000000 --cpu-rt-runtime 1000000 busybox echo hello ++ isula run --runtime $1 -ti --rm --cpu-rt-period 1000000 --cpu-rt-runtime 1000000 busybox echo hello + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --cpu-rt-period and --cpu-rt-runtime should failed" && ((ret++)) + +- isula run -ti --rm --kernel-memory 100m busybox echo hello ++ isula run --runtime $1 -ti --rm --kernel-memory 100m busybox echo hello + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --kernel-memory should failed" && ((ret++)) + +- isula run -ti --rm --memory-swappiness 50 busybox echo hello ++ isula run --runtime $1 -ti --rm --memory-swappiness 50 busybox echo hello + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --memory-swappiness should failed" && ((ret++)) + +- isula run -ti --rm --oom-kill-disable busybox echo hello ++ isula run --runtime $1 -ti --rm --oom-kill-disable busybox echo hello + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --oom-kill-disable should failed" && ((ret++)) + + isula update --cpu-rt-period 1000000 --cpu-rt-runtime 1000000 $cgroup2_update +@@ -641,7 +689,7 @@ function test_cgroup2_parent() + rmdir /sys/fs/cgroup/isulad + rmdir /sys/fs/cgroup/abc + +- id=`isula run -tid --cgroup-parent /abc -m 10m busybox sh` ++ id=`isula run --runtime $1 -tid --cgroup-parent /abc -m 10m busybox sh` + cat /sys/fs/cgroup/abc/$id/memory.max | grep ^10485760$ + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --cgroup-parent cannot work" && ((ret++)) + +@@ -657,39 +705,39 @@ function test_cgroup2_device() + mknod_num=$(echo $dev_num | sed 's/:/ /g') + + # read only +- isula run -ti --rm --device=$dev_name:/dev/sdx:r busybox sh -c 'echo q | fdisk /dev/sdx | grep "read only"' ++ isula run --runtime $1 -ti --rm --device=$dev_name:/dev/sdx:r busybox sh -c 'echo q | fdisk /dev/sdx | grep "read only"' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device r failed" && ((ret++)) + +- isula run -ti --rm --device=$dev_name:/dev/sdx:rm busybox sh -c 'echo q | fdisk /dev/sdx | grep "read only"' ++ isula run --runtime $1 -ti --rm --device=$dev_name:/dev/sdx:rm busybox sh -c 'echo q | fdisk /dev/sdx | grep "read only"' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device rm failed" && ((ret++)) + +- isula run -ti --rm --device-cgroup-rule="b $dev_num r" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx | grep 'read only'" ++ isula run --runtime $1 -ti --rm --device-cgroup-rule="b $dev_num r" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx | grep 'read only'" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device r failed" && ((ret++)) + +- isula run -ti --rm --device-cgroup-rule="b $dev_num rm" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx | grep 'read only'" ++ isula run --runtime $1 -ti --rm --device-cgroup-rule="b $dev_num rm" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx | grep 'read only'" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device rm failed" && ((ret++)) + + # can't read +- isula run -ti --rm --device=$dev_name:/dev/sdx:w busybox sh -c 'echo q | fdisk /dev/sdx 2>&1 | grep "t open"' ++ isula run --runtime $1 -ti --rm --device=$dev_name:/dev/sdx:w busybox sh -c 'echo q | fdisk /dev/sdx 2>&1 | grep "t open"' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device w failed" && ((ret++)) + +- isula run -ti --rm --device=$dev_name:/dev/sdx:wm busybox sh -c 'echo q | fdisk /dev/sdx 2>&1 | grep "t open"' ++ isula run --runtime $1 -ti --rm --device=$dev_name:/dev/sdx:wm busybox sh -c 'echo q | fdisk /dev/sdx 2>&1 | grep "t open"' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device wm failed" && ((ret++)) + +- isula run -ti --rm --device-cgroup-rule="b $dev_num w" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx 2>&1 | grep 't open'" ++ isula run --runtime $1 -ti --rm --device-cgroup-rule="b $dev_num w" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx 2>&1 | grep 't open'" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device w failed" && ((ret++)) + +- isula run -ti --rm --device-cgroup-rule="b $dev_num wm" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx 2>&1 | grep 't open'" ++ isula run --runtime $1 -ti --rm --device-cgroup-rule="b $dev_num wm" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx 2>&1 | grep 't open'" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device wm failed" && ((ret++)) + + # can't read write +- isula run -ti --rm --device=$dev_name:/dev/sdx:m busybox sh -c 'echo q | fdisk /dev/sdx 2>&1 | grep "t open"' ++ isula run --runtime $1 -ti --rm --device=$dev_name:/dev/sdx:m busybox sh -c 'echo q | fdisk /dev/sdx 2>&1 | grep "t open"' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device m" && ((ret++)) + +- isula run -ti --rm --device-cgroup-rule="b $dev_num m" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx 2>&1 | grep 't open'" ++ isula run --runtime $1 -ti --rm --device-cgroup-rule="b $dev_num m" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx 2>&1 | grep 't open'" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device wm failed" && ((ret++)) + +- isula run -ti --rm --device-cgroup-rule="b *:* m" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx 2>&1 | grep 't open'" ++ isula run --runtime $1 -ti --rm --device-cgroup-rule="b *:* m" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx 2>&1 | grep 't open'" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device wm failed" && ((ret++)) + + return ${ret} +@@ -723,7 +771,7 @@ function prepare_test_cgroupv2() + + isula rm -f `isula ps -a -q` + +- isula run -tid -n $cgroup2_update busybox sh ++ isula run --runtime $1 -tid -n $cgroup2_update busybox sh + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 run container failed" && ((ret++)) + + return ${ret} +@@ -740,25 +788,31 @@ declare -i ans=0 + msg_info "${test} starting..." + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start isulad failed" && ((ret++)) + +-prepare_test_cgroupv2 || ((ans++)) +-if [ "$cgroupv2" == "1" ];then +- test_cgroup2_cpu || ((ans++)) +- test_cgroup2_io || ((ans++)) +- test_cgroup2_memory || ((ans++)) +- test_cgroup2_pids || ((ans++)) +- test_cgroup2_hugetlb || ((ans++)) +- test_cgroup2_freeze || ((ans++)) +- test_cgroup2_files || ((ans++)) +- test_cgroup2_cpu_update || ((ans++)) +- test_cgroup2_io_update || ((ans++)) +- test_cgroup2_memory_update || ((ans++)) +- test_cgroup2_unsupported || ((ans++)) +- test_cgroup2_parent || ((ans++)) +- test_cgroup2_device || ((ans++)) +-else +- msg_info "${test} not cgroup v2 enviorment, ignore test..." +-fi +-post_test_cgroupv2 ++for element in ${RUNTIME_LIST[@]}; ++do ++ prepare_test_cgroupv2 $element || ((ans++)) ++ if [ "$cgroupv2" == "1" ];then ++ local test="cgroup v2 test => (${element})" ++ msg_info "${test} starting..." ++ test_cgroup2_cpu $element || ((ans++)) ++ test_cgroup2_io $element || ((ans++)) ++ test_cgroup2_memory $element || ((ans++)) ++ test_cgroup2_pids $element || ((ans++)) ++ test_cgroup2_hugetlb $element || ((ans++)) ++ test_cgroup2_freeze $element || ((ans++)) ++ test_cgroup2_files $element || ((ans++)) ++ test_cgroup2_cpu_update $element || ((ans++)) ++ test_cgroup2_io_update $element || ((ans++)) ++ test_cgroup2_memory_update $element || ((ans++)) ++ test_cgroup2_unsupported $element || ((ans++)) ++ test_cgroup2_parent $element || ((ans++)) ++ test_cgroup2_device $element || ((ans++)) ++ msg_info "${test} finished with return ${ans}..." ++ else ++ msg_info "${test} not cgroup v2 enviorment, ignore test..." ++ fi ++ post_test_cgroupv2 $element ++done + + msg_info "${test} finished with return ${ans}..." + +-- +2.34.1 + diff --git a/0039-fix-run-ubuntu-container-bug-in-inspect.sh.patch b/0039-fix-run-ubuntu-container-bug-in-inspect.sh.patch new file mode 100644 index 0000000..5501261 --- /dev/null +++ b/0039-fix-run-ubuntu-container-bug-in-inspect.sh.patch @@ -0,0 +1,27 @@ +From 8e1fe0302bf1a871f66a296e456811e878b1fa3b Mon Sep 17 00:00:00 2001 +From: jikai +Date: Tue, 2 Apr 2024 10:06:18 +0800 +Subject: [PATCH 39/69] fix run ubuntu container bug in inspect.sh + +Signed-off-by: jikai +--- + CI/test_cases/container_cases/inspect.sh | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/CI/test_cases/container_cases/inspect.sh b/CI/test_cases/container_cases/inspect.sh +index b4f4a785..86aed3d8 100755 +--- a/CI/test_cases/container_cases/inspect.sh ++++ b/CI/test_cases/container_cases/inspect.sh +@@ -146,7 +146,8 @@ function test_inspect_spec() + + isula rm -f $containername + +- isula run -it -m 4m --name $containername $ubuntu_image perl -e 'for ($i = 0; $i < 100000000; $i++) { $a .= " " x 1024 }' ++ # use more than 10m memory limit, otherwise it might fail to run ++ isula run -it -m 10m --name $containername $ubuntu_image perl -e 'for ($i = 0; $i < 100000000; $i++) { $a .= " " x 1024 }' + + isula inspect -f "{{json .State.OOMKilled}} {{.Name}}" $containername 2>&1 | sed -n '1p' | grep "true" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container with image: ${ubuntu_image}" && ((ret++)) +-- +2.34.1 + diff --git a/0040-add-support-for-GetContainerEvents.patch b/0040-add-support-for-GetContainerEvents.patch new file mode 100644 index 0000000..397a792 --- /dev/null +++ b/0040-add-support-for-GetContainerEvents.patch @@ -0,0 +1,2601 @@ +From 745497bdc5c5192709ecc7b3edc91a5170f5b30e Mon Sep 17 00:00:00 2001 +From: jikai +Date: Fri, 29 Mar 2024 09:33:38 +0000 +Subject: [PATCH 40/69] add support for GetContainerEvents + +Signed-off-by: jikai +--- + src/daemon/CMakeLists.txt | 3 + + src/daemon/common/cri/v1/v1_cri_helpers.cc | 205 +++++++++++++++ + src/daemon/common/cri/v1/v1_cri_helpers.h | 5 + + .../{entry => common}/cri/v1/v1_naming.cc | 0 + .../{entry => common}/cri/v1/v1_naming.h | 0 + src/daemon/config/isulad_config.c | 1 + + .../entry/connect/grpc/cri/cri_service.cc | 3 +- + .../entry/connect/grpc/cri/cri_service.h | 1 + + .../cri/v1/cri_v1_runtime_runtime_service.cc | 147 ++++++++++- + .../cri/v1/cri_v1_runtime_runtime_service.h | 11 +- + src/daemon/entry/connect/grpc/grpc_service.cc | 6 +- + .../v1/v1_cri_container_manager_service.cc | 203 +-------------- + .../cri/v1/v1_cri_container_manager_service.h | 13 - + .../v1/v1_cri_pod_sandbox_manager_service.cc | 92 ++++++- + .../v1/v1_cri_pod_sandbox_manager_service.h | 10 +- + .../entry/cri/v1/v1_cri_runtime_service.h | 4 +- + .../cri/v1/v1_cri_runtime_service_impl.cc | 10 +- + .../cri/v1/v1_cri_runtime_service_impl.h | 7 +- + src/daemon/executor/container_cb/execution.c | 25 ++ + .../executor/container_cb/execution_create.c | 12 + + src/daemon/mailbox/CMakeLists.txt | 11 + + src/daemon/mailbox/mailbox.c | 167 +++++++++++++ + src/daemon/mailbox/mailbox.h | 82 ++++++ + src/daemon/mailbox/mailbox_message.c | 94 +++++++ + src/daemon/mailbox/mailbox_message.h | 50 ++++ + src/daemon/mailbox/message_queue.c | 234 ++++++++++++++++++ + src/daemon/mailbox/message_queue.h | 57 +++++ + src/daemon/mailbox/message_subscriber.c | 85 +++++++ + src/daemon/mailbox/message_subscriber.h | 41 +++ + src/daemon/modules/api/container_api.h | 5 + + .../modules/container/supervisor/supervisor.c | 18 ++ + src/daemon/sandbox/sandbox.cc | 9 + + src/utils/cutils/blocking_queue.c | 185 ++++++++++++++ + src/utils/cutils/blocking_queue.h | 66 +++++ + test/mocks/mailbox_mock.cc | 30 +++ + test/mocks/mailbox_mock.h | 30 +++ + test/sandbox/controller/shim/CMakeLists.txt | 1 + + test/sandbox/sandbox/CMakeLists.txt | 2 + + 38 files changed, 1681 insertions(+), 244 deletions(-) + rename src/daemon/{entry => common}/cri/v1/v1_naming.cc (100%) + rename src/daemon/{entry => common}/cri/v1/v1_naming.h (100%) + create mode 100644 src/daemon/mailbox/CMakeLists.txt + create mode 100644 src/daemon/mailbox/mailbox.c + create mode 100644 src/daemon/mailbox/mailbox.h + create mode 100644 src/daemon/mailbox/mailbox_message.c + create mode 100644 src/daemon/mailbox/mailbox_message.h + create mode 100644 src/daemon/mailbox/message_queue.c + create mode 100644 src/daemon/mailbox/message_queue.h + create mode 100644 src/daemon/mailbox/message_subscriber.c + create mode 100644 src/daemon/mailbox/message_subscriber.h + create mode 100644 src/utils/cutils/blocking_queue.c + create mode 100644 src/utils/cutils/blocking_queue.h + create mode 100644 test/mocks/mailbox_mock.cc + create mode 100644 test/mocks/mailbox_mock.h + +diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt +index d5280c88..29af3dca 100644 +--- a/src/daemon/CMakeLists.txt ++++ b/src/daemon/CMakeLists.txt +@@ -3,6 +3,7 @@ + aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} daemon_top_srcs) + add_subdirectory(executor) + add_subdirectory(entry) ++add_subdirectory(mailbox) + add_subdirectory(modules) + add_subdirectory(config) + add_subdirectory(common) +@@ -11,6 +12,7 @@ set(local_daemon_srcs + ${daemon_top_srcs} + ${EXECUTOR_SRCS} + ${ENTRY_SRCS} ++ ${MAILBOX_SRCS} + ${MODULES_SRCS} + ${CONFIG_SRCS} + ${DAEMON_COMMON_SRCS} +@@ -20,6 +22,7 @@ set(local_daemon_incs + ${CMAKE_CURRENT_SOURCE_DIR} + ${EXECUTOR_INCS} + ${ENTRY_INCS} ++ ${MAILBOX_INCS} + ${MODULES_INCS} + ${CONFIG_INCS} + ${DAEMON_COMMON_INCS} +diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.cc b/src/daemon/common/cri/v1/v1_cri_helpers.cc +index c57301ce..a3488894 100644 +--- a/src/daemon/common/cri/v1/v1_cri_helpers.cc ++++ b/src/daemon/common/cri/v1/v1_cri_helpers.cc +@@ -32,6 +32,7 @@ + #include "service_container_api.h" + #include "isulad_config.h" + #include "sha256.h" ++#include "v1_naming.h" + + namespace CRIHelpersV1 { + +@@ -458,4 +459,208 @@ void ApplySandboxSecurityContextToHostConfig(const runtime::v1::LinuxSandboxSecu + } + } + ++void PackContainerImageToStatus( ++ container_inspect *inspect, std::unique_ptr &contStatus, Errors &error) ++{ ++ if (inspect->config == nullptr) { ++ return; ++ } ++ ++ if (inspect->config->image != nullptr) { ++ contStatus->mutable_image()->set_image(inspect->config->image); ++ } ++ ++ contStatus->set_image_ref(CRIHelpers::ToPullableImageID(inspect->config->image, inspect->config->image_ref)); ++} ++ ++void UpdateBaseStatusFromInspect( ++ container_inspect *inspect, int64_t &createdAt, int64_t &startedAt, int64_t &finishedAt, ++ std::unique_ptr &contStatus) ++{ ++ runtime::v1::ContainerState state { runtime::v1::CONTAINER_UNKNOWN }; ++ std::string reason; ++ std::string message; ++ int32_t exitCode { 0 }; ++ ++ if (inspect->state == nullptr) { ++ goto pack_status; ++ } ++ ++ if (inspect->state->running) { ++ // Container is running ++ state = runtime::v1::CONTAINER_RUNNING; ++ } else { ++ // Container is not running. ++ if (finishedAt != 0) { // Case 1 ++ state = runtime::v1::CONTAINER_EXITED; ++ if (inspect->state->exit_code == 0) { ++ reason = "Completed"; ++ } else { ++ reason = "Error"; ++ } ++ } else if (inspect->state->exit_code != 0) { // Case 2 ++ state = runtime::v1::CONTAINER_EXITED; ++ finishedAt = createdAt; ++ startedAt = createdAt; ++ reason = "ContainerCannotRun"; ++ } else { // Case 3 ++ state = runtime::v1::CONTAINER_CREATED; ++ } ++ if (inspect->state->error != nullptr) { ++ message = inspect->state->error; ++ } ++ exitCode = (int32_t)inspect->state->exit_code; ++ } ++ ++pack_status: ++ contStatus->set_exit_code(exitCode); ++ contStatus->set_state(state); ++ contStatus->set_created_at(createdAt); ++ contStatus->set_started_at(startedAt); ++ contStatus->set_finished_at(finishedAt); ++ contStatus->set_reason(reason); ++ contStatus->set_message(message); ++} ++ ++void PackLabelsToStatus(container_inspect *inspect, ++ std::unique_ptr &contStatus) ++{ ++ if (inspect->config == nullptr || inspect->config->labels == nullptr) { ++ return; ++ } ++ CRIHelpers::ExtractLabels(inspect->config->labels, *contStatus->mutable_labels()); ++ CRIHelpers::ExtractAnnotations(inspect->config->annotations, *contStatus->mutable_annotations()); ++ for (size_t i = 0; i < inspect->config->labels->len; i++) { ++ if (strcmp(inspect->config->labels->keys[i], CRIHelpers::Constants::CONTAINER_LOGPATH_LABEL_KEY.c_str()) == 0) { ++ contStatus->set_log_path(inspect->config->labels->values[i]); ++ break; ++ } ++ } ++} ++ ++void ConvertMountsToStatus(container_inspect *inspect, ++ std::unique_ptr &contStatus) ++{ ++ for (size_t i = 0; i < inspect->mounts_len; i++) { ++ runtime::v1::Mount *mount = contStatus->add_mounts(); ++ mount->set_host_path(inspect->mounts[i]->source); ++ mount->set_container_path(inspect->mounts[i]->destination); ++ mount->set_readonly(!inspect->mounts[i]->rw); ++ if (inspect->mounts[i]->propagation == nullptr || strcmp(inspect->mounts[i]->propagation, "rprivate") == 0) { ++ mount->set_propagation(runtime::v1::PROPAGATION_PRIVATE); ++ } else if (strcmp(inspect->mounts[i]->propagation, "rslave") == 0) { ++ mount->set_propagation(runtime::v1::PROPAGATION_HOST_TO_CONTAINER); ++ } else if (strcmp(inspect->mounts[i]->propagation, "rshared") == 0) { ++ mount->set_propagation(runtime::v1::PROPAGATION_BIDIRECTIONAL); ++ } ++ // Note: Can't set SeLinuxRelabel ++ } ++} ++ ++void ConvertResourcesToStatus(container_inspect *inspect, ++ std::unique_ptr &contStatus) ++{ ++ if (inspect->resources == nullptr) { ++ return; ++ } ++ runtime::v1::LinuxContainerResources *resources = contStatus->mutable_resources()->mutable_linux(); ++ if (inspect->resources->cpu_shares != 0) { ++ resources->set_cpu_shares(inspect->resources->cpu_shares); ++ } ++ if (inspect->resources->cpu_period != 0) { ++ resources->set_cpu_period(inspect->resources->cpu_period); ++ } ++ if (inspect->resources->cpu_quota != 0) { ++ resources->set_cpu_quota(inspect->resources->cpu_quota); ++ } ++ if (inspect->resources->memory != 0) { ++ resources->set_memory_limit_in_bytes(inspect->resources->memory); ++ } ++ if (inspect->resources->memory_swap != 0) { ++ resources->set_memory_swap_limit_in_bytes(inspect->resources->memory_swap); ++ } ++ for (size_t i = 0; i < inspect->resources->hugetlbs_len; i++) { ++ runtime::v1::HugepageLimit *hugepage = resources->add_hugepage_limits(); ++ hugepage->set_page_size(inspect->resources->hugetlbs[i]->page_size); ++ hugepage->set_limit(inspect->resources->hugetlbs[i]->limit); ++ } ++ if (inspect->resources->unified != nullptr) { ++ for (size_t i = 0; i < inspect->resources->unified->len; i++) { ++ auto &resUnified = *(resources->mutable_unified()); ++ resUnified[inspect->resources->unified->keys[i]] = inspect->resources->unified->values[i]; ++ } ++ } ++} ++ ++void ContainerStatusToGRPC(container_inspect *inspect, ++ std::unique_ptr &contStatus, ++ Errors &error) ++{ ++ if (inspect->id != nullptr) { ++ contStatus->set_id(inspect->id); ++ } ++ ++ int64_t createdAt {}; ++ int64_t startedAt {}; ++ int64_t finishedAt {}; ++ CRIHelpers::GetContainerTimeStamps(inspect, &createdAt, &startedAt, &finishedAt, error); ++ if (error.NotEmpty()) { ++ return; ++ } ++ contStatus->set_created_at(createdAt); ++ contStatus->set_started_at(startedAt); ++ contStatus->set_finished_at(finishedAt); ++ ++ PackContainerImageToStatus(inspect, contStatus, error); ++ UpdateBaseStatusFromInspect(inspect, createdAt, startedAt, finishedAt, contStatus); ++ PackLabelsToStatus(inspect, contStatus); ++ CRINamingV1::ParseContainerName(contStatus->annotations(), contStatus->mutable_metadata(), error); ++ if (error.NotEmpty()) { ++ return; ++ } ++ ConvertMountsToStatus(inspect, contStatus); ++ ConvertResourcesToStatus(inspect, contStatus); ++} ++ ++std::unique_ptr GetContainerStatus(service_executor_t *m_cb, const std::string &containerID, Errors &error) ++{ ++ if (m_cb == nullptr) { ++ error.SetError("Invalid input arguments: empty service executor"); ++ return nullptr; ++ } ++ ++ if (containerID.empty()) { ++ error.SetError("Empty container id"); ++ return nullptr; ++ } ++ ++ std::string realContainerID = CRIHelpers::GetRealContainerOrSandboxID(m_cb, containerID, false, error); ++ if (error.NotEmpty()) { ++ ERROR("Failed to find container id %s: %s", containerID.c_str(), error.GetCMessage()); ++ error.Errorf("Failed to find container id %s: %s", containerID.c_str(), error.GetCMessage()); ++ return nullptr; ++ } ++ ++ container_inspect *inspect = CRIHelpers::InspectContainer(realContainerID, error, false); ++ if (error.NotEmpty()) { ++ return nullptr; ++ } ++ if (inspect == nullptr) { ++ error.SetError("Get null inspect"); ++ return nullptr; ++ } ++ using ContainerStatusPtr = std::unique_ptr; ++ ContainerStatusPtr contStatus(new (std::nothrow) runtime::v1::ContainerStatus); ++ if (contStatus == nullptr) { ++ error.SetError("Out of memory"); ++ free_container_inspect(inspect); ++ return nullptr; ++ } ++ ++ ContainerStatusToGRPC(inspect, contStatus, error); ++ ++ free_container_inspect(inspect); ++ return contStatus; ++} ++ + } // v1 namespace CRIHelpers +diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.h b/src/daemon/common/cri/v1/v1_cri_helpers.h +index b6e6aec6..1578c428 100644 +--- a/src/daemon/common/cri/v1/v1_cri_helpers.h ++++ b/src/daemon/common/cri/v1/v1_cri_helpers.h +@@ -27,6 +27,8 @@ + #include "checkpoint_handler.h" + #include "constants.h" + #include "errors.h" ++#include "callback.h" ++#include "cstruct_wrapper.h" + + namespace CRIHelpersV1 { + +@@ -78,6 +80,9 @@ std::string CRISandboxerConvert(const std::string &runtime); + void ApplySandboxSecurityContextToHostConfig(const runtime::v1::LinuxSandboxSecurityContext &context, host_config *hc, + Errors &error); + ++auto GetContainerStatus(service_executor_t *m_cb, const std::string &containerID, Errors &error) ++-> std::unique_ptr; ++ + }; // namespace CRIHelpers + + #endif // DAEMON_ENTRY_CRI_V1ALPHA_CRI_HELPERS_H +diff --git a/src/daemon/entry/cri/v1/v1_naming.cc b/src/daemon/common/cri/v1/v1_naming.cc +similarity index 100% +rename from src/daemon/entry/cri/v1/v1_naming.cc +rename to src/daemon/common/cri/v1/v1_naming.cc +diff --git a/src/daemon/entry/cri/v1/v1_naming.h b/src/daemon/common/cri/v1/v1_naming.h +similarity index 100% +rename from src/daemon/entry/cri/v1/v1_naming.h +rename to src/daemon/common/cri/v1/v1_naming.h +diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c +index 8179558e..778ff921 100644 +--- a/src/daemon/config/isulad_config.c ++++ b/src/daemon/config/isulad_config.c +@@ -1760,6 +1760,7 @@ int merge_json_confs_into_global(struct service_arguments *args) + args->json_confs->cri_sandboxers = tmp_json_confs->cri_sandboxers; + tmp_json_confs->cri_sandboxers = NULL; + args->json_confs->enable_cri_v1 = tmp_json_confs->enable_cri_v1; ++ args->json_confs->enable_pod_events = tmp_json_confs->enable_pod_events; + #endif + + args->json_confs->systemd_cgroup = tmp_json_confs->systemd_cgroup; +diff --git a/src/daemon/entry/connect/grpc/cri/cri_service.cc b/src/daemon/entry/connect/grpc/cri/cri_service.cc +index c1986c44..d10a60b5 100644 +--- a/src/daemon/entry/connect/grpc/cri/cri_service.cc ++++ b/src/daemon/entry/connect/grpc/cri/cri_service.cc +@@ -89,8 +89,9 @@ int CRIService::Init(const isulad_daemon_configs *config) + + #ifdef ENABLE_CRI_API_V1 + m_enableCRIV1 = config->enable_cri_v1; ++ m_enablePodEvents = config->enable_pod_events; + if (m_enableCRIV1) { +- m_runtimeV1RuntimeService.Init(m_podSandboxImage, m_pluginManager, err); ++ m_runtimeV1RuntimeService.Init(m_podSandboxImage, m_pluginManager, m_enablePodEvents, err); + if (err.NotEmpty()) { + ERROR("Init CRI v1 runtime service failed: %s", err.GetCMessage()); + return -1; +diff --git a/src/daemon/entry/connect/grpc/cri/cri_service.h b/src/daemon/entry/connect/grpc/cri/cri_service.h +index 77b2eb72..041c7c63 100644 +--- a/src/daemon/entry/connect/grpc/cri/cri_service.h ++++ b/src/daemon/entry/connect/grpc/cri/cri_service.h +@@ -56,6 +56,7 @@ private: + std::string m_podSandboxImage; + std::shared_ptr m_pluginManager; + bool m_enableCRIV1; ++ bool m_enablePodEvents; + }; + + } +diff --git a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc +index 76e393f3..bc5ab591 100644 +--- a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc ++++ b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc +@@ -22,11 +22,37 @@ + #include "callback.h" + #include "network_plugin.h" + #include "v1_cri_runtime_service_impl.h" ++#include "mailbox.h" ++#include "mailbox_message.h" + + using namespace CRIV1; + ++static void *cri_container_topic_handler(void *context, void *arg) ++{ ++ if (context == nullptr || arg == nullptr) { ++ ERROR("Invalid input arguments"); ++ return nullptr; ++ } ++ ++ auto v1runtimeService = static_cast(context); ++ auto msg = static_cast(arg); ++ return v1runtimeService->GenerateCRIContainerEvent(msg->container_id, msg->sandbox_id, ++ static_cast(msg->type)); ++} ++ ++static void cri_container_topic_release(void *arg) ++{ ++ if (arg == nullptr) { ++ return; ++ } ++ ++ auto resp = static_cast(arg); ++ delete resp; ++} ++ + void RuntimeV1RuntimeServiceImpl::Init(std::string &podSandboxImage, +- std::shared_ptr networkPlugin, Errors &err) ++ std::shared_ptr networkPlugin, ++ bool enablePodEvents, Errors &err) + { + // Assembly implementation for CRIRuntimeServiceImpl + service_executor_t *cb = get_service_executor(); +@@ -36,7 +62,18 @@ void RuntimeV1RuntimeServiceImpl::Init(std::string &podSandboxImage, + return; + } + +- m_rService = std::unique_ptr(new CRIRuntimeServiceImpl(podSandboxImage, cb, networkPlugin)); ++ if (enablePodEvents) { ++ if (mailbox_register_topic_handler(MAILBOX_TOPIC_CRI_CONTAINER, cri_container_topic_handler, ++ this, cri_container_topic_release, true) != 0) { ++ ERROR("Failed to register container topic handler"); ++ err.SetError("Failed to register container topic handler"); ++ return; ++ } ++ m_enablePodEvents = enablePodEvents; ++ } ++ ++ ++ m_rService = std::unique_ptr(new CRIRuntimeServiceImpl(podSandboxImage, cb, networkPlugin, m_enablePodEvents)); + } + + void RuntimeV1RuntimeServiceImpl::Wait() +@@ -45,6 +82,54 @@ void RuntimeV1RuntimeServiceImpl::Wait() + + void RuntimeV1RuntimeServiceImpl::Shutdown() + { ++ mailbox_unregister_topic_handler(MAILBOX_TOPIC_CRI_CONTAINER); ++} ++ ++auto RuntimeV1RuntimeServiceImpl::GenerateCRIContainerEvent(const char *container_id, const char *sandbox_id, ++ runtime::v1::ContainerEventType type) -> runtime::v1::ContainerEventResponse * ++{ ++ if (container_id == nullptr || sandbox_id == nullptr) { ++ ERROR("Invalid input arguments"); ++ return nullptr; ++ } ++ ++ if (type < runtime::v1::ContainerEventType::CONTAINER_CREATED_EVENT || ++ type > runtime::v1::ContainerEventType::CONTAINER_DELETED_EVENT) { ++ ERROR("Invalid container event type %d", type); ++ return nullptr; ++ } ++ ++ std::string containerID(container_id), sandboxID(sandbox_id); ++ Errors error; ++ runtime::v1::ContainerEventResponse *response = new (std::nothrow) runtime::v1::ContainerEventResponse(); ++ if (response == nullptr) { ++ ERROR("Out of memory"); ++ return nullptr; ++ } ++ ++ runtime::v1::PodSandboxStatusResponse *statusReply = new (std::nothrow) runtime::v1::PodSandboxStatusResponse(); ++ if (statusReply == nullptr) { ++ ERROR("Out of memory"); ++ delete response; ++ return nullptr; ++ } ++ ++ m_rService->PodSandboxStatus(sandboxID, statusReply, error); ++ if (!error.Empty()) { ++ WARN("Object: CRI, Type: Failed to status pod:%s due to %s", sandboxID.c_str(), ++ error.GetMessage().c_str()); ++ } else { ++ *(response->mutable_pod_sandbox_status()) = *(statusReply->mutable_status()); ++ for (auto &containerStatus : statusReply->containers_statuses()) { ++ *(response->add_containers_statuses()) = containerStatus; ++ } ++ } ++ ++ response->set_container_event_type((runtime::v1::ContainerEventType)type); ++ response->set_container_id(containerID); ++ response->set_created_at(util_get_now_time_nanos()); ++ ++ return response; + } + + grpc::Status RuntimeV1RuntimeServiceImpl::Version(grpc::ServerContext *context, +@@ -398,14 +483,12 @@ grpc::Status RuntimeV1RuntimeServiceImpl::PodSandboxStatus(grpc::ServerContext * + + INFO("Event: {Object: CRI, Type: Status Pod: %s}", request->pod_sandbox_id().c_str()); + +- std::unique_ptr podStatus; +- podStatus = m_rService->PodSandboxStatus(request->pod_sandbox_id(), error); +- if (!error.Empty() || !podStatus) { ++ m_rService->PodSandboxStatus(request->pod_sandbox_id(), reply, error); ++ if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to status pod:%s due to %s", request->pod_sandbox_id().c_str(), + error.GetMessage().c_str()); + return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); + } +- *(reply->mutable_status()) = *podStatus; + + INFO("Event: {Object: CRI, Type: Statused Pod: %s}", request->pod_sandbox_id().c_str()); + +@@ -657,3 +740,55 @@ RuntimeV1RuntimeServiceImpl::RuntimeConfig(grpc::ServerContext *context, + + return grpc::Status::OK; + } ++ ++grpc::Status RuntimeV1RuntimeServiceImpl::GetContainerEvents(grpc::ServerContext *context, ++ const runtime::v1::GetEventsRequest *request, ++ grpc::ServerWriter *writer) ++{ ++ Errors error; ++ ++ if (context == nullptr || request == nullptr || writer == nullptr) { ++ ERROR("Invalid input arguments"); ++ return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, "Invalid input arguments"); ++ } ++ ++ if (!m_enablePodEvents) { ++ ERROR("Pod events is not enabled"); ++ return grpc::Status(grpc::StatusCode::UNIMPLEMENTED, "Pod events is not enabled"); ++ } ++ ++ INFO("Event: {Object: CRI, Type: Getting Container Events}"); ++ ++ __isula_auto_subscriber auto sub = mailbox_subscribe(MAILBOX_TOPIC_CRI_CONTAINER); ++ if (sub == nullptr) { ++ ERROR("Object: CRI, Type: Failed to subscribe container events"); ++ return grpc::Status(grpc::StatusCode::UNKNOWN, "Failed to subscribe container events"); ++ } ++ ++ for (;;) { ++ __isula_auto_mailbox_message mailbox_message *msg = NULL; ++ int ret = message_subscriber_pop(sub, &msg); ++ if (ret == 0) { ++ if (msg == nullptr) { ++ // nullptr response indicates eventqueue being shutdown, not need to unscribe now ++ return grpc::Status(grpc::StatusCode::UNKNOWN, "Event queue is shutdown"); ++ } ++ auto *response = static_cast(msg->data); ++ if (!writer->Write(*response)) { ++ break; ++ } ++ } else if (ret != ETIMEDOUT) { ++ ERROR("Failed to pop message from subscriber"); ++ break; ++ } ++ if (context->IsCancelled()) { ++ INFO("Object: CRI, Type: GetContainerEvents is cancelled"); ++ break; ++ } ++ } ++ ++ mailbox_unsubscribe(MAILBOX_TOPIC_CRI_CONTAINER, sub); ++ INFO("Event: {Object: CRI, Type: Got Container Events}"); ++ ++ return grpc::Status::OK; ++} +diff --git a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h +index 52cc6b99..842d1811 100644 +--- a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h ++++ b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h +@@ -26,9 +26,13 @@ + // Implement of runtime RuntimeService + class RuntimeV1RuntimeServiceImpl : public runtime::v1::RuntimeService::Service { + public: +- void Init(std::string &podSandboxImage, std::shared_ptr networkPlugin, Errors &err); ++ void Init(std::string &podSandboxImage, std::shared_ptr networkPlugin, ++ bool enablePodEvents, Errors &err); + void Wait(); + void Shutdown(); ++ auto GenerateCRIContainerEvent(const char *container_id, const char *sandbox_id, runtime::v1::ContainerEventType type) ++ -> runtime::v1::ContainerEventResponse *; ++ + grpc::Status Version(grpc::ServerContext *context, const runtime::v1::VersionRequest *request, + runtime::v1::VersionResponse *reply) override; + +@@ -105,8 +109,13 @@ public: + const runtime::v1::RuntimeConfigRequest *request, + runtime::v1::RuntimeConfigResponse *reply) override; + ++ grpc::Status GetContainerEvents(grpc::ServerContext *context, ++ const runtime::v1::GetEventsRequest *request, ++ grpc::ServerWriter *writer) override; ++ + private: + std::unique_ptr m_rService; ++ bool m_enablePodEvents; + }; + + #endif // DAEMON_ENTRY_CONNECT_GRPC_CRI_V1_RUNTIME_RUNTIME_SERVICE_H +diff --git a/src/daemon/entry/connect/grpc/grpc_service.cc b/src/daemon/entry/connect/grpc/grpc_service.cc +index 61e284f3..1d8de922 100644 +--- a/src/daemon/entry/connect/grpc/grpc_service.cc ++++ b/src/daemon/entry/connect/grpc/grpc_service.cc +@@ -108,11 +108,11 @@ public: + + void Shutdown(void) + { +- m_server->Shutdown(); +- +- // call CRI to shutdown stream server ++ // call CRI to shutdown stream server, shutdown cri first to notify events thread to exit + m_criService.Shutdown(); + ++ m_server->Shutdown(); ++ + // Shutdown daemon, this operation should remove socket file. + for (const auto &address : m_socketPath) { + if (address.find(UNIX_SOCKET_PREFIX) == 0) { +diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +index cac5c0ba..e86dafae 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +@@ -1007,208 +1007,9 @@ cleanup: + return contStats; + } + +-void ContainerManagerService::PackContainerImageToStatus( +- container_inspect *inspect, std::unique_ptr &contStatus, Errors &error) ++std::unique_ptr ContainerManagerService::ContainerStatus(const std::string &containerID, Errors &error) + { +- if (inspect->config == nullptr) { +- return; +- } +- +- if (inspect->config->image != nullptr) { +- contStatus->mutable_image()->set_image(inspect->config->image); +- } +- +- contStatus->set_image_ref(CRIHelpers::ToPullableImageID(inspect->config->image, inspect->config->image_ref)); +- return; +-} +- +-void ContainerManagerService::UpdateBaseStatusFromInspect( +- container_inspect *inspect, int64_t &createdAt, int64_t &startedAt, int64_t &finishedAt, +- std::unique_ptr &contStatus) +-{ +- runtime::v1::ContainerState state { runtime::v1::CONTAINER_UNKNOWN }; +- std::string reason; +- std::string message; +- int32_t exitCode { 0 }; +- +- if (inspect->state == nullptr) { +- goto pack_status; +- } +- +- if (inspect->state->running) { +- // Container is running +- state = runtime::v1::CONTAINER_RUNNING; +- } else { +- // Container is not running. +- if (finishedAt != 0) { // Case 1 +- state = runtime::v1::CONTAINER_EXITED; +- if (inspect->state->exit_code == 0) { +- reason = "Completed"; +- } else { +- reason = "Error"; +- } +- } else if (inspect->state->exit_code != 0) { // Case 2 +- state = runtime::v1::CONTAINER_EXITED; +- finishedAt = createdAt; +- startedAt = createdAt; +- reason = "ContainerCannotRun"; +- } else { // Case 3 +- state = runtime::v1::CONTAINER_CREATED; +- } +- if (inspect->state->oom_killed) { +- reason = "OOMKilled"; +- } +- if (inspect->state->error != nullptr) { +- message = inspect->state->error; +- } +- exitCode = (int32_t)inspect->state->exit_code; +- } +- +-pack_status: +- contStatus->set_exit_code(exitCode); +- contStatus->set_state(state); +- contStatus->set_created_at(createdAt); +- contStatus->set_started_at(startedAt); +- contStatus->set_finished_at(finishedAt); +- contStatus->set_reason(reason); +- contStatus->set_message(message); +-} +- +-void ContainerManagerService::PackLabelsToStatus(container_inspect *inspect, +- std::unique_ptr &contStatus) +-{ +- if (inspect->config == nullptr || inspect->config->labels == nullptr) { +- return; +- } +- CRIHelpers::ExtractLabels(inspect->config->labels, *contStatus->mutable_labels()); +- CRIHelpers::ExtractAnnotations(inspect->config->annotations, *contStatus->mutable_annotations()); +- for (size_t i = 0; i < inspect->config->labels->len; i++) { +- if (strcmp(inspect->config->labels->keys[i], CRIHelpers::Constants::CONTAINER_LOGPATH_LABEL_KEY.c_str()) == 0) { +- contStatus->set_log_path(inspect->config->labels->values[i]); +- break; +- } +- } +-} +- +-void ContainerManagerService::ConvertMountsToStatus(container_inspect *inspect, +- std::unique_ptr &contStatus) +-{ +- for (size_t i = 0; i < inspect->mounts_len; i++) { +- runtime::v1::Mount *mount = contStatus->add_mounts(); +- mount->set_host_path(inspect->mounts[i]->source); +- mount->set_container_path(inspect->mounts[i]->destination); +- mount->set_readonly(!inspect->mounts[i]->rw); +- if (inspect->mounts[i]->propagation == nullptr || strcmp(inspect->mounts[i]->propagation, "rprivate") == 0) { +- mount->set_propagation(runtime::v1::PROPAGATION_PRIVATE); +- } else if (strcmp(inspect->mounts[i]->propagation, "rslave") == 0) { +- mount->set_propagation(runtime::v1::PROPAGATION_HOST_TO_CONTAINER); +- } else if (strcmp(inspect->mounts[i]->propagation, "rshared") == 0) { +- mount->set_propagation(runtime::v1::PROPAGATION_BIDIRECTIONAL); +- } +- // Note: Can't set SeLinuxRelabel +- } +-} +- +-void ContainerManagerService::ConvertResourcesToStatus(container_inspect *inspect, +- std::unique_ptr &contStatus) +-{ +- if (inspect->resources == nullptr) { +- return; +- } +- runtime::v1::LinuxContainerResources *resources = contStatus->mutable_resources()->mutable_linux(); +- if (inspect->resources->cpu_shares != 0) { +- resources->set_cpu_shares(inspect->resources->cpu_shares); +- } +- if (inspect->resources->cpu_period != 0) { +- resources->set_cpu_period(inspect->resources->cpu_period); +- } +- if (inspect->resources->cpu_quota != 0) { +- resources->set_cpu_quota(inspect->resources->cpu_quota); +- } +- if (inspect->resources->memory != 0) { +- resources->set_memory_limit_in_bytes(inspect->resources->memory); +- } +- if (inspect->resources->memory_swap != 0) { +- resources->set_memory_swap_limit_in_bytes(inspect->resources->memory_swap); +- } +- for (size_t i = 0; i < inspect->resources->hugetlbs_len; i++) { +- runtime::v1::HugepageLimit *hugepage = resources->add_hugepage_limits(); +- hugepage->set_page_size(inspect->resources->hugetlbs[i]->page_size); +- hugepage->set_limit(inspect->resources->hugetlbs[i]->limit); +- } +- if (inspect->resources->unified != nullptr) { +- for (size_t i = 0; i < inspect->resources->unified->len; i++) { +- auto &resUnified = *(resources->mutable_unified()); +- resUnified[inspect->resources->unified->keys[i]] = inspect->resources->unified->values[i]; +- } +- } +-} +- +-void ContainerManagerService::ContainerStatusToGRPC(container_inspect *inspect, +- std::unique_ptr &contStatus, +- Errors &error) +-{ +- if (inspect->id != nullptr) { +- contStatus->set_id(inspect->id); +- } +- +- int64_t createdAt {}; +- int64_t startedAt {}; +- int64_t finishedAt {}; +- CRIHelpers::GetContainerTimeStamps(inspect, &createdAt, &startedAt, &finishedAt, error); +- if (error.NotEmpty()) { +- return; +- } +- contStatus->set_created_at(createdAt); +- contStatus->set_started_at(startedAt); +- contStatus->set_finished_at(finishedAt); +- +- PackContainerImageToStatus(inspect, contStatus, error); +- UpdateBaseStatusFromInspect(inspect, createdAt, startedAt, finishedAt, contStatus); +- PackLabelsToStatus(inspect, contStatus); +- CRINamingV1::ParseContainerName(contStatus->annotations(), contStatus->mutable_metadata(), error); +- if (error.NotEmpty()) { +- return; +- } +- ConvertMountsToStatus(inspect, contStatus); +- ConvertResourcesToStatus(inspect, contStatus); +-} +- +-std::unique_ptr +-ContainerManagerService::ContainerStatus(const std::string &containerID, Errors &error) +-{ +- if (containerID.empty()) { +- error.SetError("Empty container id"); +- return nullptr; +- } +- +- std::string realContainerID = CRIHelpers::GetRealContainerOrSandboxID(m_cb, containerID, false, error); +- if (error.NotEmpty()) { +- ERROR("Failed to find container id %s: %s", containerID.c_str(), error.GetCMessage()); +- error.Errorf("Failed to find container id %s: %s", containerID.c_str(), error.GetCMessage()); +- return nullptr; +- } +- +- container_inspect *inspect = CRIHelpers::InspectContainer(realContainerID, error, false); +- if (error.NotEmpty()) { +- return nullptr; +- } +- if (inspect == nullptr) { +- error.SetError("Get null inspect"); +- return nullptr; +- } +- using ContainerStatusPtr = std::unique_ptr; +- ContainerStatusPtr contStatus(new (std::nothrow) runtime::v1::ContainerStatus); +- if (contStatus == nullptr) { +- error.SetError("Out of memory"); +- free_container_inspect(inspect); +- return nullptr; +- } +- +- ContainerStatusToGRPC(inspect, contStatus, error); +- +- free_container_inspect(inspect); +- return contStatus; ++ return CRIHelpersV1::GetContainerStatus(m_cb, containerID, error); + } + + void ContainerManagerService::UpdateContainerResources(const std::string &containerID, +diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.h b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.h +index 4e772bda..50f5ed69 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.h ++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.h +@@ -111,19 +111,6 @@ private: + std::unique_ptr &container); + void SetFsUsage(const imagetool_fs_info *fs_usage, int64_t timestamp, + std::unique_ptr &container); +- void ContainerStatusToGRPC(container_inspect *inspect, +- std::unique_ptr &contStatus, Errors &error); +- void PackContainerImageToStatus(container_inspect *inspect, +- std::unique_ptr &contStatus, Errors &error); +- void UpdateBaseStatusFromInspect(container_inspect *inspect, int64_t &createdAt, int64_t &startedAt, +- int64_t &finishedAt, +- std::unique_ptr &contStatus); +- void PackLabelsToStatus(container_inspect *inspect, +- std::unique_ptr &contStatus); +- void ConvertMountsToStatus(container_inspect *inspect, +- std::unique_ptr &contStatus); +- void ConvertResourcesToStatus(container_inspect *inspect, +- std::unique_ptr &contStatus); + void ExecSyncFromGRPC(const std::string &containerID, const google::protobuf::RepeatedPtrField &cmd, + int64_t timeout, container_exec_request **request, Errors &error); + auto ValidateExecRequest(const runtime::v1::ExecRequest &req, Errors &error) -> int; +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +index f125e714..4291d8a0 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +@@ -36,6 +36,7 @@ + #include "sandbox_manager.h" + #include "transform.h" + #include "isulad_config.h" ++#include "mailbox.h" + + namespace CRIV1 { + void PodSandboxManagerService::PrepareSandboxData(const runtime::v1::PodSandboxConfig &config, +@@ -302,6 +303,7 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1::PodSandboxConfig + std::string jsonCheckpoint; + std::string network_setting_json; + runtime::v1::PodSandboxConfig copyConfig = config; ++ cri_container_message_t msg = { 0 }; + + // Step 1: Parepare sandbox name, runtime and networkMode + PrepareSandboxData(config, runtimeHandler, sandboxName, runtimeInfo, networkMode, error); +@@ -372,6 +374,11 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1::PodSandboxConfig + goto cleanup_network; + } + ++ msg.container_id = sandbox->GetId().c_str(); ++ msg.sandbox_id = sandbox->GetId().c_str(); ++ msg.type = CRI_CONTAINER_MESSAGE_TYPE_CREATED; ++ mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &msg); ++ + // Step 10: Save network settings json to disk + // Update network settings before start sandbox since sandbox container will use the sandbox key + if (namespace_is_cni(networkMode.c_str())) { +@@ -391,6 +398,9 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1::PodSandboxConfig + return response_id; + } + ++ msg.type = CRI_CONTAINER_MESSAGE_TYPE_STARTED; ++ mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &msg); ++ + return sandbox->GetId(); + + cleanup_network: +@@ -700,6 +710,13 @@ void PodSandboxManagerService::RemovePodSandbox(const std::string &podSandboxID, + ERROR("Failed to delete sandbox %s: %s", podSandboxID.c_str(), error.GetCMessage()); + } + ++ if (error.Empty()) { ++ cri_container_message_t msg = { 0 }; ++ msg.container_id = sandbox->GetId().c_str(); ++ msg.sandbox_id = sandbox->GetId().c_str(); ++ msg.type = CRI_CONTAINER_MESSAGE_TYPE_DELETED; ++ mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &msg); ++ } + } + + auto PodSandboxManagerService::SharesHostNetwork(const container_inspect *inspect) -> runtime::v1::NamespaceMode +@@ -800,10 +817,29 @@ void PodSandboxManagerService::SetSandboxStatusNetwork(std::shared_ptr +-PodSandboxManagerService::PodSandboxStatus(const std::string &podSandboxID, Errors &error) ++void PodSandboxManagerService::GetContainerStatuses(const std::string &podSandboxID, ++ std::vector> &containerStatuses, ++ std::vector &errors) { ++ auto list_response_wrapper = GetContainerListResponse(podSandboxID, errors); ++ if (list_response_wrapper == nullptr) { ++ return; ++ } ++ ++ auto list_response = list_response_wrapper->get(); ++ // Remove all containers in the sandbox. ++ for (size_t i = 0; i < list_response->containers_len; i++) { ++ Errors stError; ++ containerStatuses.push_back(CRIHelpersV1::GetContainerStatus(m_cb, list_response->containers[i]->id, stError)); ++ if (stError.NotEmpty()) { ++ ERROR("Error get container status: %s: %s", list_response->containers[i]->id, stError.GetCMessage()); ++ errors.push_back(stError.GetMessage()); ++ } ++ } ++} ++ ++std::unique_ptr PodSandboxManagerService::GetPodSandboxStatus(const std::string &podSandboxID, Errors &error) + { +- std::unique_ptr podStatus(new runtime::v1::PodSandboxStatus); ++ std::unique_ptr podStatus(new (std::nothrow) runtime::v1::PodSandboxStatus); + if (podStatus == nullptr) { + ERROR("Out of memory"); + error.SetError("Out of memory"); +@@ -831,6 +867,50 @@ PodSandboxManagerService::PodSandboxStatus(const std::string &podSandboxID, Erro + return podStatus; + } + ++void PodSandboxManagerService::PodSandboxStatus(const std::string &podSandboxID, ++ runtime::v1::PodSandboxStatusResponse *reply, Errors &error) ++{ ++ if (reply == nullptr) { ++ ERROR("Invalid NULL reply"); ++ error.SetError("Invalid NULL reply"); ++ return; ++ } ++ ++ ++ auto podStatus = GetPodSandboxStatus(podSandboxID, error); ++ if (error.NotEmpty()) { ++ ERROR("Failed to get pod sandbox status: %s", error.GetCMessage()); ++ return; ++ } ++ ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(podSandboxID); ++ if (sandbox == nullptr) { ++ ERROR("Failed to find sandbox id %s", podSandboxID.c_str()); ++ error.Errorf("Failed to find sandbox id %s", podSandboxID.c_str()); ++ return; ++ } ++ ++ *(reply->mutable_status()) = *podStatus; ++ ++ ++ if (!m_enablePodEvents) { ++ return; ++ } ++ ++ std::vector> containerStatuses; ++ std::vector errors; ++ GetContainerStatuses(sandbox->GetId(), containerStatuses, errors); ++ if (errors.size() != 0) { ++ error.SetAggregate(errors); ++ return; ++ } ++ ++ for (auto &containerStatus : containerStatuses) { ++ *(reply->add_containers_statuses()) = *containerStatus; ++ } ++ return; ++} ++ + void PodSandboxManagerService::ListPodSandbox(const runtime::v1::PodSandboxFilter &filter, + std::vector> &pods, + Errors &error) +@@ -944,7 +1024,7 @@ void PodSandboxManagerService::GetPodSandboxNetworkMetrics(const std::string &ne + void PodSandboxManagerService::PackagePodSandboxStatsAttributes( + const std::string &id, std::unique_ptr &podStatsPtr, Errors &error) + { +- auto status = PodSandboxStatus(id, error); ++ auto status = GetPodSandboxStatus(id, error); + if (error.NotEmpty()) { + return; + } +@@ -1111,8 +1191,8 @@ auto PodSandboxManagerService::PodSandboxStats(const std::string &podSandboxID, + auto &config = sandbox->GetSandboxConfig(); + auto oldStatsRec = sandbox->GetStatsInfo(); + +- auto status = PodSandboxStatus(sandbox->GetId(), tmpErr); +- if (error.NotEmpty()) { ++ auto status = GetPodSandboxStatus(sandbox->GetId(), tmpErr); ++ if (tmpErr.NotEmpty()) { + ERROR("Failed to get podsandbox %s status: %s", sandbox->GetId().c_str(), tmpErr.GetCMessage()); + error.Errorf("Failed to get podsandbox %s status", sandbox->GetId().c_str()); + return nullptr; +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h +index c3d98b8c..3872c4c9 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h +@@ -38,10 +38,11 @@ namespace CRIV1 { + class PodSandboxManagerService { + public: + PodSandboxManagerService(const std::string &podSandboxImage, service_executor_t *cb, +- std::shared_ptr pluginManager) ++ std::shared_ptr pluginManager, bool enablePodEvents) + : m_podSandboxImage(podSandboxImage) + , m_cb(cb) + , m_pluginManager(pluginManager) ++ , m_enablePodEvents(enablePodEvents) + { + } + PodSandboxManagerService(const PodSandboxManagerService &) = delete; +@@ -55,8 +56,7 @@ public: + + void RemovePodSandbox(const std::string &podSandboxID, Errors &error); + +- auto PodSandboxStatus(const std::string &podSandboxID, Errors &error) +- -> std::unique_ptr; ++ void PodSandboxStatus(const std::string &podSandboxID, runtime::v1::PodSandboxStatusResponse *reply, Errors &error); + + void ListPodSandbox(const runtime::v1::PodSandboxFilter &filter, + std::vector> &pods, Errors &error); +@@ -129,6 +129,9 @@ private: + std::vector &podSandboxIDs, Errors &error); + void ApplySandboxLinuxOptions(const runtime::v1::LinuxPodSandboxConfig &lc, host_config *hc, + container_config *custom_config, Errors &error); ++ auto GetPodSandboxStatus(const std::string &podSandboxID, Errors &error) -> std::unique_ptr; ++ void GetContainerStatuses(const std::string &podSandboxID, std::vector> &containerStatuses, ++ std::vector &errors); + + private: + std::string m_podSandboxImage; +@@ -136,6 +139,7 @@ private: + std::map m_networkReady; + service_executor_t *m_cb { nullptr }; + std::shared_ptr m_pluginManager { nullptr }; ++ bool m_enablePodEvents; + }; + } // namespace CRI + +diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service.h b/src/daemon/entry/cri/v1/v1_cri_runtime_service.h +index 839f6724..4521e3df 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_runtime_service.h ++++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service.h +@@ -70,8 +70,8 @@ public: + + virtual void RemovePodSandbox(const std::string &podSandboxID, Errors &error) = 0; + +- virtual auto PodSandboxStatus(const std::string &podSandboxID, +- Errors &error) -> std::unique_ptr = 0; ++ virtual void PodSandboxStatus(const std::string &podSandboxID, runtime::v1::PodSandboxStatusResponse *reply, ++ Errors &error) = 0; + + virtual void ListPodSandbox(const runtime::v1::PodSandboxFilter &filter, + std::vector> &pods, Errors &error) = 0; +diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc +index aa5ae516..7b40e29d 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc +@@ -19,11 +19,12 @@ + + namespace CRIV1 { + CRIRuntimeServiceImpl::CRIRuntimeServiceImpl(const std::string &podSandboxImage, service_executor_t *cb, +- std::shared_ptr pluginManager) ++ std::shared_ptr pluginManager, bool enablePodEvents) + : m_runtimeVersioner(new RuntimeVersionerService(cb)) + , m_containerManager(new ContainerManagerService(cb)) +- , m_podSandboxManager(new PodSandboxManagerService(podSandboxImage, cb, pluginManager)) ++ , m_podSandboxManager(new PodSandboxManagerService(podSandboxImage, cb, pluginManager, enablePodEvents)) + , m_runtimeManager(new RuntimeManagerService(cb, pluginManager)) ++ , m_enablePodEvents(enablePodEvents) + { + } + +@@ -124,10 +125,9 @@ void CRIRuntimeServiceImpl::RemovePodSandbox(const std::string &podSandboxID, Er + m_podSandboxManager->RemovePodSandbox(podSandboxID, error); + } + +-auto CRIRuntimeServiceImpl::PodSandboxStatus(const std::string &podSandboxID, Errors &error) +--> std::unique_ptr ++void CRIRuntimeServiceImpl::PodSandboxStatus(const std::string &podSandboxID, runtime::v1::PodSandboxStatusResponse *reply, Errors &error) + { +- return m_podSandboxManager->PodSandboxStatus(podSandboxID, error); ++ m_podSandboxManager->PodSandboxStatus(podSandboxID, reply, error); + } + + void CRIRuntimeServiceImpl::ListPodSandbox(const runtime::v1::PodSandboxFilter &filter, +diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h +index 0a25749f..6ae59bfa 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h ++++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h +@@ -26,7 +26,8 @@ namespace CRIV1 { + class CRIRuntimeServiceImpl : public CRIRuntimeService { + public: + CRIRuntimeServiceImpl(const std::string &podSandboxImage, service_executor_t *cb, +- std::shared_ptr pluginManager); ++ std::shared_ptr pluginManager, ++ bool enablePodEvents); + CRIRuntimeServiceImpl(const CRIRuntimeServiceImpl &) = delete; + auto operator=(const CRIRuntimeServiceImpl &) -> CRIRuntimeServiceImpl & = delete; + virtual ~CRIRuntimeServiceImpl() = default; +@@ -72,8 +73,7 @@ public: + + void RemovePodSandbox(const std::string &podSandboxID, Errors &error) override; + +- auto PodSandboxStatus(const std::string &podSandboxID, Errors &error) +- -> std::unique_ptr override; ++ void PodSandboxStatus(const std::string &podSandboxID, runtime::v1::PodSandboxStatusResponse *reply, Errors &error) override; + + void ListPodSandbox(const runtime::v1::PodSandboxFilter &filter, + std::vector> &pods, Errors &error) override; +@@ -103,6 +103,7 @@ protected: + private: + std::string m_podSandboxImage; + std::shared_ptr m_pluginManager { nullptr }; ++ bool m_enablePodEvents; + }; + } // namespace CRIV1 + #endif // DAEMON_ENTRY_CRI_V1_CRI_RUNTIME_SERVICE_IMPL_H +diff --git a/src/daemon/executor/container_cb/execution.c b/src/daemon/executor/container_cb/execution.c +index 88c6b354..e5c96628 100644 +--- a/src/daemon/executor/container_cb/execution.c ++++ b/src/daemon/executor/container_cb/execution.c +@@ -62,6 +62,7 @@ + #include "event_type.h" + #include "utils_timestamp.h" + #include "utils_verify.h" ++#include "mailbox.h" + #ifdef ENABLE_NATIVE_NETWORK + #include "service_network_api.h" + +@@ -542,6 +543,9 @@ static int container_start_cb(const container_start_request *request, container_ + container_t *cont = NULL; + int sync_fd = -1; + pthread_t thread_id = 0; ++#ifdef ENABLE_CRI_API_V1 ++ cri_container_message_t message; ++#endif + + DAEMON_CLEAR_ERRMSG(); + +@@ -596,6 +600,15 @@ static int container_start_cb(const container_start_request *request, container_ + EVENT("Event: {Object: %s, Type: Running}", id); + (void)isulad_monitor_send_container_event(id, START, -1, 0, NULL, NULL); + ++#ifdef ENABLE_CRI_API_V1 ++ if (is_container_in_sandbox(cont->common_config->sandbox_info)) { ++ message.container_id = id; ++ message.sandbox_id = cont->common_config->sandbox_info->id; ++ message.type = CRI_CONTAINER_MESSAGE_TYPE_STARTED; ++ mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &message); ++ } ++#endif ++ + pack_response: + handle_start_io_thread_by_cc(cc, sync_fd, thread_id); + delete_daemon_fifos(fifopath, (const char **)fifos); +@@ -1009,6 +1022,9 @@ static int container_delete_cb(const container_delete_request *request, containe + char *name = NULL; + char *id = NULL; + container_t *cont = NULL; ++#ifdef ENABLE_CRI_API_V1 ++ cri_container_message_t message; ++#endif + + DAEMON_CLEAR_ERRMSG(); + if (request == NULL || response == NULL) { +@@ -1063,6 +1079,15 @@ static int container_delete_cb(const container_delete_request *request, containe + + EVENT("Event: {Object: %s, Type: Deleted}", id); + ++#ifdef ENABLE_CRI_API_V1 ++ if (is_container_in_sandbox(cont->common_config->sandbox_info)) { ++ message.container_id = cont->common_config->id; ++ message.sandbox_id = cont->common_config->sandbox_info->id; ++ message.type = CRI_CONTAINER_MESSAGE_TYPE_DELETED; ++ mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &message); ++ } ++#endif ++ + pack_response: + pack_delete_response(*response, cc, id); + container_unref(cont); +diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c +index e00afb68..a9102226 100644 +--- a/src/daemon/executor/container_cb/execution_create.c ++++ b/src/daemon/executor/container_cb/execution_create.c +@@ -62,6 +62,7 @@ + #include "opt_log.h" + #include "runtime_api.h" + #include "id_name_manager.h" ++#include "mailbox.h" + + #ifdef ENABLE_CRI_API_V1 + static bool validate_sandbox_info(const container_sandbox_info *sandbox) +@@ -1389,6 +1390,9 @@ int container_create_cb(const container_create_request *request, container_creat + bool skip_id_name_manage = false; + bool skip_sandbox_key_manage = false; + __isula_auto_sysinfo_t sysinfo_t *sysinfo = NULL; ++#ifdef ENABLE_CRI_API_V1 ++ cri_container_message_t message; ++#endif + + DAEMON_CLEAR_ERRMSG(); + +@@ -1572,6 +1576,14 @@ int container_create_cb(const container_create_request *request, container_creat + + EVENT("Event: {Object: %s, Type: Created %s}", id, name); + (void)isulad_monitor_send_container_event(id, CREATE, -1, 0, NULL, NULL); ++#ifdef ENABLE_CRI_API_V1 ++ if (is_container_in_sandbox(request->sandbox)) { ++ message.container_id = id; ++ message.sandbox_id = request->sandbox->id; ++ message.type = CRI_CONTAINER_MESSAGE_TYPE_CREATED; ++ mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &message); ++ } ++#endif + goto pack_response; + + umount_channel: +diff --git a/src/daemon/mailbox/CMakeLists.txt b/src/daemon/mailbox/CMakeLists.txt +new file mode 100644 +index 00000000..984f9acb +--- /dev/null ++++ b/src/daemon/mailbox/CMakeLists.txt +@@ -0,0 +1,11 @@ ++# get current directory sources files ++aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} mailbox_top_srcs) ++ ++set(MAILBOX_SRCS ++ ${mailbox_top_srcs} ++ PARENT_SCOPE ++ ) ++set(MAILBOX_INCS ++ ${CMAKE_CURRENT_SOURCE_DIR} ++ PARENT_SCOPE ++ ) +\ No newline at end of file +diff --git a/src/daemon/mailbox/mailbox.c b/src/daemon/mailbox/mailbox.c +new file mode 100644 +index 00000000..732b91b9 +--- /dev/null ++++ b/src/daemon/mailbox/mailbox.c +@@ -0,0 +1,167 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide common event definition ++ ******************************************************************************/ ++ ++#include "mailbox.h" ++ ++#include ++ ++#include "message_queue.h" ++#include "mailbox_message.h" ++#include "message_subscriber.h" ++ ++mailbox_topic_handler_t mailbox_topic_handlers[MAILBOX_TOPIC_MAX] = { 0 }; ++ ++static bool mailbox_topic_valid(mailbox_topic topic) { ++ return topic > MAILBOX_TOPIC_INVALID && topic < MAILBOX_TOPIC_MAX; ++} ++ ++static bool mailbox_should_publish(mailbox_topic topic) ++{ ++ if (!mailbox_topic_valid(topic)) { ++ ERROR("Invalid topic %d", topic); ++ return false; ++ } ++ ++ if (!mailbox_topic_handlers[topic].registered) { ++ return false; ++ } ++ ++ if (mailbox_topic_handlers[topic].queue == NULL) { ++ return true; ++ } ++ ++ // for async queues, only publish if anyone subscribe ++ return message_queue_have_subscribers(mailbox_topic_handlers[topic].queue); ++} ++ ++// only register once when iSulad start, no need to free the queue ++int mailbox_register_topic_handler(mailbox_topic topic, message_generator_t generator, void *context, ++ message_release_t release, bool async) ++{ ++ if (!mailbox_topic_valid(topic)) { ++ ERROR("Invalid topic %d", topic); ++ return -1; ++ } ++ ++ if (generator == NULL) { ++ ERROR("Invalid generator for topic %d", topic); ++ return -1; ++ } ++ ++ mailbox_topic_handlers[topic].generator = generator; ++ mailbox_topic_handlers[topic].context = context; ++ mailbox_topic_handlers[topic].release = release; ++ if (async) { ++ mailbox_topic_handlers[topic].queue = message_queue_create(release); ++ if (mailbox_topic_handlers[topic].queue == NULL) { ++ ERROR("Failed to create message queue for topic %d", topic); ++ return -1; ++ } ++ } ++ mailbox_topic_handlers[topic].registered = true; ++ return 0; ++} ++ ++// unregister only when iSulad shutdown, no need to free the queue ++void mailbox_unregister_topic_handler(mailbox_topic topic) ++{ ++ if (!mailbox_topic_valid(topic)) { ++ ERROR("Invalid topic %d", topic); ++ return; ++ } ++ ++ if (mailbox_topic_handlers[topic].queue != NULL) { ++ message_queue_shutdown(mailbox_topic_handlers[topic].queue); ++ } ++ mailbox_topic_handlers[topic].registered = false; ++} ++ ++void mailbox_publish(mailbox_topic topic, void *data) ++{ ++ if (!mailbox_should_publish(topic)) { ++ return; ++ } ++ ++ message_generator_t generator = mailbox_topic_handlers[topic].generator; ++ void *context = mailbox_topic_handlers[topic].context; ++ message_release_t release = mailbox_topic_handlers[topic].release; ++ message_queue *queue = mailbox_topic_handlers[topic].queue; ++ ++ if (generator == NULL) { ++ ERROR("No handler for topic %d", topic); ++ return; ++ } ++ ++ void *middle = generator(context, data); ++ if (middle == NULL) { ++ return; ++ } ++ ++ if (queue != NULL) { ++ mailbox_message *msg = mailbox_message_create(middle, release); ++ if (msg == NULL) { ++ ERROR("Failed to create mailbox message"); ++ if (release) { ++ release(middle); ++ } ++ return; ++ } ++ if (message_queue_publish(queue, msg) != 0) { ++ ERROR("Failed to publish event"); ++ mailbox_message_unref(msg); ++ return; ++ } ++ } ++} ++ ++message_subscriber *mailbox_subscribe(mailbox_topic topic) ++{ ++ if (!mailbox_topic_valid(topic)) { ++ ERROR("Invalid topic %d", topic); ++ return NULL; ++ } ++ ++ if (!mailbox_topic_handlers[topic].registered) { ++ ERROR("Handler for topic %d not registered", topic); ++ return NULL; ++ } ++ ++ if (mailbox_topic_handlers[topic].queue != NULL) { ++ return message_queue_subscribe(mailbox_topic_handlers[topic].queue, ++ mailbox_topic_handlers[topic].release); ++ } ++ ++ // For sync queues, there is no need to subscribe, just return ++ return NULL; ++} ++ ++void mailbox_unsubscribe(mailbox_topic topic, message_subscriber *sub) ++{ ++ if (!mailbox_topic_valid(topic)) { ++ ERROR("Invalid topic %d", topic); ++ return; ++ } ++ ++ if (!mailbox_topic_handlers[topic].registered) { ++ ERROR("Handler for topic %d not registered", topic); ++ return; ++ } ++ ++ if (mailbox_topic_handlers[topic].queue != NULL) { ++ return message_queue_unsubscribe(mailbox_topic_handlers[topic].queue, sub); ++ } ++ ++ return; ++} +diff --git a/src/daemon/mailbox/mailbox.h b/src/daemon/mailbox/mailbox.h +new file mode 100644 +index 00000000..1dc2e934 +--- /dev/null ++++ b/src/daemon/mailbox/mailbox.h +@@ -0,0 +1,82 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide common event definition ++ ******************************************************************************/ ++ ++#ifndef DAEMON_MAILBOX_MAILBOX_H ++#define DAEMON_MAILBOX_MAILBOX_H ++ ++#include "daemon_arguments.h" ++#include "blocking_queue.h" ++#include "message_queue.h" ++#include "message_subscriber.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++typedef enum { ++ MAILBOX_TOPIC_INVALID = -1, ++ MAILBOX_TOPIC_CRI_CONTAINER, ++ MAILBOX_TOPIC_MAX ++} mailbox_topic; ++ ++// for async message, it generates a true message to publish ++// for sync message, it is the callback function to handle the data to publish ++typedef void *(*message_generator_t)(void *, void *); ++// release function of message generated by generator, if any ++typedef void (*message_release_t)(void *); ++ ++typedef struct { ++ // to generate a message ++ message_generator_t generator; ++ // context of handler ++ void *context; ++ // release function of message, if any ++ message_release_t release; ++ // message queue ++ message_queue *queue; ++ // if registered ++ bool registered; ++} mailbox_topic_handler_t; ++ ++typedef enum { ++ CRI_CONTAINER_MESSAGE_TYPE_INVALID = -1, ++ CRI_CONTAINER_MESSAGE_TYPE_CREATED, ++ CRI_CONTAINER_MESSAGE_TYPE_STARTED, ++ CRI_CONTAINER_MESSAGE_TYPE_STOPPED, ++ CRI_CONTAINER_MESSAGE_TYPE_DELETED, ++ CRI_CONTAINER_MESSAGE_TYPE_MAX ++} cri_container_message_type; ++ ++typedef struct { ++ const char *container_id; ++ const char *sandbox_id; ++ cri_container_message_type type; ++} cri_container_message_t; ++ ++int mailbox_register_topic_handler(mailbox_topic topic, message_generator_t handle, void *context, ++ message_release_t release, bool async); ++ ++void mailbox_unregister_topic_handler(mailbox_topic topic); ++ ++void mailbox_publish(mailbox_topic topic, void *data); ++ ++message_subscriber *mailbox_subscribe(mailbox_topic topic); ++ ++void mailbox_unsubscribe(mailbox_topic, message_subscriber *sub); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +diff --git a/src/daemon/mailbox/mailbox_message.c b/src/daemon/mailbox/mailbox_message.c +new file mode 100644 +index 00000000..b16a1bdd +--- /dev/null ++++ b/src/daemon/mailbox/mailbox_message.c +@@ -0,0 +1,94 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide mailbox message definition ++ ******************************************************************************/ ++ ++#include "mailbox_message.h" ++ ++#include ++ ++#include "utils.h" ++ ++// Once the create succeeds, the ownership is transferred to the mailbox_message. ++mailbox_message *mailbox_message_create(void *data, void (*destroy)(void *)) { ++ __isula_auto_free mailbox_message *msg = NULL; ++ msg = util_common_calloc_s(sizeof(mailbox_message)); ++ if (msg == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ msg->data = data; ++ msg->destroy = destroy; ++ msg->ref_count = 1; ++ ++ if (pthread_mutex_init(&msg->lock, NULL) != 0) { ++ ERROR("Failed to init mutex"); ++ return NULL; ++ } ++ ++ return isula_transfer_ptr(msg); ++} ++ ++int mailbox_message_ref(mailbox_message *dest) { ++ __isula_auto_pm_unlock pthread_mutex_t *lock = NULL; ++ if (dest == NULL) { ++ ERROR("Invalid mailbox_message"); ++ return -1; ++ } ++ ++ if (pthread_mutex_lock(&dest->lock) != 0) { ++ ERROR("Failed to lock mutex"); ++ return -1; ++ } ++ lock = &dest->lock; ++ ++ if (dest->ref_count == INT_MAX) { ++ ERROR("Reference count overflow"); ++ return -1; ++ } ++ ++ dest->ref_count++; ++ ++ return 0; ++} ++ ++void mailbox_message_unref(mailbox_message *dest) { ++ __isula_auto_pm_unlock pthread_mutex_t *lock = NULL; ++ if (dest == NULL) { ++ return; ++ } ++ ++ if (pthread_mutex_lock(&dest->lock) != 0) { ++ ERROR("Failed to lock mutex"); ++ return; ++ } ++ lock = &dest->lock; ++ ++ if (dest->ref_count == 0) { ++ ERROR("Reference count underflow, should not reach here"); ++ return; ++ } ++ ++ dest->ref_count--; ++ if (dest->ref_count == 0) { ++ if (dest->destroy) { ++ dest->destroy(dest->data); ++ } ++ lock = NULL; ++ (void)pthread_mutex_unlock(&dest->lock); ++ (void)pthread_mutex_destroy(&dest->lock); ++ free(dest); ++ } ++ return; ++} +diff --git a/src/daemon/mailbox/mailbox_message.h b/src/daemon/mailbox/mailbox_message.h +new file mode 100644 +index 00000000..39e40b70 +--- /dev/null ++++ b/src/daemon/mailbox/mailbox_message.h +@@ -0,0 +1,50 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide ref counted ptr definition ++ ******************************************************************************/ ++ ++#ifndef DAEMON_MAILBOX_MAILBOX_MESSAGE_H ++#define DAEMON_MAILBOX_MAILBOX_MESSAGE_H ++ ++#include ++#include ++ ++#include ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++typedef struct mailbox_message { ++ void *data; ++ size_t ref_count; ++ pthread_mutex_t lock; ++ void (*destroy)(void *); ++} mailbox_message; ++ ++mailbox_message *mailbox_message_create(void *ptr, void (*destroy)(void *)); ++ ++int mailbox_message_ref(mailbox_message *p); ++ ++void mailbox_message_unref(mailbox_message *p); ++ ++// define auto free function callback for mailbox_message ++define_auto_cleanup_callback(mailbox_message_unref, mailbox_message); ++// define auto free macro for char * ++#define __isula_auto_mailbox_message auto_cleanup_tag(mailbox_message_unref) ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +diff --git a/src/daemon/mailbox/message_queue.c b/src/daemon/mailbox/message_queue.c +new file mode 100644 +index 00000000..7fe044f2 +--- /dev/null ++++ b/src/daemon/mailbox/message_queue.c +@@ -0,0 +1,234 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide message queue definition ++ ******************************************************************************/ ++ ++#include "message_queue.h" ++ ++#include ++#include ++ ++#include "utils.h" ++ ++// default set subscriber timeout to 1000ms, maybe could be configured later ++const int64_t subscribe_timeout = 1000; ++ ++static void message_queue_subscriber_free(void *key, void *val) ++{ ++ return; ++} ++ ++static void *message_queue_thread(void *arg) ++{ ++ int ret = 0; ++ ++ ret = pthread_detach(pthread_self()); ++ if (ret != 0) { ++ CRIT("Set thread detach fail"); ++ return NULL; ++ } ++ ++ prctl(PR_SET_NAME, "Message Queue"); ++ ++ message_queue *mq = (message_queue *)arg; ++ if (mq == NULL) { ++ ERROR("Invalid argument"); ++ return NULL; ++ } ++ ++ for (;;) { ++ void *data = NULL; ++ if (blocking_queue_pop(mq->messages, &data) != 0) { ++ ERROR("Fail to get message, message queue thread exit"); ++ break; ++ } ++ ++ __isula_auto_mailbox_message mailbox_message *msg = (mailbox_message *)data; ++ // an empty msg indicates shutdown ++ if (pthread_rwlock_rdlock(&mq->rwlock) != 0) { ++ ERROR("Failed to lock rwlock"); ++ continue; ++ } ++ ++ bool should_shutdown = (msg == NULL); ++ map_itor *itor = map_itor_new(mq->subscribers); ++ if (itor == NULL) { ++ ERROR("Out of memory"); ++ if (pthread_rwlock_unlock(&mq->rwlock) != 0) { ++ ERROR("Failed to lock rwlock"); ++ } ++ break; ++ } ++ ++ for (; map_itor_valid(itor); map_itor_next(itor)) { ++ void *sub = map_itor_key(itor); ++ if (should_shutdown) { ++ message_subscriber_shutdown(sub); ++ } else { ++ if (message_subscriber_push(sub, msg) != 0) { ++ ERROR("Failed to push event to subscriber"); ++ } ++ } ++ } ++ map_itor_free(itor); ++ ++ if (pthread_rwlock_unlock(&mq->rwlock) != 0) { ++ ERROR("Failed to unlock rwlock"); ++ } ++ ++ // if msg is NULL, it is a shutdown signal ++ if (should_shutdown) { ++ break; ++ } ++ } ++ ++ return NULL; ++} ++ ++message_queue *message_queue_create(void (*release)(void *)) ++{ ++ __isula_auto_free message_queue *mq = NULL; ++ __isula_auto_blocking_queue blocking_queue *bq = NULL; ++ pthread_t message_queue_tid; ++ mq = util_common_calloc_s(sizeof(message_queue)); ++ if (mq == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ bq = blocking_queue_create(BLOCKING_QUEUE_NO_TIMEOUT, release); ++ if (bq == NULL) { ++ ERROR("Failed to create events queue"); ++ return NULL; ++ } ++ ++ mq->subscribers = map_new(MAP_PTR_INT, MAP_DEFAULT_CMP_FUNC, message_queue_subscriber_free); ++ if (mq->subscribers == NULL) { ++ ERROR("Failed to create subscribers map"); ++ return NULL; ++ } ++ ++ if (pthread_rwlock_init(&mq->rwlock, NULL) != 0) { ++ ERROR("Failed to init rwlock"); ++ map_free(mq->subscribers); ++ return NULL; ++ } ++ ++ if (pthread_create(&message_queue_tid, NULL, message_queue_thread, mq) != 0) { ++ ERROR("Failed to create message queue thread"); ++ pthread_rwlock_destroy(&mq->rwlock); ++ map_free(mq->subscribers); ++ return NULL; ++ } ++ ++ mq->messages = isula_transfer_ptr(bq); ++ return isula_transfer_ptr(mq); ++} ++ ++// message queue should be global value, it will be destroyed when daemon exit ++void message_queue_shutdown(message_queue *mq) ++{ ++ if (mq == NULL) { ++ return; ++ } ++ ++ blocking_queue_clear(mq->messages); ++ ++ // push a nullptr to notify the thread to exit ++ if (blocking_queue_push(mq->messages, NULL) != 0) { ++ ERROR("Failed to push nullptr to message queue"); ++ } ++} ++ ++message_subscriber *message_queue_subscribe(message_queue *mq, void (*release)(void *)) ++{ ++ __isula_auto_subscriber message_subscriber *sub = NULL; ++ __isula_auto_prw_unlock pthread_rwlock_t *lock = NULL; ++ int val = 0; ++ if (mq == NULL) { ++ ERROR("Invalid argument"); ++ return NULL; ++ } ++ ++ sub = message_subscriber_create(subscribe_timeout, release); ++ if (sub == NULL) { ++ ERROR("Failed to create subscriber"); ++ return NULL; ++ } ++ ++ if (pthread_rwlock_wrlock(&mq->rwlock) != 0) { ++ ERROR("Failed to lock rwlock"); ++ return NULL; ++ } ++ lock = &mq->rwlock; ++ ++ if (map_insert(mq->subscribers, sub, (void *)&val) == false) { ++ ERROR("Failed to insert subscriber"); ++ return NULL; ++ } ++ ++ return isula_transfer_ptr(sub); ++} ++ ++void message_queue_unsubscribe(message_queue *mq, message_subscriber *sub) ++{ ++ __isula_auto_prw_unlock pthread_rwlock_t *lock = NULL; ++ if (mq == NULL) { ++ ERROR("Invalid argument"); ++ return; ++ } ++ ++ if (pthread_rwlock_wrlock(&mq->rwlock) != 0) { ++ ERROR("Failed to lock rwlock"); ++ return; ++ } ++ lock = &mq->rwlock; ++ ++ if (map_remove(mq->subscribers, sub) == false) { ++ ERROR("Failed to remove subscriber"); ++ return; ++ } ++ ++ return; ++} ++ ++int message_queue_publish(message_queue *mq, mailbox_message *msg) ++{ ++ if (mq == NULL || msg == NULL) { ++ ERROR("Invalid argument"); ++ return -1; ++ } ++ ++ if (blocking_queue_push(mq->messages, msg) != 0) { ++ ERROR("Failed to push message"); ++ return -1; ++ } ++ return 0; ++} ++ ++bool message_queue_have_subscribers(message_queue *mq) ++{ ++ __isula_auto_prw_unlock pthread_rwlock_t *lock = NULL; ++ if (mq == NULL) { ++ ERROR("Invalid argument"); ++ return false; ++ } ++ ++ if (pthread_rwlock_wrlock(&mq->rwlock) != 0) { ++ ERROR("Failed to lock rwlock"); ++ return false; ++ } ++ lock = &mq->rwlock; ++ ++ return map_size(mq->subscribers) > 0; ++} +diff --git a/src/daemon/mailbox/message_queue.h b/src/daemon/mailbox/message_queue.h +new file mode 100644 +index 00000000..7905840f +--- /dev/null ++++ b/src/daemon/mailbox/message_queue.h +@@ -0,0 +1,57 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide message queue definition ++ ******************************************************************************/ ++ ++#ifndef DAEMON_MESSAGE_MESSAGE_QUEUE_H ++#define DAEMON_MESSAGE_MESSAGE_QUEUE_H ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++#include ++ ++#include "blocking_queue.h" ++#include "mailbox_message.h" ++#include "map.h" ++#include "message_subscriber.h" ++ ++typedef struct message_queue { ++ blocking_queue *messages; ++ ++ // lock for set of subscribers ++ pthread_rwlock_t rwlock; ++ ++ map_t *subscribers; ++ ++ int64_t sub_timeout; ++} message_queue; ++ ++message_queue *message_queue_create(void (*release)(void *)); ++ ++void message_queue_shutdown(message_queue *mq); ++ ++message_subscriber *message_queue_subscribe(message_queue *mq, void (*release)(void *)); ++ ++void message_queue_unsubscribe(message_queue *mq, message_subscriber *sub); ++ ++int message_queue_publish(message_queue *mq, mailbox_message *msg); ++ ++bool message_queue_have_subscribers(message_queue *mq); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +diff --git a/src/daemon/mailbox/message_subscriber.c b/src/daemon/mailbox/message_subscriber.c +new file mode 100644 +index 00000000..8ef3cb58 +--- /dev/null ++++ b/src/daemon/mailbox/message_subscriber.c +@@ -0,0 +1,85 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide message subscriber definition ++ ******************************************************************************/ ++ ++#include "message_subscriber.h" ++ ++#include ++ ++#include "utils.h" ++ ++message_subscriber *message_subscriber_create(int64_t timeout, void (*release)(void *)) ++{ ++ message_subscriber *sub = (message_subscriber *)util_common_calloc_s(sizeof(message_subscriber)); ++ if (sub == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ sub->queue = blocking_queue_create(timeout, release); ++ if (sub->queue == NULL) { ++ ERROR("Failed to create blocking queue"); ++ free(sub); ++ return NULL; ++ } ++ return sub; ++} ++ ++int message_subscriber_push(message_subscriber *sub, mailbox_message *msg) ++{ ++ if (sub == NULL || msg == NULL) { ++ ERROR("Invalid argument"); ++ return -1; ++ } ++ ++ if (mailbox_message_ref(msg) != 0) { ++ ERROR("Failed to get message"); ++ return -1; ++ } ++ ++ if (blocking_queue_push(sub->queue, msg) != 0) { ++ ERROR("Failed to push message to queue"); ++ mailbox_message_unref(msg); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++int message_subscriber_pop(message_subscriber *sub, mailbox_message **msg) ++{ ++ if (sub == NULL) { ++ ERROR("Invalid argument"); ++ return -1; ++ } ++ return blocking_queue_pop(sub->queue, (void **)msg); ++} ++ ++void message_subscriber_shutdown(message_subscriber *sub) ++{ ++ if (sub == NULL) { ++ return; ++ } ++ ++ blocking_queue_clear(sub->queue); ++ (void)blocking_queue_push(sub->queue, NULL); ++} ++ ++void message_subscriber_destroy(message_subscriber *sub) ++{ ++ if (sub == NULL) { ++ return; ++ } ++ blocking_queue_destroy(sub->queue); ++ free(sub); ++} +diff --git a/src/daemon/mailbox/message_subscriber.h b/src/daemon/mailbox/message_subscriber.h +new file mode 100644 +index 00000000..de4574d9 +--- /dev/null ++++ b/src/daemon/mailbox/message_subscriber.h +@@ -0,0 +1,41 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide message subscriber definition ++ ******************************************************************************/ ++ ++#ifndef SRC_DAEMON_MAILBOX_MESSAGE_SUBSCRIBER_H ++#define SRC_DAEMON_MAILBOX_MESSAGE_SUBSCRIBER_H ++ ++#include "blocking_queue.h" ++#include "mailbox_message.h" ++ ++typedef struct { ++ blocking_queue *queue; ++} message_subscriber; ++ ++message_subscriber *message_subscriber_create(int64_t timeout, void (*release)(void *)); ++ ++void message_subscriber_shutdown(message_subscriber *sub); ++ ++void message_subscriber_destroy(message_subscriber *sub); ++ ++int message_subscriber_push(message_subscriber *sub, mailbox_message *msg); ++ ++int message_subscriber_pop(message_subscriber *sub, mailbox_message **msg); ++ ++// define auto free function callback for blocking queue ++define_auto_cleanup_callback(message_subscriber_destroy, message_subscriber); ++// define auto free macro for blocking queue ++#define __isula_auto_subscriber auto_cleanup_tag(message_subscriber_destroy) ++ ++#endif +diff --git a/src/daemon/modules/api/container_api.h b/src/daemon/modules/api/container_api.h +index 830fd696..55c59980 100644 +--- a/src/daemon/modules/api/container_api.h ++++ b/src/daemon/modules/api/container_api.h +@@ -289,6 +289,11 @@ static inline bool is_sandbox_container(container_sandbox_info *sandbox) + { + return sandbox != NULL && sandbox->is_sandbox_container; + } ++ ++static inline bool is_container_in_sandbox(container_sandbox_info *sandbox) ++{ ++ return sandbox != NULL && !sandbox->is_sandbox_container; ++} + #endif + + #if defined(__cplusplus) || defined(c_plusplus) +diff --git a/src/daemon/modules/container/supervisor/supervisor.c b/src/daemon/modules/container/supervisor/supervisor.c +index 1b7da383..83d46268 100644 +--- a/src/daemon/modules/container/supervisor/supervisor.c ++++ b/src/daemon/modules/container/supervisor/supervisor.c +@@ -38,6 +38,7 @@ + #include "container_api.h" + #include "event_type.h" + #include "utils_file.h" ++#include "mailbox.h" + #ifdef ENABLE_CRI_API_V1 + #include "sandbox_ops.h" + #endif +@@ -51,6 +52,7 @@ struct supervisor_handler_data { + int fd; + int exit_code; + char *name; ++ char *sandbox_name; + char *runtime; + bool is_sandbox_container; + pid_ppid_info_t pid_info; +@@ -152,6 +154,9 @@ static void supervisor_handler_data_free(struct supervisor_handler_data *data) + free(data->name); + data->name = NULL; + ++ free(data->sandbox_name); ++ data->sandbox_name = NULL; ++ + free(data->runtime); + data->runtime = NULL; + +@@ -172,6 +177,9 @@ static void *clean_resources_thread(void *arg) + pid_t pid = data->pid_info.pid; + int retry_count = 0; + int max_retry = 10; ++#ifdef ENABLE_CRI_API_V1 ++ cri_container_message_t msg; ++#endif + + ret = pthread_detach(pthread_self()); + if (ret != 0) { +@@ -218,6 +226,13 @@ retry: + (void)isulad_monitor_send_container_event(name, STOPPED, (int)pid, data->exit_code, NULL, NULL); + + #ifdef ENABLE_CRI_API_V1 ++ if (data->sandbox_name) { ++ msg.container_id = name; ++ msg.sandbox_id = data->sandbox_name; ++ msg.type = CRI_CONTAINER_MESSAGE_TYPE_STOPPED; ++ mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &msg); ++ } ++ + if (data->is_sandbox_container) { + if (sandbox_on_sandbox_exit(name, data->exit_code) < 0) { + ERROR("Failed to handle sandbox %s exit", name); +@@ -329,6 +344,9 @@ int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const p + data->runtime = util_strdup_s(cont->runtime); + #ifdef ENABLE_CRI_API_V1 + data->is_sandbox_container = is_sandbox_container(cont->common_config->sandbox_info); ++ if (is_container_in_sandbox(cont->common_config->sandbox_info)) { ++ data->sandbox_name = util_strdup_s(cont->common_config->sandbox_info->id); ++ } + #endif + data->pid_info.pid = pid_info->pid; + data->pid_info.start_time = pid_info->start_time; +diff --git a/src/daemon/sandbox/sandbox.cc b/src/daemon/sandbox/sandbox.cc +index 7b6496ed..c70116c1 100644 +--- a/src/daemon/sandbox/sandbox.cc ++++ b/src/daemon/sandbox/sandbox.cc +@@ -37,6 +37,7 @@ + #include "cxxutils.h" + #include "controller_manager.h" + #include "utils_timestamp.h" ++#include "mailbox.h" + + #define SANDBOX_READY_STATE_STR "SANDBOX_READY" + #define SANDBOX_NOTREADY_STATE_STR "SANDBOX_NOTREADY" +@@ -527,6 +528,14 @@ void Sandbox::OnSandboxExit(const ControllerExitInfo &exitInfo) + if (!SaveState(error)) { + ERROR("Failed to save sandbox state, %s", m_id.c_str()); + } ++ ++ if (error.Empty()) { ++ cri_container_message_t msg = { 0 }; ++ msg.container_id = GetId().c_str(); ++ msg.sandbox_id = GetId().c_str(); ++ msg.type = CRI_CONTAINER_MESSAGE_TYPE_STOPPED; ++ mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &msg); ++ } + } + + auto Sandbox::UpdateStatus(Errors &error) -> bool +diff --git a/src/utils/cutils/blocking_queue.c b/src/utils/cutils/blocking_queue.c +new file mode 100644 +index 00000000..7c9c5f50 +--- /dev/null ++++ b/src/utils/cutils/blocking_queue.c +@@ -0,0 +1,185 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide blocking queue definition ++ ******************************************************************************/ ++ ++#include "blocking_queue.h" ++ ++#include ++#include ++#include ++ ++#include "utils.h" ++#include "utils_timestamp.h" ++ ++// create blocking queue with timeout(ms), if timeout < 0, then with no timeout ++blocking_queue *blocking_queue_create(int64_t timeout, void (*release)(void *)) ++{ ++ __isula_auto_free blocking_queue *queue = NULL; ++ __isula_auto_free blocking_node *node = NULL; ++ queue = (blocking_queue *)util_common_calloc_s(sizeof(blocking_queue)); ++ if (queue == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ node = (blocking_node *)util_common_calloc_s(sizeof(blocking_node)); ++ if (node == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ if (pthread_mutex_init(&queue->lock, NULL) != 0) { ++ ERROR("Failed to init mutex"); ++ return NULL; ++ } ++ ++ if (pthread_cond_init(&queue->not_empty, NULL) != 0) { ++ ERROR("Failed to init cond"); ++ (void)pthread_mutex_destroy(&queue->lock); ++ return NULL; ++ } ++ ++ queue->head = node; ++ queue->tail = node; ++ node = NULL; ++ queue->release = release; ++ ++ if (timeout >= 0) { ++ queue->timeout.tv_sec = timeout / (Time_Second / Time_Milli); ++ queue->timeout.tv_nsec = (timeout % (Time_Second / Time_Milli) ) * Time_Milli; ++ } else { ++ queue->timeout.tv_sec = -1; ++ } ++ ++ return isula_transfer_ptr(queue); ++} ++ ++int blocking_queue_push(blocking_queue *queue, void *data) ++{ ++ __isula_auto_free blocking_node *new_node = NULL; ++ __isula_auto_pm_unlock pthread_mutex_t *lock = NULL; ++ if (queue == NULL) { ++ ERROR("Invalid NULL arguments"); ++ return -1; ++ } ++ ++ new_node = (blocking_node *)util_common_calloc_s(sizeof(blocking_node)); ++ if (new_node == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ new_node->data = data; ++ new_node->next = NULL; ++ ++ if (pthread_mutex_lock(&queue->lock) != 0) { ++ ERROR("Failed to lock mutex"); ++ return -1; ++ } ++ lock = &queue->lock; ++ ++ queue->tail->next = new_node; ++ queue->tail = new_node; ++ new_node = NULL; ++ ++ if (pthread_cond_broadcast(&queue->not_empty) != 0) { ++ ERROR("Failed to broadcast cond"); ++ } ++ ++ return 0; ++} ++ ++int blocking_queue_pop(blocking_queue *queue, void **data) { ++ if (queue == NULL || data == NULL) { ++ ERROR("Invalid NULL arguments"); ++ return -1; ++ } ++ ++ __isula_auto_pm_unlock pthread_mutex_t *lock = NULL; ++ if (pthread_mutex_lock(&queue->lock) != 0) { ++ ERROR("Failed to lock mutex"); ++ return -1; ++ } ++ lock = &queue->lock; ++ ++ while (queue->head->next == NULL) { ++ if (queue->timeout.tv_sec >= 0) { ++ int ret = pthread_cond_timedwait(&queue->not_empty, &queue->lock, &queue->timeout); ++ if (ret != 0) { ++ if (ret != ETIMEDOUT) { ++ ERROR("Failed to wait cond"); ++ } ++ return ret; ++ } ++ } else { ++ int ret = pthread_cond_wait(&queue->not_empty, &queue->lock); ++ if (ret != 0) { ++ ERROR("Failed to wait cond"); ++ return ret; ++ } ++ } ++ } ++ ++ blocking_node *old_head = queue->head; ++ blocking_node *new_head = old_head->next; ++ *data = new_head->data; ++ queue->head = new_head; ++ ++ free(old_head); ++ return 0; ++} ++ ++void blocking_queue_clear(blocking_queue *queue) ++{ ++ if (queue == NULL) { ++ return; ++ } ++ ++ __isula_auto_pm_unlock pthread_mutex_t *lock = NULL; ++ // clear all nodes in queue ++ if (queue == NULL) { ++ ERROR("Invalid NULL arguments"); ++ return; ++ } ++ ++ if (pthread_mutex_lock(&queue->lock) != 0) { ++ ERROR("Failed to lock mutex"); ++ return; ++ } ++ lock = &queue->lock; ++ ++ while (queue->head->next != NULL) { ++ blocking_node *old_head = queue->head; ++ blocking_node *new_head = old_head->next; ++ if (queue->release) { ++ queue->release(old_head->data); ++ } ++ free(old_head); ++ queue->head = new_head; ++ } ++} ++ ++// ensure there is no other thread executing enqueue or dequeue operation ++void blocking_queue_destroy(blocking_queue *queue) ++{ ++ if (queue == NULL) { ++ return; ++ } ++ ++ blocking_queue_clear(queue); ++ ++ (void)pthread_mutex_destroy(&queue->lock); ++ ++ (void)pthread_cond_destroy(&queue->not_empty); ++ ++ free(queue); ++} +diff --git a/src/utils/cutils/blocking_queue.h b/src/utils/cutils/blocking_queue.h +new file mode 100644 +index 00000000..1c52a9d3 +--- /dev/null ++++ b/src/utils/cutils/blocking_queue.h +@@ -0,0 +1,66 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide blocking queue definition ++ ******************************************************************************/ ++ ++#ifndef DAEMON_UTILS_CUTILS_BLOCKING_QUEUE_H ++#define DAEMON_UTILS_CUTILS_BLOCKING_QUEUE_H ++ ++#include ++#include ++#include ++ ++#include "utils_timestamp.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++#define BLOCKING_QUEUE_NO_TIMEOUT -1 ++ ++typedef struct blocking_node { ++ void *data; ++ struct blocking_node *next; ++} blocking_node; ++ ++typedef struct blocking_queue { ++ blocking_node *head; ++ blocking_node *tail; ++ pthread_mutex_t lock; ++ struct timespec timeout; ++ pthread_cond_t not_empty; ++ void (*release)(void *); ++} blocking_queue; ++ ++// create blocking queue with timeout(ms), if timeout < 0, then with no timeout ++blocking_queue *blocking_queue_create(int64_t timeout, void (*release)(void *)); ++ ++int blocking_queue_push(blocking_queue *queue, void *data); ++ ++int blocking_queue_pop(blocking_queue *queue, void **data); ++ ++void blocking_queue_clear(blocking_queue *queue); ++ ++// ensure there is no other thread executing enqueue or dequeue operation ++void blocking_queue_destroy(blocking_queue *queue); ++ ++// define auto free function callback for blocking queue ++define_auto_cleanup_callback(blocking_queue_destroy, blocking_queue); ++// define auto free macro for blocking queue ++#define __isula_auto_blocking_queue auto_cleanup_tag(blocking_queue_destroy) ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +diff --git a/test/mocks/mailbox_mock.cc b/test/mocks/mailbox_mock.cc +new file mode 100644 +index 00000000..601b804e +--- /dev/null ++++ b/test/mocks/mailbox_mock.cc +@@ -0,0 +1,30 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2022. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-04-02 ++ * Description: mailbox mock ++ ******************************************************************************/ ++ ++#include "mailbox_mock.h" ++ ++MockMailbox *g_mailbox_mock = nullptr; ++ ++void Mailbox_SetMock(MockMailbox* mock) ++{ ++ g_mailbox_mock = mock; ++} ++ ++void mailbox_publish(mailbox_topic topic, void *data) ++{ ++ if (g_mailbox_mock != nullptr) { ++ g_mailbox_mock->MailboxPublish(topic, data); ++ } ++} +diff --git a/test/mocks/mailbox_mock.h b/test/mocks/mailbox_mock.h +new file mode 100644 +index 00000000..ce48f0fc +--- /dev/null ++++ b/test/mocks/mailbox_mock.h +@@ -0,0 +1,30 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2022. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-04-02 ++ * Description: mailbox mock ++ ******************************************************************************/ ++ ++#ifndef _ISULAD_TEST_MOCKS_MAILBOX_MOCK_H ++#define _ISULAD_TEST_MOCKS_MAILBOX_MOCK_H ++ ++#include ++#include "mailbox.h" ++ ++class MockMailbox { ++public: ++ virtual ~MockMailbox() = default; ++ MOCK_METHOD2(MailboxPublish, void(mailbox_topic topic, void *data)); ++}; ++ ++void Mailbox_SetMock(MockMailbox* mock); ++ ++#endif +diff --git a/test/sandbox/controller/shim/CMakeLists.txt b/test/sandbox/controller/shim/CMakeLists.txt +index 6423bb80..26a66e51 100644 +--- a/test/sandbox/controller/shim/CMakeLists.txt ++++ b/test/sandbox/controller/shim/CMakeLists.txt +@@ -7,6 +7,7 @@ add_executable(${EXE} + ${CMAKE_BINARY_DIR}/grpc/src/api/services/cri/gogo.pb.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cri/v1/v1_cri_helpers.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cri/v1/v1_cri_security_context.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cri/v1/v1_naming.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cri/checkpoint_handler.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cri/cri_constants.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cri/cri_helpers.cc +diff --git a/test/sandbox/sandbox/CMakeLists.txt b/test/sandbox/sandbox/CMakeLists.txt +index 138d4d8d..2a35388f 100644 +--- a/test/sandbox/sandbox/CMakeLists.txt ++++ b/test/sandbox/sandbox/CMakeLists.txt +@@ -23,6 +23,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/grpc_sandboxer_client_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/controller_stub_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/shim_controller_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/mailbox_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c + ${CMAKE_CURRENT_SOURCE_DIR}/sandbox_ut.cc) + +@@ -33,6 +34,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/entry/cri + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/executor ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/mailbox + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/controller + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/controller/shim + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/controller/sandboxer +-- +2.34.1 + diff --git a/0041-fix-cpurt-init-bug-for-systemd-cgroup.patch b/0041-fix-cpurt-init-bug-for-systemd-cgroup.patch new file mode 100644 index 0000000..1f5e9eb --- /dev/null +++ b/0041-fix-cpurt-init-bug-for-systemd-cgroup.patch @@ -0,0 +1,74 @@ +From fe11b34a3c2843ea2198b310160b182d63aeb63b Mon Sep 17 00:00:00 2001 +From: jikai +Date: Tue, 2 Apr 2024 11:22:09 +0800 +Subject: [PATCH 41/69] fix cpurt init bug for systemd-cgroup + +Signed-off-by: jikai +--- + src/daemon/common/cgroup/cgroup.c | 13 +++++++------ + src/daemon/executor/container_cb/execution.c | 13 +++++++------ + 2 files changed, 14 insertions(+), 12 deletions(-) + +diff --git a/src/daemon/common/cgroup/cgroup.c b/src/daemon/common/cgroup/cgroup.c +index d3f1445a..007dbb70 100644 +--- a/src/daemon/common/cgroup/cgroup.c ++++ b/src/daemon/common/cgroup/cgroup.c +@@ -146,17 +146,18 @@ char *common_convert_cgroup_path(const char *cgroup_path) + return NULL; + } + +- // for cgroup fs cgroup path, return directly +- if (!util_has_suffix(cgroup_path, ".slice")) { +- return util_strdup_s(cgroup_path); +- } +- + // for systemd cgroup, cgroup_path should have the form slice:prefix:id, + // convert it to a true path, such as from test-a.slice:isulad:id + // to test.slice/test-a.slice/isulad-id.scope + arr = util_string_split_n(cgroup_path, ':', 3); + if (arr == NULL || util_array_len((const char **)arr) != 3) { +- ERROR("Invalid systemd cgroup parent"); ++ // not a systemd cgroup, return cgroup path directly ++ return util_strdup_s(cgroup_path); ++ } ++ ++ // for cgroup fs cgroup path, return directly ++ if (!util_has_suffix(arr[0], ".slice")) { ++ ERROR("Invalid systemd cgroup path: %s", cgroup_path); + return NULL; + } + +diff --git a/src/daemon/executor/container_cb/execution.c b/src/daemon/executor/container_cb/execution.c +index 88c6b354..4bf3621d 100644 +--- a/src/daemon/executor/container_cb/execution.c ++++ b/src/daemon/executor/container_cb/execution.c +@@ -435,11 +435,12 @@ static int cpurt_controller_init(const char *id, const host_config *host_spec) + } + + if (conf_get_systemd_cgroup()) { +- // currently it is the same as docker, yet it is unclear that +- // if systemd cgroup is used and cgroup parent is set to a slice rather than system.slice +- // should iSulad set cpu.rt_runtime_us and cpu.rt_period_us for the parent path? +- // in fact, even if system.slice is used, +- // cpu.rt_runtime_us and cpu.rt_period_us might still needed to be set manually ++ __isula_auto_free char *converted_cgroup = common_convert_cgroup_path(cgroups_path); ++ if (converted_cgroup == NULL) { ++ ERROR("Failed to convert cgroup path"); ++ return -1; ++ } ++ + __isula_auto_free char *init_cgroup = common_get_init_cgroup_path("cpu"); + if (init_cgroup == NULL) { + ERROR("Failed to get init cgroup"); +@@ -451,7 +452,7 @@ static int cpurt_controller_init(const char *id, const host_config *host_spec) + ERROR("Failed to get own cgroup"); + return -1; + } +- char *new_cgroups_path = util_path_join(init_cgroup, cgroups_path); ++ char *new_cgroups_path = util_path_join(init_cgroup, converted_cgroup); + if (new_cgroups_path == NULL) { + ERROR("Failed to join path"); + return -1; +-- +2.34.1 + diff --git a/0042-fix-message-queue-concurrent-bug.patch b/0042-fix-message-queue-concurrent-bug.patch new file mode 100644 index 0000000..063742a --- /dev/null +++ b/0042-fix-message-queue-concurrent-bug.patch @@ -0,0 +1,41 @@ +From f90a145d9d29682295aebf2bcd30865ee5f6491f Mon Sep 17 00:00:00 2001 +From: jikai +Date: Tue, 2 Apr 2024 07:53:54 +0000 +Subject: [PATCH 42/69] fix message queue concurrent bug + +Signed-off-by: jikai +--- + src/daemon/mailbox/message_queue.c | 7 ++++--- + 1 file changed, 4 insertions(+), 3 deletions(-) + +diff --git a/src/daemon/mailbox/message_queue.c b/src/daemon/mailbox/message_queue.c +index 7fe044f2..7e53301e 100644 +--- a/src/daemon/mailbox/message_queue.c ++++ b/src/daemon/mailbox/message_queue.c +@@ -106,11 +106,12 @@ message_queue *message_queue_create(void (*release)(void *)) + return NULL; + } + +- bq = blocking_queue_create(BLOCKING_QUEUE_NO_TIMEOUT, release); +- if (bq == NULL) { ++ mq->messages = blocking_queue_create(BLOCKING_QUEUE_NO_TIMEOUT, release); ++ if (mq->messages == NULL) { + ERROR("Failed to create events queue"); + return NULL; + } ++ bq = mq->messages; + + mq->subscribers = map_new(MAP_PTR_INT, MAP_DEFAULT_CMP_FUNC, message_queue_subscriber_free); + if (mq->subscribers == NULL) { +@@ -131,7 +132,7 @@ message_queue *message_queue_create(void (*release)(void *)) + return NULL; + } + +- mq->messages = isula_transfer_ptr(bq); ++ bq = NULL; + return isula_transfer_ptr(mq); + } + +-- +2.34.1 + diff --git a/0043-specify-runtime-as-runc-for-oom-test-CI.patch b/0043-specify-runtime-as-runc-for-oom-test-CI.patch new file mode 100644 index 0000000..3f807c8 --- /dev/null +++ b/0043-specify-runtime-as-runc-for-oom-test-CI.patch @@ -0,0 +1,26 @@ +From 7af700c4021ef9961aaac37ffa5767bd4f3dd184 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Tue, 2 Apr 2024 08:00:37 +0000 +Subject: [PATCH 43/69] specify runtime as runc for oom test CI + +Signed-off-by: jikai +--- + CI/test_cases/container_cases/inspect.sh | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/CI/test_cases/container_cases/inspect.sh b/CI/test_cases/container_cases/inspect.sh +index 86aed3d8..5d976281 100755 +--- a/CI/test_cases/container_cases/inspect.sh ++++ b/CI/test_cases/container_cases/inspect.sh +@@ -147,7 +147,7 @@ function test_inspect_spec() + isula rm -f $containername + + # use more than 10m memory limit, otherwise it might fail to run +- isula run -it -m 10m --name $containername $ubuntu_image perl -e 'for ($i = 0; $i < 100000000; $i++) { $a .= " " x 1024 }' ++ isula run -it -m 10m --runtime runc --name $containername $ubuntu_image perl -e 'for ($i = 0; $i < 100000000; $i++) { $a .= " " x 1024 }' + + isula inspect -f "{{json .State.OOMKilled}} {{.Name}}" $containername 2>&1 | sed -n '1p' | grep "true" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container with image: ${ubuntu_image}" && ((ret++)) +-- +2.34.1 + diff --git a/0044-set-oomkilled-in-cri.patch b/0044-set-oomkilled-in-cri.patch new file mode 100644 index 0000000..a9d21ea --- /dev/null +++ b/0044-set-oomkilled-in-cri.patch @@ -0,0 +1,27 @@ +From 5393ce7d02bb73ce4760edefa959dfb4846f1958 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Tue, 2 Apr 2024 11:19:06 +0000 +Subject: [PATCH 44/69] set oomkilled in cri + +Signed-off-by: jikai +--- + src/daemon/common/cri/v1/v1_cri_helpers.cc | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.cc b/src/daemon/common/cri/v1/v1_cri_helpers.cc +index a3488894..ea5c8bb5 100644 +--- a/src/daemon/common/cri/v1/v1_cri_helpers.cc ++++ b/src/daemon/common/cri/v1/v1_cri_helpers.cc +@@ -506,6 +506,9 @@ void UpdateBaseStatusFromInspect( + } else { // Case 3 + state = runtime::v1::CONTAINER_CREATED; + } ++ if (inspect->state->oom_killed == true) { ++ reason = "OOMKilled"; ++ } + if (inspect->state->error != nullptr) { + message = inspect->state->error; + } +-- +2.34.1 + diff --git a/0045-add-cri-1.29-update-design-doc.patch b/0045-add-cri-1.29-update-design-doc.patch new file mode 100644 index 0000000..7098ecf --- /dev/null +++ b/0045-add-cri-1.29-update-design-doc.patch @@ -0,0 +1,312 @@ +From a1f8641f7b440362a615e72246c92654008f61e9 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Thu, 22 Feb 2024 11:09:50 +0800 +Subject: [PATCH 45/69] add cri 1.29 update design doc + +Signed-off-by: jikai +--- + README.md | 2 +- + docs/design/README_zh.md | 3 +- + .../detailed/CRI/CRI_1.29_update_design.md | 237 ++++++++++++++++++ + .../cri_api_1.29_GetContainerEvents.svg | 8 + + docs/images/cri_api_1.29_overview.svg | 5 + + 5 files changed, 253 insertions(+), 2 deletions(-) + create mode 100644 docs/design/detailed/CRI/CRI_1.29_update_design.md + create mode 100644 docs/images/cri_api_1.29_GetContainerEvents.svg + create mode 100644 docs/images/cri_api_1.29_overview.svg + +diff --git a/README.md b/README.md +index 970b6e72..694ddbc2 100644 +--- a/README.md ++++ b/README.md +@@ -240,4 +240,4 @@ iSulad Version | Kubernetes Version | CRI Version + --- | --- | --- + v2.0.0+ | v1.13-v1.18 | v1alpha2 + v2.0.8+ | v1.19-v1.22 | v1alpha2 +-v2.1.4+ | v1.23-v1.26 | v1, v1alpha2 ++v2.1.4+ | v1.23-v1.29 | v1, v1alpha2 +diff --git a/docs/design/README_zh.md b/docs/design/README_zh.md +index b7ec3ddb..3382bfbe 100644 +--- a/docs/design/README_zh.md ++++ b/docs/design/README_zh.md +@@ -69,4 +69,5 @@ + + - 查看 Sandbox 模块的设计文档： [sandbox_design_zh](./detailed/Sandbox/sandbox_design_zh.md) 。 + - 查看 Controller 模块的设计文档： [controller_design_zh](./detailed/Sandbox/controller_design_zh.md) 。 +-- 查看 CRI V1 模块的设计文档： [podsandbox_cri_interface_design](./detailed/Sandbox/podsandbox_cri_interface_design.md) 。 +\ No newline at end of file ++- 查看 CRI V1 模块的设计文档： [podsandbox_cri_interface_design](./detailed/Sandbox/podsandbox_cri_interface_design.md) 。 ++- 查看 CRI 1.29 更新模块的设计文档： [cri_1.29_update_design](./detailed/CRI/CRI_1.29_update_design.md) 。 +diff --git a/docs/design/detailed/CRI/CRI_1.29_update_design.md b/docs/design/detailed/CRI/CRI_1.29_update_design.md +new file mode 100644 +index 00000000..0a0c860b +--- /dev/null ++++ b/docs/design/detailed/CRI/CRI_1.29_update_design.md +@@ -0,0 +1,237 @@ ++| Author | 吉凯 | ++| ------ | ----------------------| ++| Date | 2024-02-20 | ++| Email | jikai11@huawei.com | ++ ++### 方案目标 ++ ++当前iSulad CRI接口版本采用的1.25版本, 预期升级至1.29版本，iSulad CRI接口需要对升级后的新增CRI字段进行补充。 ++ ++该需求需要iSulad对K8S新版本CRI接口1.29提供支持，在K8S 1.25及之前，K8S存在CRI V1alpha2 和 CRI V1两种版本的CRI接口， ++但从1.26开始，K8S开始仅提供对于新版本CRI V1的支持，对1.25及之前的CRI接口，此次升级中，iSulad引用CRI V1， ++对于1.25及之前的CRI接口，V1alpha2和V1功能保持一致，1.26及之后新增的特性仅在CRI V1中提供支持。 ++ ++CRI V1使能： ++ ++1、编译iSulad时开启ENABLE_CRI_API_V1编译选项 ++ ++```bash ++cmake ../ -D ENABLE_CRI_API_V1=ON ++``` ++ ++2、daemon.json中enable-cri-v1设置为true ++ ++```json ++{ ++ "group": "isula", ++ "default-runtime": "runc", ++ "graph": "/var/lib/isulad", ++ "state": "/var/run/isulad", ++ "log-level": "ERROR", ++ "pidfile": "/var/run/isulad.pid", ++ ... ++ "enable-cri-v1": true, ++} ++``` ++ ++### 总体设计 ++ ++#### 变更文档 ++ ++ ++ ++#### 总体设计图 ++ ++![](../../../images/cri_api_1.29_overview.svg) ++ ++#### 此次更新包含以下变更 ++ ++##### [kubelet: get cgroup driver config from CRI](https://github.com/kubernetes/kubernetes/pull/118770) ++ ++`RuntimeConfig` 获取cgroup驱动配置 cgroupfs 或 systemd-cgroup ++ ++##### [Add Support for Evented PLEG](https://github.com/kubernetes/kubernetes/pull/111384) ++ ++`GetContainerEents`，提供对pod生命周期相关事件流 ++ ++`PodSandboxStatus`有相应调整 ++ ++##### [Add swap to stats to Summary API and Prometheus endpoints (/stats/summary and /metrics/resource)](https://github.com/kubernetes/kubernetes/pull/118865) ++ ++`ContainerStats`新增虚拟内存使用情况信息： `SwapUsage` ++ ++##### 变更排查 ++ ++1、[Propose comment to standardize the reason field](https://github.com/kubernetes/kubernetes/pull/112977) ++ ++Must be set to "OOMKilled" for containers terminated by cgroup-based Out-of-Memory killer. ++ ++container status中reason字段在cgroup out-of-memory时应该设置为OOMKilled ++ ++2、[Improve the API description of `PodSecurityContext.SupplementalGroups` to clarify its unfamiliar behavior](https://github.com/kubernetes/kubernetes/pull/113047) ++ ++描述修改，优化`PodSecurityContext.SupplementalGroups`的注释，明确容器镜像定义的主UID不在该列表下的行为 ++ ++- Clarified the expected behavior of `SupplementalGroups` field of `PodSecurityContext` ++ ++```txt ++ // List of groups applied to the first process run in the sandbox, in ++ // addition to the sandbox's primary GID, and group memberships defined ++ // in the container image for the sandbox's primary UID of the container process. ++ // If the list is empty, no additional groups are added to any container. ++ // Note that group memberships defined in the container image for the sandbox's primary UID ++ // of the container process are still effective, even if they are not included in this list. ++``` ++ ++3、[cri-api: document expectation of 16 MB limit](https://github.com/kubernetes/kubernetes/pull/110435) ++ ++```txt ++ // Captured command stdout output. ++ // The runtime should cap the output of this response to 16MB. ++ // If the stdout of the command produces more than 16MB, the remaining output ++ // should be discarded, and the command should proceed with no error. ++ // See CVE-2022-1708 and CVE-2022-31030 for more information. ++``` ++ ++##### 其它不涉及 ++ ++1、windows相关 ++ ++[CRI: Add Windows Podsandbox Stats](https://github.com/kubernetes/kubernetes/pull/110754) ++ ++[Windows hostnetwork alpha](https://github.com/kubernetes/kubernetes/pull/112961) ++ ++[Expose commit memory used in WindowsMemoryUsage struct](https://github.com/kubernetes/kubernetes/pull/119238) ++ ++2、iSulad 底层暂不支持 ++ ++[Add mappings for volumes](https://github.com/kubernetes/kubernetes/pull/116377) ++ ++[CRI: Add CDI device info for containers](https://github.com/kubernetes/kubernetes/pull/115891/) ++ ++3、K8S KEP 初期，其它容器引擎未支持，iSulad暂不支持 ++ ++[kubelet: add support for broadcasting metrics from CRI](https://github.com/kubernetes/kubernetes/pull/113609) ++ ++4、其它 ++ ++[Add user specified image to CRI ContainerConfig](https://github.com/kubernetes/kubernetes/pull/118652) ++ ++[Add runtime handler field to ImageSpec struct](https://github.com/kubernetes/kubernetes/pull/121121) ++ ++[cri-api: fix comment lines about PROPAGATION_PRIVATE](https://github.com/kubernetes/kubernetes/pull/115704) ++ ++[Kubelet disk api cri update](https://github.com/kubernetes/kubernetes/pull/120914) ++ ++### 详细设计 ++ ++#### Add swap stats to ContainerStats ++ ++`ContainerStats`新增虚拟内存使用情况信息： `SwapUsage` ++ ++```mermaid ++sequenceDiagram ++ participant kubelet ++ participant isulad ++ participant runc ++ participant cgroup ++ ++ kubelet->>isulad:CRI request(ContainerStats)isulad->>isulad:container stats request ++ isulad->>runc:container create request(w/wo --sytemd-cgroup) ++ runc->>cgroup:read cgroup file ++ runc->>isulad:container stats info ++ isulad ->> kubelet:response ++``` ++ ++#### define reason to ContainerStatus ++ ++container status中reason字段在cgroup out-of-memory时应该设置为OOMKilled ++ ++```mermaid ++sequenceDiagram ++ participant kubelet ++ participant isulad ++ participant runc ++ participant cgroup ++ ++ kubelet->>isulad:CRI request ++ alt CreateContainer&StartContainer ++ isulad->>runc:container create request(w memory limit set) ++ runc ->> cgroup:create cgroup path ++ isulad->>cgroup:monitor cgroup oom event ++ cgroup->>isulad:oom triggered ++ else ContainerStatus ++ isulad->>isulad:get container status ++ end ++ isulad ->> kubelet:response ++``` ++ ++#### kubelet: get cgroup driver config from CRI ++ ++KEP： ++ ++##### RuntimeConfig CRI API修改 ++ ++新增rpc，获取cgroup驱动配置: cgroupfs 或者 systemd-cgroup ++ ++##### iSulad RuntimeConfig 设计 ++ ++仅提供对runc的支持 ++ ++配置文件daemon.json: systemd-cgroup: true表示使用systemd-cgroup否则使用cgroupfs ++ ++在iSulad配置cgroup driver为systemd-cgroup时，底层runc采用对应的systemd-cgroup ++ ++```mermaid ++sequenceDiagram ++ participant kubelet ++ participant isulad ++ participant runc ++ participant cgroup ++ ++ kubelet->>isulad:CRI request ++ alt RuntimeConfig ++ isulad->>isulad:get cgroup driver from conf ++ else RunPodSandbox/CreateContainer ++ isulad->>isulad:get cgroup driver from conf ++ isulad->>runc:container create request(w/wo --sytemd-cgroup) ++ runc ->> cgroup:create cgroup path ++ end ++ isulad ->> kubelet:response ++``` ++ ++#### Add Support for Evented PLEG ++ ++##### GetContainerEvents CRI API修改 ++ ++`GetContainerEvents` rpc接口此前尚未支持，此次提供对该接口的支持 ++ ++##### iSulad GetContainerEvents设计 ++ ++CRIContainerEvents 模块 ++ ++增加接口mailbox_publish ++ ++事件包括沙箱和容器以下几种类型生命周期事件 ++ ++```proto ++enum ContainerEventType { ++ // Container created ++ CONTAINER_CREATED_EVENT = 0; ++ ++ // Container started ++ CONTAINER_STARTED_EVENT = 1; ++ ++ // Container stopped ++ CONTAINER_STOPPED_EVENT = 2; ++ ++ // Container deleted ++ CONTAINER_DELETED_EVENT = 3; ++} ++``` ++ ++![](../../../images/cri_api_1.29_GetContainerEvents.svg) ++ ++### 使用限制 ++ ++以上特性仅保证容器运行时设置为runc时支持。 +diff --git a/docs/images/cri_api_1.29_GetContainerEvents.svg b/docs/images/cri_api_1.29_GetContainerEvents.svg +new file mode 100644 +index 00000000..be855b5b +--- /dev/null ++++ b/docs/images/cri_api_1.29_GetContainerEvents.svg +@@ -0,0 +1,8 @@ ++ ++ ++ ++ ++ ++ ++ ++ KubletCRI requestGetContainerEventsiSuladCRI ModuleSandboxModuleContainer Runtime ModuleGetContainerEventsMailbox ModulePublishSubscribe +\ No newline at end of file +diff --git a/docs/images/cri_api_1.29_overview.svg b/docs/images/cri_api_1.29_overview.svg +new file mode 100644 +index 00000000..94911913 +--- /dev/null ++++ b/docs/images/cri_api_1.29_overview.svg +@@ -0,0 +1,5 @@ ++ ++ ++ ++ ++ ListPodSandboxStatsPodSandboxStatsPodSandboxStatsContainerStatsSwapUsageListContainerStatsContainerStatsLinuxPodSandboxStatsGetContainerEventsContainerEventResponse无变更新增修改接口内部数据RuntimeConfigLinuxRuntimeConfigurationCgroupDriverPodSandboxStatusPodSandboxStatusResponseContainerStatusContainerStatusContainerStatusResponse +\ No newline at end of file +-- +2.34.1 + diff --git a/0046-oom-monitor-in-manual-cases.patch b/0046-oom-monitor-in-manual-cases.patch new file mode 100644 index 0000000..37d7d40 --- /dev/null +++ b/0046-oom-monitor-in-manual-cases.patch @@ -0,0 +1,120 @@ +From aa77c85ea6879698663d4ef9e01bb63a0db1e57d Mon Sep 17 00:00:00 2001 +From: jikai +Date: Wed, 3 Apr 2024 09:34:39 +0000 +Subject: [PATCH 46/69] oom monitor in manual cases + +Signed-off-by: jikai +--- + CI/test_cases/container_cases/inspect.sh | 15 ------ + CI/test_cases/manual_cases/oom_monitor.sh | 59 +++++++++++++++++++++++ + 2 files changed, 59 insertions(+), 15 deletions(-) + create mode 100755 CI/test_cases/manual_cases/oom_monitor.sh + +diff --git a/CI/test_cases/container_cases/inspect.sh b/CI/test_cases/container_cases/inspect.sh +index 5d976281..cde9ea1f 100755 +--- a/CI/test_cases/container_cases/inspect.sh ++++ b/CI/test_cases/container_cases/inspect.sh +@@ -27,7 +27,6 @@ function test_inspect_spec() + { + local ret=0 + local image="busybox" +- local ubuntu_image="ubuntu" + local test="container inspect test => (${FUNCNAME[@]})" + + msg_info "${test} starting..." +@@ -38,12 +37,6 @@ function test_inspect_spec() + isula images | grep busybox + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++)) + +- isula pull ${ubuntu_image} +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${ubuntu_image}" && return ${FAILURE} +- +- isula images | grep ubuntu +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${ubuntu_image}" && ((ret++)) +- + containername=test_inspect + + isula create --name $containername --ipc host --pid host --uts host --restart=on-failure:10 --hook-spec ${test_data_path}/test-hookspec.json --cpu-shares 100 --memory 5MB --memory-reservation 4MB --cpu-period 1000000 --cpu-quota 200000 --cpuset-cpus 1 --cpuset-mems 0 --kernel-memory 50M --pids-limit=10000 --volume /home:/root --env a=1 $image /bin/sh ls +@@ -146,14 +139,6 @@ function test_inspect_spec() + + isula rm -f $containername + +- # use more than 10m memory limit, otherwise it might fail to run +- isula run -it -m 10m --runtime runc --name $containername $ubuntu_image perl -e 'for ($i = 0; $i < 100000000; $i++) { $a .= " " x 1024 }' +- +- isula inspect -f "{{json .State.OOMKilled}} {{.Name}}" $containername 2>&1 | sed -n '1p' | grep "true" +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container with image: ${ubuntu_image}" && ((ret++)) +- +- isula rm -f $containername +- + msg_info "${test} finished with return ${ret}..." + return ${ret} + } +diff --git a/CI/test_cases/manual_cases/oom_monitor.sh b/CI/test_cases/manual_cases/oom_monitor.sh +new file mode 100755 +index 00000000..a1c2503d +--- /dev/null ++++ b/CI/test_cases/manual_cases/oom_monitor.sh +@@ -0,0 +1,59 @@ ++#!/bin/bash ++# ++# attributes: isulad oom monitor ++# concurrent: NA ++# spend time: 6 ++ ++####################################################################### ++##- Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. ++# - iSulad licensed under the Mulan PSL v2. ++# - You can use this software according to the terms and conditions of the Mulan PSL v2. ++# - You may obtain a copy of Mulan PSL v2 at: ++# - http://license.coscl.org.cn/MulanPSL2 ++# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++# - PURPOSE. ++# - See the Mulan PSL v2 for more details. ++##- @Description:CI ++##- @Author: jikai ++##- @Create: 2024-04-03 ++####################################################################### ++ ++declare -r curr_path=$(dirname $(readlink -f "$0")) ++source ../helpers.sh ++test_data_path=$(realpath $curr_path/test_data) ++ ++function test_oom_monitor() ++{ ++ local ret=0 ++ local ubuntu_image="ubuntu" ++ local test="container oom monitor test => (${FUNCNAME[@]})" ++ containername="oommonitor" ++ ++ msg_info "${test} starting..." ++ ++ isula pull ${ubuntu_image} ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${ubuntu_image}" && return ${FAILURE} ++ ++ isula images | grep ubuntu ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${ubuntu_image}" && ((ret++)) ++ ++ # use more than 10m memory limit, otherwise it might fail to run ++ # iSulad monitor cgroup file for oom event, however oom triggers cgroup files delete ++ # if cgroup files were deleted before oom event was handled for iSulad we might failed to detect oom event ++ isula run -it -m 10m --runtime runc --name $containername $ubuntu_image perl -e 'for ($i = 0; $i < 100000000; $i++) { $a .= " " x 1024 }' ++ ++ isula inspect -f "{{json .State.OOMKilled}} {{.Name}}" $containername 2>&1 | sed -n '1p' | grep "true" ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container with image: ${ubuntu_image}" && ((ret++)) ++ ++ isula rm -f $containername ++ ++ msg_info "${test} finished with return ${ret}..." ++ return ${ret} ++} ++ ++declare -i ans=0 ++ ++test_oom_monitor || ((ans++)) ++ ++show_result ${ans} "${curr_path}/${0}" +-- +2.34.1 + diff --git a/0047-add-usage-restrictions-for-CRI-1.29-update.patch b/0047-add-usage-restrictions-for-CRI-1.29-update.patch new file mode 100644 index 0000000..e6990b7 --- /dev/null +++ b/0047-add-usage-restrictions-for-CRI-1.29-update.patch @@ -0,0 +1,27 @@ +From 9f066a405a95299c182ef7356b6518a9457af298 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Mon, 8 Apr 2024 02:52:11 +0000 +Subject: [PATCH 47/69] add usage restrictions for CRI 1.29 update + +Signed-off-by: jikai +--- + docs/design/detailed/CRI/CRI_1.29_update_design.md | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/docs/design/detailed/CRI/CRI_1.29_update_design.md b/docs/design/detailed/CRI/CRI_1.29_update_design.md +index 0a0c860b..0c66db48 100644 +--- a/docs/design/detailed/CRI/CRI_1.29_update_design.md ++++ b/docs/design/detailed/CRI/CRI_1.29_update_design.md +@@ -234,4 +234,8 @@ enum ContainerEventType { + + ### 使用限制 + +-以上特性仅保证容器运行时设置为runc时支持。 ++1. 以上新增特性，iSulad仅提供容器运行时设置为runc时的支持。 ++2. 由于cgroup oom会同时触发容器cgroup路径删除，若iSulad对oom事件处理发生在 ++cgroup路径删除之后，iSulad则无法成功捕捉容器oom事件， ++可能导致ContainerStatus中reason字段设置不正确。 ++3. iSulad不支持交叉使用不同的cgroup驱动管理容器，启动容器后iSulad的cgroup驱动配置不应该发生变化。 +-- +2.34.1 + diff --git a/0048-CDI-interface-definition.patch b/0048-CDI-interface-definition.patch new file mode 100644 index 0000000..e84db63 --- /dev/null +++ b/0048-CDI-interface-definition.patch @@ -0,0 +1,1192 @@ +From 040e39827049cd557987f505d1bd6c6e4f18b4b3 Mon Sep 17 00:00:00 2001 +From: liuxu +Date: Mon, 11 Mar 2024 17:48:11 +0800 +Subject: [PATCH 48/69] CDI:interface definition + +--- + cmake/options.cmake | 11 +++ + src/daemon/modules/CMakeLists.txt | 11 +++ + src/daemon/modules/api/cdi_operate_api.h | 39 ++++++++++ + src/daemon/modules/device/CMakeLists.txt | 23 ++++++ + src/daemon/modules/device/cdi/CMakeLists.txt | 13 ++++ + .../device/cdi/behavior/CMakeLists.txt | 13 ++++ + .../device/cdi/behavior/cdi_container_edits.c | 49 ++++++++++++ + .../device/cdi/behavior/cdi_container_edits.h | 39 ++++++++++ + .../modules/device/cdi/behavior/cdi_device.c | 40 ++++++++++ + .../modules/device/cdi/behavior/cdi_device.h | 46 ++++++++++++ + .../modules/device/cdi/behavior/cdi_spec.c | 60 +++++++++++++++ + .../modules/device/cdi/behavior/cdi_spec.h | 56 ++++++++++++++ + .../device/cdi/behavior/cdi_spec_dirs.c | 29 +++++++ + .../device/cdi/behavior/cdi_spec_dirs.h | 47 ++++++++++++ + .../modules/device/cdi/behavior/cdi_version.c | 40 ++++++++++ + .../modules/device/cdi/behavior/cdi_version.h | 34 +++++++++ + .../device/cdi/behavior/parser/CMakeLists.txt | 3 + + .../device/cdi/behavior/parser/cdi_parser.c | 55 ++++++++++++++ + .../device/cdi/behavior/parser/cdi_parser.h | 38 ++++++++++ + .../modules/device/cdi/cdi_annotations.c | 31 ++++++++ + .../modules/device/cdi/cdi_annotations.h | 32 ++++++++ + src/daemon/modules/device/cdi/cdi_cache.c | 69 +++++++++++++++++ + src/daemon/modules/device/cdi/cdi_cache.h | 75 +++++++++++++++++++ + src/daemon/modules/device/cdi/cdi_registry.c | 25 +++++++ + src/daemon/modules/device/cdi/cdi_registry.h | 36 +++++++++ + src/daemon/modules/device/cdi_operate.c | 35 +++++++++ + 26 files changed, 949 insertions(+) + create mode 100644 src/daemon/modules/api/cdi_operate_api.h + create mode 100644 src/daemon/modules/device/CMakeLists.txt + create mode 100644 src/daemon/modules/device/cdi/CMakeLists.txt + create mode 100644 src/daemon/modules/device/cdi/behavior/CMakeLists.txt + create mode 100644 src/daemon/modules/device/cdi/behavior/cdi_container_edits.c + create mode 100644 src/daemon/modules/device/cdi/behavior/cdi_container_edits.h + create mode 100644 src/daemon/modules/device/cdi/behavior/cdi_device.c + create mode 100644 src/daemon/modules/device/cdi/behavior/cdi_device.h + create mode 100644 src/daemon/modules/device/cdi/behavior/cdi_spec.c + create mode 100644 src/daemon/modules/device/cdi/behavior/cdi_spec.h + create mode 100644 src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c + create mode 100644 src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h + create mode 100644 src/daemon/modules/device/cdi/behavior/cdi_version.c + create mode 100644 src/daemon/modules/device/cdi/behavior/cdi_version.h + create mode 100644 src/daemon/modules/device/cdi/behavior/parser/CMakeLists.txt + create mode 100644 src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c + create mode 100644 src/daemon/modules/device/cdi/behavior/parser/cdi_parser.h + create mode 100644 src/daemon/modules/device/cdi/cdi_annotations.c + create mode 100644 src/daemon/modules/device/cdi/cdi_annotations.h + create mode 100644 src/daemon/modules/device/cdi/cdi_cache.c + create mode 100644 src/daemon/modules/device/cdi/cdi_cache.h + create mode 100644 src/daemon/modules/device/cdi/cdi_registry.c + create mode 100644 src/daemon/modules/device/cdi/cdi_registry.h + create mode 100644 src/daemon/modules/device/cdi_operate.c + +diff --git a/cmake/options.cmake b/cmake/options.cmake +index b7a7c65d..c1eac472 100644 +--- a/cmake/options.cmake ++++ b/cmake/options.cmake +@@ -40,6 +40,17 @@ if (ENABLE_CRI_API_V1 STREQUAL "ON") + message("${Green}-- Enable CRI API V1${ColourReset}") + endif() + ++option(ENABLE_CDI "Enable CDI" OFF) ++if (ENABLE_CDI STREQUAL "ON") ++ if (ENABLE_CRI_API_V1) ++ add_definitions(-DENABLE_CDI) ++ set(ENABLE_CDI 1) ++ message("${Green}-- Enable CDI${ColourReset}") ++ else() ++ message("${Yellow}-- Can not enable CDI, CDI need enable CRI API V1 first ${ColourReset}") ++ endif() ++endif() ++ + option(ENABLE_SANDBOXER "Enable sandbox API" ON) + if (ENABLE_SANDBOXER STREQUAL "ON") + add_definitions(-DENABLE_SANDBOXER) +diff --git a/src/daemon/modules/CMakeLists.txt b/src/daemon/modules/CMakeLists.txt +index a70c094f..ff2ea226 100644 +--- a/src/daemon/modules/CMakeLists.txt ++++ b/src/daemon/modules/CMakeLists.txt +@@ -49,6 +49,16 @@ if (ENABLE_PLUGIN) + ) + endif() + ++if (ENABLE_CDI) ++ add_subdirectory(device) ++ list(APPEND local_modules_srcs ++ ${DEVICE_SRCS} ++ ) ++ list(APPEND local_modules_incs ++ ${DEVICE_INCS} ++ ) ++endif() ++ + set(MODULES_SRCS + ${local_modules_srcs} + PARENT_SCOPE +@@ -70,3 +80,4 @@ if(ENABLE_NETWORK) + ) + endif() + ++ +diff --git a/src/daemon/modules/api/cdi_operate_api.h b/src/daemon/modules/api/cdi_operate_api.h +new file mode 100644 +index 00000000..4f4c339e +--- /dev/null ++++ b/src/daemon/modules/api/cdi_operate_api.h +@@ -0,0 +1,39 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi device manager function definition ++ ******************************************************************************/ ++#ifndef CDI_OPERATE_API_H ++#define CDI_OPERATE_API_H ++ ++#include ++#include ++ ++#include "utils_array.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++int cdi_operate_registry_init(char **specs_dirs, size_t specs_dirs_len); ++ ++char *cdi_operate_refresh(void); ++ ++string_array *cdi_operate_inject_devices(oci_runtime_spec *spec, string_array *devices, char **error); ++ ++char *cdi_operate_parse_annotations(json_map_string_string *annotations, string_array **keys, string_array **devices); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +diff --git a/src/daemon/modules/device/CMakeLists.txt b/src/daemon/modules/device/CMakeLists.txt +new file mode 100644 +index 00000000..e3468c0b +--- /dev/null ++++ b/src/daemon/modules/device/CMakeLists.txt +@@ -0,0 +1,23 @@ ++# get current directory sources files ++aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} device_top_srcs) ++ ++add_subdirectory(cdi) ++ ++set(local_device_srcs ++ ${device_top_srcs} ++ ${CDI_SRCS} ++ ) ++ ++set(local_device_incs ++ ${CMAKE_CURRENT_SOURCE_DIR} ++ ${CDI_INCS} ++ ) ++ ++set(DEVICE_SRCS ++ ${local_device_srcs} ++ PARENT_SCOPE ++ ) ++set(DEVICE_INCS ++ ${local_device_incs} ++ PARENT_SCOPE ++ ) +\ No newline at end of file +diff --git a/src/daemon/modules/device/cdi/CMakeLists.txt b/src/daemon/modules/device/cdi/CMakeLists.txt +new file mode 100644 +index 00000000..12d9b222 +--- /dev/null ++++ b/src/daemon/modules/device/cdi/CMakeLists.txt +@@ -0,0 +1,13 @@ ++aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} cdi_top_srcs) ++add_subdirectory(behavior) ++ ++set(CDI_SRCS ++ ${cdi_top_srcs} ++ ${BEHAVIOR_SRCS} ++ PARENT_SCOPE ++ ) ++set(CDI_INCS ++ ${CMAKE_CURRENT_SOURCE_DIR} ++ ${BEHAVIOR_INCS} ++ PARENT_SCOPE ++ ) +diff --git a/src/daemon/modules/device/cdi/behavior/CMakeLists.txt b/src/daemon/modules/device/cdi/behavior/CMakeLists.txt +new file mode 100644 +index 00000000..39881a7b +--- /dev/null ++++ b/src/daemon/modules/device/cdi/behavior/CMakeLists.txt +@@ -0,0 +1,13 @@ ++aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} behavior_top_srcs) ++add_subdirectory(parser) ++ ++set(BEHAVIOR_SRCS ++ ${behavior_top_srcs} ++ ${PARSER_SRCS} ++ PARENT_SCOPE ++ ) ++set(BEHAVIOR_INCS ++ ${CMAKE_CURRENT_SOURCE_DIR} ++ ${CMAKE_CURRENT_SOURCE_DIR}/parser/ ++ PARENT_SCOPE ++ ) +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c +new file mode 100644 +index 00000000..ce7b16db +--- /dev/null ++++ b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c +@@ -0,0 +1,49 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi container edits function ++ ******************************************************************************/ ++#include "cdi_container_edits.h" ++ ++// PRESTART_HOOK is the name of the OCI "prestart" hook. ++#define PRESTART_HOOK "prestart" ++// CREATE_RUNTIME_HOOK is the name of the OCI "createRuntime" hook. ++#define CREATE_RUNTIME_HOOK "createRuntime" ++// CREATE_CONTAINER_HOOK is the name of the OCI "createContainer" hook. ++#define CREATE_CONTAINER_HOOK "createContainer" ++// START_CONTAINER_HOOK is the name of the OCI "startContainer" hook. ++#define START_CONTAINER_HOOK "startContainer" ++// POSTSTART_HOOK is the name of the OCI "poststart" hook. ++#define POSTSTART_HOOK "poststart" ++// POSTSTOP_HOOK is the name of the OCI "poststop" hook. ++#define POSTSTOP_HOOK "poststop" ++ ++char *cdi_container_edits_apply(cdi_container_edits *e, oci_runtime_spec *spec) ++{ ++ return NULL; ++} ++ ++char *cdi_container_edits_validate(cdi_container_edits *e) ++{ ++ return NULL; ++} ++ ++cdi_container_edits *cdi_container_edits_append(cdi_container_edits *e, cdi_container_edits *o) ++{ ++ return NULL; ++} ++ ++bool cdi_container_edits_is_empty(cdi_container_edits *e) ++{ ++ return true; ++} ++ +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_container_edits.h b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.h +new file mode 100644 +index 00000000..7b16d2bc +--- /dev/null ++++ b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.h +@@ -0,0 +1,39 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi container edits function definition ++ ******************************************************************************/ ++#ifndef CDI_CONTAINER_EDITS_H ++#define CDI_CONTAINER_EDITS_H ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include "utils_array.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++char *cdi_container_edits_apply(cdi_container_edits *e, oci_runtime_spec *spec); ++char *cdi_container_edits_validate(cdi_container_edits *e); ++cdi_container_edits *cdi_container_edits_append(cdi_container_edits *e, cdi_container_edits *o); ++bool cdi_container_edits_is_empty(cdi_container_edits *e); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +\ No newline at end of file +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_device.c b/src/daemon/modules/device/cdi/behavior/cdi_device.c +new file mode 100644 +index 00000000..9904e9ee +--- /dev/null ++++ b/src/daemon/modules/device/cdi/behavior/cdi_device.c +@@ -0,0 +1,40 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi device function ++ ******************************************************************************/ ++#include "cdi_device.h" ++ ++void free_cdi_cache_device(struct cdi_cache_device *d) ++{ ++ (void)d; ++} ++ ++struct cdi_cache_device *cdi_device_new_device(struct cdi_cache_spec *spec, cdi_device *d, char **error) ++{ ++ return NULL; ++} ++ ++struct cdi_cache_spec *cdi_device_get_spec(struct cdi_cache_device *d) ++{ ++ return NULL; ++} ++ ++char *cdi_device_get_qualified_name(struct cdi_cache_device *d) ++{ ++ return NULL; ++} ++ ++cdi_container_edits *cdi_device_edits(struct cdi_cache_device *d) ++{ ++ return NULL; ++} +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_device.h b/src/daemon/modules/device/cdi/behavior/cdi_device.h +new file mode 100644 +index 00000000..3f460152 +--- /dev/null ++++ b/src/daemon/modules/device/cdi/behavior/cdi_device.h +@@ -0,0 +1,46 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi device function definition ++ ******************************************************************************/ ++#ifndef CDI_DEVICE_H ++#define CDI_DEVICE_H ++ ++#include ++#include ++#include ++ ++#include "cdi_container_edits.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++struct cdi_cache_spec; ++ ++struct cdi_cache_device { ++ cdi_device *raw_device; ++ struct cdi_cache_spec *cache_spec; ++}; ++ ++void free_cdi_cache_device(struct cdi_cache_device *d); ++ ++struct cdi_cache_device *cdi_device_new_device(struct cdi_cache_spec *spec, cdi_device *d, char **error); ++struct cdi_cache_spec *cdi_device_get_spec(struct cdi_cache_device *d); ++char *cdi_device_get_qualified_name(struct cdi_cache_device *d); ++cdi_container_edits *cdi_device_edits(struct cdi_cache_device *d); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +\ No newline at end of file +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec.c b/src/daemon/modules/device/cdi/behavior/cdi_spec.c +new file mode 100644 +index 00000000..38fc9e38 +--- /dev/null ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec.c +@@ -0,0 +1,60 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi spec function ++ ******************************************************************************/ ++#include "cdi_spec.h" ++ ++void free_cdi_cache_spec(struct cdi_cache_spec *s) ++{ ++ (void)s; ++} ++ ++struct cdi_cache_spec *cdi_spec_read_spec(const char *path, int priority, char **error) ++{ ++ return NULL; ++} ++ ++struct cdi_cache_spec *cdi_spec_new_spec(cdi_spec *raw, const char *path, int priority, char **error) ++{ ++ return NULL; ++} ++ ++const char *cdi_spec_get_vendor(struct cdi_cache_spec *s) ++{ ++ return NULL; ++} ++ ++const char *cdi_spec_get_class(struct cdi_cache_spec *s) ++{ ++ return NULL; ++} ++ ++struct cdi_cache_device *cdi_spec_get_cache_device(struct cdi_cache_spec *s, const char *name) ++{ ++ return NULL; ++} ++ ++const char *cdi_spec_get_path(struct cdi_cache_spec *s) ++{ ++ return NULL; ++} ++ ++int cdi_spec_get_priority(struct cdi_cache_spec *s) ++{ ++ return 0; ++} ++ ++cdi_container_edits *cdi_spec_edits(struct cdi_cache_spec *s) ++{ ++ return NULL; ++} +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec.h b/src/daemon/modules/device/cdi/behavior/cdi_spec.h +new file mode 100644 +index 00000000..bd4fc9d1 +--- /dev/null ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec.h +@@ -0,0 +1,56 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi spec function definition ++ ******************************************************************************/ ++#ifndef CDI_SPEC_H ++#define CDI_SPEC_H ++ ++#include ++#include ++ ++#include "map.h" ++#include "cdi_container_edits.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++struct cdi_cache_device; ++ ++struct cdi_cache_spec { ++ cdi_spec *raw_spec; ++ char *vendor; ++ char *class; ++ char *path; ++ int priority; ++ map_t *devices; // MAP_STR_PTR devices[cdi_device.name] = cdi_cache_device* ++}; ++ ++#define CDI_DEFAULT_SPEC_EXT ".json" ++ ++void free_cdi_cache_spec(struct cdi_cache_spec *s); ++ ++struct cdi_cache_spec *cdi_spec_read_spec(const char *path, int priority, char **error); ++struct cdi_cache_spec *cdi_spec_new_spec(cdi_spec *raw, const char *path, int priority, char **error); ++const char *cdi_spec_get_vendor(struct cdi_cache_spec *s); ++const char *cdi_spec_get_class(struct cdi_cache_spec *s); ++struct cdi_cache_device *cdi_spec_get_cache_device(struct cdi_cache_spec *s, const char *name); ++const char *cdi_spec_get_path(struct cdi_cache_spec *s); ++int cdi_spec_get_priority(struct cdi_cache_spec *s); ++cdi_container_edits *cdi_spec_edits(struct cdi_cache_spec *s); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +\ No newline at end of file +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c +new file mode 100644 +index 00000000..5df4c937 +--- /dev/null ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c +@@ -0,0 +1,29 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi spec dirs function ++ ******************************************************************************/ ++#include "cdi_spec_dirs.h" ++ ++#define DEFAULT_SPEC_DIRS_LEN 2 ++static char *default_spec_dirs_items[DEFAULT_SPEC_DIRS_LEN] = {CDI_DEFAULT_STATIC_DIR, CDI_DEFAULT_DYNAMIC_DIR}; ++ ++string_array g_default_spec_dirs = { ++ .items = default_spec_dirs_items, ++ .len = DEFAULT_SPEC_DIRS_LEN, ++ .cap = DEFAULT_SPEC_DIRS_LEN, ++}; ++ ++char *cdi_scan_spec_dirs(string_array *dirs, struct cdi_scan_fn_maps *scan_fn_maps, cdi_scan_spec_func scan_fn) ++{ ++ return NULL; ++} +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h +new file mode 100644 +index 00000000..bd00e318 +--- /dev/null ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h +@@ -0,0 +1,47 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi spec dirs function definition ++ ******************************************************************************/ ++#ifndef CDI_SPEC_DIRS_H ++#define CDI_SPEC_DIRS_H ++ ++#include "cdi_cache.h" ++#include "utils_array.h" ++#include "map.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++#define CDI_DEFAULT_STATIC_DIR "/etc/cdi" ++#define CDI_DEFAULT_DYNAMIC_DIR "/var/run/cdi" ++ ++extern string_array g_default_spec_dirs; ++ ++struct cdi_scan_fn_maps { ++ map_t *specs; ++ map_t *devices; ++ map_t *conflicts; ++ map_t *spec_errors; ++ string_array *result; ++}; ++typedef char *(*cdi_scan_spec_func)(struct cdi_scan_fn_maps *scan_fn_maps, const char *path, int priority, ++ struct cdi_cache_spec *spec, char **error); ++ ++char *cdi_scan_spec_dirs(string_array *dirs, struct cdi_scan_fn_maps *scan_fn_maps, cdi_scan_spec_func scan_fn); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +\ No newline at end of file +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_version.c b/src/daemon/modules/device/cdi/behavior/cdi_version.c +new file mode 100644 +index 00000000..3e87c111 +--- /dev/null ++++ b/src/daemon/modules/device/cdi/behavior/cdi_version.c +@@ -0,0 +1,40 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi version function ++ ******************************************************************************/ ++#include "cdi_version.h" ++ ++#define CDI_V_CURRENT_VERSION "v"##CDI_CURRENT_VERSION ++ ++#define CDI_V010 "v0.1.0" ++#define CDI_V020 "v0.2.0" ++#define CDI_V030 "v0.3.0" ++#define CDI_V040 "v0.4.0" ++#define CDI_V050 "v0.5.0" ++#define CDI_V060 "v0.6.0" ++#define CDI_V_EARLIEST CDI_V030 ++ ++const char *cdi_minimum_required_version(cdi_spec *spec) ++{ ++ return NULL; ++} ++ ++bool cdi_is_greater_than_version(const char *v, const char *o) ++{ ++ return true; ++} ++ ++bool cdi_is_valid_version(const char *spec_version) ++{ ++ return true; ++} +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_version.h b/src/daemon/modules/device/cdi/behavior/cdi_version.h +new file mode 100644 +index 00000000..99b7e692 +--- /dev/null ++++ b/src/daemon/modules/device/cdi/behavior/cdi_version.h +@@ -0,0 +1,34 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi version function definition ++ ******************************************************************************/ ++#ifndef CDI_VERSION_H ++#define CDI_VERSION_H ++ ++#include ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++#define CDI_CURRENT_VERSION "0.6.0" ++ ++const char *cdi_minimum_required_version(cdi_spec *spec); ++bool cdi_is_greater_than_version(const char *v, const char *o); ++bool cdi_is_valid_version(const char *spec_version); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +\ No newline at end of file +diff --git a/src/daemon/modules/device/cdi/behavior/parser/CMakeLists.txt b/src/daemon/modules/device/cdi/behavior/parser/CMakeLists.txt +new file mode 100644 +index 00000000..9bf5f3f2 +--- /dev/null ++++ b/src/daemon/modules/device/cdi/behavior/parser/CMakeLists.txt +@@ -0,0 +1,3 @@ ++# get current directory sources files ++aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_cdi_parser_srcs) ++set(PARSER_SRCS ${local_cdi_parser_srcs} PARENT_SCOPE) +diff --git a/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c +new file mode 100644 +index 00000000..45048f9a +--- /dev/null ++++ b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c +@@ -0,0 +1,55 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi parser linux function ++ ******************************************************************************/ ++#include "cdi_parser.h" ++ ++char *cdi_parser_qualified_name(const char *vendor, const char *class, const char *name) ++{ ++ return NULL; ++} ++ ++bool cdi_parser_is_qualified_name(const char *device) ++{ ++ return true; ++} ++ ++char *cdi_parser_parse_qualified_name(const char *device, char **vendor, char **class, char **name) ++{ ++ return NULL; ++} ++ ++int cdi_parser_parse_device(const char *device, char **vendor, char **class, char **name) ++{ ++ return 0; ++} ++ ++int cdi_parser_parse_qualifier(const char *kind, char **vendor, char **class) ++{ ++ return 0; ++} ++ ++char *cdi_parser_validate_vendor_name(const char *vendor) ++{ ++ return NULL; ++} ++ ++char *cdi_parser_validate_class_name(const char *class) ++{ ++ return NULL; ++} ++ ++char *cdi_parser_validate_device_name(const char *name) ++{ ++ return NULL; ++} +diff --git a/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.h b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.h +new file mode 100644 +index 00000000..d9c057ea +--- /dev/null ++++ b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.h +@@ -0,0 +1,38 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi parser function definition ++ ******************************************************************************/ ++#ifndef CDI_PARSER_H ++#define CDI_PARSER_H ++ ++#include ++#include ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++char *cdi_parser_qualified_name(const char *vendor, const char *class, const char *name); ++bool cdi_parser_is_qualified_name(const char *device); ++char *cdi_parser_parse_qualified_name(const char *device, char **vendor, char **class, char **name); ++int cdi_parser_parse_device(const char *device, char **vendor, char **class, char **name); ++int cdi_parser_parse_qualifier(const char *kind, char **vendor, char **class); ++char *cdi_parser_validate_vendor_name(const char *vendor); ++char *cdi_parser_validate_class_name(const char *class); ++char *cdi_parser_validate_device_name(const char *name); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +\ No newline at end of file +diff --git a/src/daemon/modules/device/cdi/cdi_annotations.c b/src/daemon/modules/device/cdi/cdi_annotations.c +new file mode 100644 +index 00000000..3cb9be84 +--- /dev/null ++++ b/src/daemon/modules/device/cdi/cdi_annotations.c +@@ -0,0 +1,31 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi annotations function ++ ******************************************************************************/ ++#include "cdi_annotations.h" ++ ++#include ++#include ++#include ++#include ++ ++#include "error.h" ++#include "utils.h" ++#include "cdi_parser.h" ++ ++#define CDI_ANNOTATIONS_PREFIX "cdi.k8s.io/" ++ ++char *cdi_parse_annotations(json_map_string_string *annotations, string_array **keys, string_array **devices) ++{ ++ return NULL; ++} +diff --git a/src/daemon/modules/device/cdi/cdi_annotations.h b/src/daemon/modules/device/cdi/cdi_annotations.h +new file mode 100644 +index 00000000..52355099 +--- /dev/null ++++ b/src/daemon/modules/device/cdi/cdi_annotations.h +@@ -0,0 +1,32 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi annotations function definition ++ ******************************************************************************/ ++#ifndef CDI_ANNOTATIONS_H ++#define CDI_ANNOTATIONS_H ++ ++#include ++ ++#include "utils_array.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++char *cdi_parse_annotations(json_map_string_string *annotations, string_array **keys, string_array **devices); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +\ No newline at end of file +diff --git a/src/daemon/modules/device/cdi/cdi_cache.c b/src/daemon/modules/device/cdi/cdi_cache.c +new file mode 100644 +index 00000000..9c54acbf +--- /dev/null ++++ b/src/daemon/modules/device/cdi/cdi_cache.c +@@ -0,0 +1,69 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi cache function ++ ******************************************************************************/ ++#include "cdi_cache.h" ++ ++void free_cdi_cache(struct cdi_cache *c) ++{ ++ (void)c; ++} ++ ++struct cdi_cache *cdi_new_cache(string_array *spec_dirs, char **error) ++{ ++ return NULL; ++} ++ ++static string_array *cdi_inject_devices(struct cdi_cache *c, oci_runtime_spec *oci_spec, string_array *devices, char **error) ++{ ++ return NULL; ++} ++ ++static char *cdi_configure(struct cdi_cache *c, string_array *spec_dirs) ++{ ++ return NULL; ++} ++ ++static char *cdi_refresh(struct cdi_cache *c) ++{ ++ return NULL; ++} ++ ++static map_t *cdi_get_errors(struct cdi_cache *c) ++{ ++ return NULL; ++} ++ ++static string_array *cdi_get_spec_directories(struct cdi_cache *c) ++{ ++ return NULL; ++} ++ ++static map_t *cdi_get_spec_dir_errors(struct cdi_cache *c) ++{ ++ return NULL; ++} ++ ++static struct cdi_cache_ops g_cdi_cache_ops = { ++ .inject_devices = cdi_inject_devices, ++ .configure = cdi_configure, ++ .refresh = cdi_refresh, ++ .get_errors = cdi_get_errors, ++ .get_spec_directories = cdi_get_spec_directories, ++ .get_spec_dir_errors = cdi_get_spec_dir_errors ++}; ++ ++struct cdi_cache_ops *cdi_get_cache_ops(void) ++{ ++ return &g_cdi_cache_ops; ++} +\ No newline at end of file +diff --git a/src/daemon/modules/device/cdi/cdi_cache.h b/src/daemon/modules/device/cdi/cdi_cache.h +new file mode 100644 +index 00000000..92fb64af +--- /dev/null ++++ b/src/daemon/modules/device/cdi/cdi_cache.h +@@ -0,0 +1,75 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi cache function definition ++ ******************************************************************************/ ++#ifndef CDI_CACHE_H ++#define CDI_CACHE_H ++ ++#include ++#include ++#include ++ ++#include "utils_array.h" ++#include "map.h" ++#include "cdi_device.h" ++#include "cdi_spec.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++struct cdi_cache; ++ ++struct cdi_cache_ops { ++ // injecting CDI devices into an OCI Spec. ++ // Resolver ++ string_array *(*inject_devices)(struct cdi_cache *c, oci_runtime_spec *spec, string_array *devices, char **error); ++ ++ // refreshing the cache of CDI Specs and devices. ++ // Refresher ++ char *(*configure)(struct cdi_cache *c, string_array *spec_dirs); ++ char *(*refresh)(struct cdi_cache *c); ++ map_t *(*get_errors)(struct cdi_cache *c); ++ string_array *(*get_spec_directories)(struct cdi_cache *c); ++ map_t *(*get_spec_dir_errors)(struct cdi_cache *c); ++}; ++ ++struct cdi_watch { ++ int watcher_fd; // inotify fd ++ map_t *tracked; // MAP_STR_BOOL tracked[spec_dirs[i]] = bool ++ map_t *wd_dirs; // MAP_INT_STR wd_dirs[wd] = spec_dirs[i] ++}; ++ ++// Cache stores CDI Specs loaded from Spec directories. ++struct cdi_cache { ++ pthread_mutex_t mutex; ++ string_array *spec_dirs; // cdi-spec-dirs will scan for CDI Spec files ++ map_t *specs; // MAP_STR_PTR specs[vendor] = cdi_cache_spec** ++ map_t *devices; // MAP_STR_PTR devices[cdi_device.name] = cdi_cache_device* ++ map_t *errors; // MAP_STR_PTR errors[cdi_cache_spec.path] = string_array *errors ++ map_t *dir_errors; // MAP_STR_STR dir_errors[spec_dirs[i]] = error ++ ++ bool auto_refresh; ++ struct cdi_watch *watch; ++}; ++ ++void free_cdi_cache(struct cdi_cache *c); ++ ++struct cdi_cache *cdi_new_cache(string_array *spec_dirs, char **error); ++struct cdi_cache_ops *cdi_get_cache_ops(void); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +\ No newline at end of file +diff --git a/src/daemon/modules/device/cdi/cdi_registry.c b/src/daemon/modules/device/cdi/cdi_registry.c +new file mode 100644 +index 00000000..68767a5f +--- /dev/null ++++ b/src/daemon/modules/device/cdi/cdi_registry.c +@@ -0,0 +1,25 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi registry function ++ ******************************************************************************/ ++#include "cdi_registry.h" ++ ++int cdi_registry_init(string_array *spec_dirs) ++{ ++ return 0; ++} ++ ++struct cdi_registry *cdi_get_registry(void) ++{ ++ return NULL; ++} +diff --git a/src/daemon/modules/device/cdi/cdi_registry.h b/src/daemon/modules/device/cdi/cdi_registry.h +new file mode 100644 +index 00000000..c27d37e3 +--- /dev/null ++++ b/src/daemon/modules/device/cdi/cdi_registry.h +@@ -0,0 +1,36 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi registry function definition ++ ******************************************************************************/ ++#ifndef CDI_REGISTRY_H ++#define CDI_REGISTRY_H ++ ++#include "cdi_cache.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++struct cdi_registry { ++ struct cdi_cache *cdi_cache; ++ struct cdi_cache_ops *ops; ++}; ++ ++int cdi_registry_init(string_array *spec_dirs); ++struct cdi_registry *cdi_get_registry(void); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +\ No newline at end of file +diff --git a/src/daemon/modules/device/cdi_operate.c b/src/daemon/modules/device/cdi_operate.c +new file mode 100644 +index 00000000..c7aa77d8 +--- /dev/null ++++ b/src/daemon/modules/device/cdi_operate.c +@@ -0,0 +1,35 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-03-06 ++ * Description: provide cdi device manager function ++ ******************************************************************************/ ++#include "cdi_operate_api.h" ++ ++int cdi_operate_registry_init(char **specs_dirs, size_t specs_dirs_len) ++{ ++ return 0; ++} ++ ++char *cdi_operate_refresh(void) ++{ ++ return NULL; ++} ++ ++string_array *cdi_operate_inject_devices(oci_runtime_spec *spec, string_array *devices, char **error) ++{ ++ return NULL; ++} ++ ++char *cdi_operate_parse_annotations(json_map_string_string *annotations, string_array **keys, string_array **devices) ++{ ++ return NULL; ++} +\ No newline at end of file +-- +2.34.1 + diff --git a/0049-distinguish-between-runtime-and-runtime_cmd-in-isula.patch b/0049-distinguish-between-runtime-and-runtime_cmd-in-isula.patch new file mode 100644 index 0000000..4366d7a --- /dev/null +++ b/0049-distinguish-between-runtime-and-runtime_cmd-in-isula.patch @@ -0,0 +1,145 @@ +From 491baece02522128720b3bd992a76dc5148aa7b2 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Mon, 8 Apr 2024 11:37:13 +0800 +Subject: [PATCH 49/69] distinguish between runtime and runtime_cmd in + isulad-shim + +Signed-off-by: zhongtao +--- + src/cmd/isulad-shim/process.c | 20 +++++++++---------- + src/cmd/isulad-shim/process.h | 4 ++-- + .../modules/runtime/isula/isula_rt_ops.c | 2 ++ + 3 files changed, 14 insertions(+), 12 deletions(-) + +diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c +index 8a4ca175..6b5f8f7f 100644 +--- a/src/cmd/isulad-shim/process.c ++++ b/src/cmd/isulad-shim/process.c +@@ -1131,7 +1131,7 @@ static int init_root_path(process_t *p) + return SHIM_ERR; + } + +- if (buffer->nappend(buffer, PATH_MAX, "%s/%s", state_path, p->runtime) < 0) { ++ if (buffer->nappend(buffer, PATH_MAX, "%s/%s", state_path, p->state->runtime) < 0) { + ERROR("Failed to append state_path\n"); + isula_buffer_free(buffer); + return SHIM_ERR; +@@ -1146,7 +1146,7 @@ static int init_root_path(process_t *p) + return SHIM_OK; + } + +-process_t *new_process(char *id, char *bundle, char *runtime) ++process_t *new_process(char *id, char *bundle, char *runtime_cmd) + { + shim_client_process_state *p_state; + process_t *p = NULL; +@@ -1174,7 +1174,7 @@ process_t *new_process(char *id, char *bundle, char *runtime) + + p->id = id; + p->bundle = bundle; +- p->runtime = runtime; ++ p->runtime_cmd = runtime_cmd; + p->state = p_state; + p->console_sock_path = NULL; + p->exit_fd = -1; +@@ -1247,7 +1247,7 @@ static void set_common_params(process_t *p, const char *params[], int *index, co + { + int j; + +- params[(*index)++] = p->runtime; ++ params[(*index)++] = p->runtime_cmd; + for (j = 0; j < p->state->runtime_args_len; j++) { + params[(*index)++] = p->state->runtime_args[j]; + } +@@ -1261,7 +1261,7 @@ static void set_common_params(process_t *p, const char *params[], int *index, co + + // In addition to kata, other commonly used oci runtimes (runc, crun, youki, gvisor) + // need to set the --root option +- if (strcasecmp(p->runtime, "kata-runtime") != 0) { ++ if (strcasecmp(p->state->runtime, "kata-runtime") != 0) { + params[(*index)++] = "--root"; + params[(*index)++] = p->root_path; + } +@@ -1347,7 +1347,7 @@ static void process_kill_all(process_t *p) + params[i++] = p->id; + params[i++] = "SIGKILL"; + +- (void)cmd_combined_output(p->runtime, params, output, &output_len); ++ (void)cmd_combined_output(p->runtime_cmd, params, output, &output_len); + + return; + } +@@ -1375,7 +1375,7 @@ static void process_delete(process_t *p) + params[i++] = "--force"; + params[i++] = p->id; + +- (void)cmd_combined_output(p->runtime, params, output, &output_len); ++ (void)cmd_combined_output(p->runtime_cmd, params, output, &output_len); + + return; + } +@@ -1444,8 +1444,8 @@ static void exec_runtime_process(process_t *p, int exec_fd) + + const char *params[MAX_RUNTIME_ARGS] = { 0 }; + get_runtime_cmd(p, log_path, pid_path, process_desc, params); +- execvp(p->runtime, (char * const *)params); +- (void)dprintf(exec_fd, "run process: %s error: %s", p->runtime, strerror(errno)); ++ execvp(p->runtime_cmd, (char * const *)params); ++ (void)dprintf(exec_fd, "run process: %s error: %s", p->runtime_cmd, strerror(errno)); + _exit(EXIT_FAILURE); + } + +@@ -1586,7 +1586,7 @@ static int waitpid_with_timeout(int ctr_pid, int *status, const uint64_t timeou + static int wait_container_process_with_timeout(process_t *p, const uint64_t timeout, int *status) + { + // currently, kata runtime does not support setting timeout during exec +- if (strcasecmp(p->runtime, "kata-runtime") != 0 && timeout > 0) { ++ if (strcasecmp(p->state->runtime, "kata-runtime") != 0 && timeout > 0) { + return waitpid_with_timeout(p->ctr_pid, status, timeout); + } + +diff --git a/src/cmd/isulad-shim/process.h b/src/cmd/isulad-shim/process.h +index 32ba7366..05fd87b0 100644 +--- a/src/cmd/isulad-shim/process.h ++++ b/src/cmd/isulad-shim/process.h +@@ -44,7 +44,7 @@ typedef struct { + typedef struct process { + char *id; + char *bundle; +- char *runtime; ++ char *runtime_cmd; + char *console_sock_path; // pty socket path + char *workdir; + char *root_path; +@@ -70,7 +70,7 @@ typedef struct { + int status; + } process_exit_t; + +-process_t* new_process(char *id, char *bundle, char *runtime); ++process_t* new_process(char *id, char *bundle, char *runtime_cmd); + + int prepare_attach_socket(process_t *p); + +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index b9aba3e3..bc3c36c8 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -1154,6 +1154,7 @@ int rt_isula_create(const char *id, const char *runtime, const rt_create_params_ + p.isulad_stdin = (char *)params->stdin; + p.isulad_stdout = (char *)params->stdout; + p.isulad_stderr = (char *)params->stderr; ++ p.runtime = (char *)runtime; + p.runtime_args = (char **)runtime_args; + p.runtime_args_len = runtime_args_len; + p.attach_socket = attach_socket; +@@ -1409,6 +1410,7 @@ static int preparation_exec(const char *id, const char *runtime, const char *wor + p.isulad_stdout = (char *)params->console_fifos[1]; + p.isulad_stderr = (char *)params->console_fifos[2]; + p.resize_fifo = resize_fifo_dir; ++ p.runtime = (char *)runtime; + p.runtime_args = (char **)runtime_args; + p.runtime_args_len = runtime_args_len; + copy_process(&p, process); +-- +2.34.1 + diff --git a/0050-Use-user-defined-shm-for-CRI-request.patch b/0050-Use-user-defined-shm-for-CRI-request.patch new file mode 100644 index 0000000..2d61c9d --- /dev/null +++ b/0050-Use-user-defined-shm-for-CRI-request.patch @@ -0,0 +1,171 @@ +From 162123bdec0f45f7b2001b2b0b83705cc6b9b1b1 Mon Sep 17 00:00:00 2001 +From: xuxuepeng +Date: Mon, 8 Apr 2024 20:53:57 +0800 +Subject: [PATCH 50/69] Use user defined shm for CRI request + +Signed-off-by: xuxuepeng +--- + src/daemon/modules/spec/specs_mount.c | 128 +++++++++++++++++--------- + 1 file changed, 85 insertions(+), 43 deletions(-) + +diff --git a/src/daemon/modules/spec/specs_mount.c b/src/daemon/modules/spec/specs_mount.c +index 20bf5378..6903ae40 100644 +--- a/src/daemon/modules/spec/specs_mount.c ++++ b/src/daemon/modules/spec/specs_mount.c +@@ -2799,33 +2799,31 @@ out_free: + return ret; + } + +-#define SHM_MOUNT_POINT "/dev/shm" +-static int set_shm_path(host_config *host_spec, container_config_v2_common_config *v2_spec) ++static inline int set_sharable_ipc_mode(host_config *host_spec, container_config_v2_common_config *v2_spec) + { +- int ret = 0; +- container_t *cont = NULL; +- char *tmp_cid = NULL; +- char *right_path = NULL; +- +- // ignore shm of system container +- if (host_spec->system_container) { ++ free(v2_spec->shm_path); ++#ifdef ENABLE_CRI_API_V1 ++ // In the case of sandbox API is used, the shm path has already been created in CRI, ++ // so we need to use the sandbox's shm path ++ if (is_sandbox_container(v2_spec->sandbox_info)) { ++ v2_spec->shm_path = util_strdup_s(v2_spec->sandbox_info->shm_path); + return 0; + } +- // setup shareable dirs +- if (is_shareable_ipc(host_spec->ipc_mode)) { +- // has mount for /dev/shm +- if (has_mount_shm(host_spec, v2_spec)) { +- return 0; +- } ++#endif ++ v2_spec->shm_path = get_prepare_share_shm_path(host_spec->runtime, v2_spec->id); ++ if (v2_spec->shm_path == NULL) { ++ ERROR("Failed to get prepare share shm path"); ++ return -1; ++ } + +- v2_spec->shm_path = get_prepare_share_shm_path(host_spec->runtime, v2_spec->id); +- if (v2_spec->shm_path == NULL) { +- ERROR("Failed to get prepare share shm path"); +- return -1; +- } ++ return 0; ++} + +- return 0; +- } ++static inline int set_connected_container_shm_path(host_config *host_spec, container_config_v2_common_config *v2_spec) ++{ ++ container_t *cont = NULL; ++ char *tmp_cid = NULL; ++ char *right_path = NULL; + + #ifdef ENABLE_CRI_API_V1 + // Sandbox API is used and the connected container is actually a sandbox +@@ -2833,34 +2831,78 @@ static int set_shm_path(host_config *host_spec, container_config_v2_common_confi + if (namespace_is_sandbox(host_spec->ipc_mode, v2_spec->sandbox_info)) { + free(v2_spec->shm_path); + v2_spec->shm_path = util_strdup_s(v2_spec->sandbox_info->shm_path); +- goto out; ++ return 0; + } + #endif + +- if (namespace_is_container(host_spec->ipc_mode)) { +- tmp_cid = namespace_get_connected_container(host_spec->ipc_mode); +- cont = containers_store_get(tmp_cid); +- if (cont == NULL) { +- ERROR("Invalid share path: %s", host_spec->ipc_mode); +- ret = -1; +- goto out; +- } +- right_path = util_strdup_s(cont->common_config->shm_path); +- container_unref(cont); +- } else if (namespace_is_host(host_spec->ipc_mode)) { +- if (!util_file_exists(SHM_MOUNT_POINT)) { +- ERROR("/dev/shm is not mounted, but must be for --ipc=host"); +- ret = -1; +- goto out; +- } +- right_path = util_strdup_s(SHM_MOUNT_POINT); ++ tmp_cid = namespace_get_connected_container(host_spec->ipc_mode); ++ cont = containers_store_get(tmp_cid); ++ if (cont == NULL) { ++ ERROR("Invalid share path: %s", host_spec->ipc_mode); ++ return -1; + } ++ right_path = util_strdup_s(cont->common_config->shm_path); ++ container_unref(cont); + + free(v2_spec->shm_path); + v2_spec->shm_path = right_path; +-out: +- free(tmp_cid); +- return ret; ++ ++ return 0; ++} ++ ++#define SHM_MOUNT_POINT "/dev/shm" ++static inline int set_host_ipc_shm_path(container_config_v2_common_config *v2_spec) ++{ ++ if (!util_file_exists(SHM_MOUNT_POINT)) { ++ ERROR("/dev/shm is not mounted, but must be for --ipc=host"); ++ return -1; ++ } ++ free(v2_spec->shm_path); ++ v2_spec->shm_path = util_strdup_s(SHM_MOUNT_POINT); ++ return 0; ++} ++ ++/** ++ * There are 4 cases for setting shm path: ++ * 1. The user defined /dev/shm in mounts, which takes the first priority ++ * 2. If sharable is set in ipc mode (or by default ipc_mode is null), the container provides shm path, ++ * in the case of sandbox API is used, the sandbox module has already provided shm path ++ * 3. Use the connected container's shm path if ipc_mode is set to container:, ++ * if connected containerd is a sandbox, use the sandbox's shm path ++ * 4. Use /dev/shm if ipc_mode is set to host ++ */ ++static int set_shm_path(host_config *host_spec, container_config_v2_common_config *v2_spec) ++{ ++ // ignore shm of system container ++ if (host_spec->system_container) { ++ return 0; ++ } ++ ++ // case 1: Defined in mounts already ++ if (has_mount_shm(host_spec, v2_spec)) { ++ return 0; ++ } ++ ++ // case 2: Container has its own IPC namespace ++ if (is_shareable_ipc(host_spec->ipc_mode)) { ++ return set_sharable_ipc_mode(host_spec, v2_spec); ++ } ++ ++ // case 3: Connected container ++ if (namespace_is_container(host_spec->ipc_mode)) { ++ return set_connected_container_shm_path(host_spec, v2_spec); ++ } ++ ++ // case 4: Host IPC namespace ++ if (namespace_is_host(host_spec->ipc_mode)) { ++ return set_host_ipc_shm_path(v2_spec); ++ } ++ ++ // Otherwise, the case is unknown, nothing is set ++ free(v2_spec->shm_path); ++ v2_spec->shm_path = NULL; ++ ++ return 0; + } + + int destination_compare(const void *p1, const void *p2) +-- +2.34.1 + diff --git a/0051-Fix-memory-leak-in-set_connected_container_shm_path.patch b/0051-Fix-memory-leak-in-set_connected_container_shm_path.patch new file mode 100644 index 0000000..c31e092 --- /dev/null +++ b/0051-Fix-memory-leak-in-set_connected_container_shm_path.patch @@ -0,0 +1,26 @@ +From 1052a7b67a35b3504045729e1408d4ace8bf50ca Mon Sep 17 00:00:00 2001 +From: xuxuepeng +Date: Tue, 9 Apr 2024 06:35:03 +0800 +Subject: [PATCH 51/69] Fix memory leak in set_connected_container_shm_path + +Signed-off-by: xuxuepeng +--- + src/daemon/modules/spec/specs_mount.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/daemon/modules/spec/specs_mount.c b/src/daemon/modules/spec/specs_mount.c +index 6903ae40..50ee9a85 100644 +--- a/src/daemon/modules/spec/specs_mount.c ++++ b/src/daemon/modules/spec/specs_mount.c +@@ -2822,7 +2822,7 @@ static inline int set_sharable_ipc_mode(host_config *host_spec, container_config + static inline int set_connected_container_shm_path(host_config *host_spec, container_config_v2_common_config *v2_spec) + { + container_t *cont = NULL; +- char *tmp_cid = NULL; ++ __isula_auto_free char *tmp_cid = NULL; + char *right_path = NULL; + + #ifdef ENABLE_CRI_API_V1 +-- +2.34.1 + diff --git a/0052-init-enable_pod_events-as-false.patch b/0052-init-enable_pod_events-as-false.patch new file mode 100644 index 0000000..08cfa2f --- /dev/null +++ b/0052-init-enable_pod_events-as-false.patch @@ -0,0 +1,62 @@ +From cfbb9f5ea40b3b654d7b6f9ad861877e97ed24be Mon Sep 17 00:00:00 2001 +From: jikai +Date: Thu, 11 Apr 2024 02:04:47 +0000 +Subject: [PATCH 52/69] init enable_pod_events as false + +Signed-off-by: jikai +--- + src/daemon/entry/connect/grpc/cri/cri_service.cc | 3 +-- + src/daemon/entry/connect/grpc/cri/cri_service.h | 1 - + .../connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc | 2 +- + 3 files changed, 2 insertions(+), 4 deletions(-) + +diff --git a/src/daemon/entry/connect/grpc/cri/cri_service.cc b/src/daemon/entry/connect/grpc/cri/cri_service.cc +index d10a60b5..80bcfef0 100644 +--- a/src/daemon/entry/connect/grpc/cri/cri_service.cc ++++ b/src/daemon/entry/connect/grpc/cri/cri_service.cc +@@ -89,9 +89,8 @@ int CRIService::Init(const isulad_daemon_configs *config) + + #ifdef ENABLE_CRI_API_V1 + m_enableCRIV1 = config->enable_cri_v1; +- m_enablePodEvents = config->enable_pod_events; + if (m_enableCRIV1) { +- m_runtimeV1RuntimeService.Init(m_podSandboxImage, m_pluginManager, m_enablePodEvents, err); ++ m_runtimeV1RuntimeService.Init(m_podSandboxImage, m_pluginManager, config->enable_pod_events, err); + if (err.NotEmpty()) { + ERROR("Init CRI v1 runtime service failed: %s", err.GetCMessage()); + return -1; +diff --git a/src/daemon/entry/connect/grpc/cri/cri_service.h b/src/daemon/entry/connect/grpc/cri/cri_service.h +index 041c7c63..77b2eb72 100644 +--- a/src/daemon/entry/connect/grpc/cri/cri_service.h ++++ b/src/daemon/entry/connect/grpc/cri/cri_service.h +@@ -56,7 +56,6 @@ private: + std::string m_podSandboxImage; + std::shared_ptr m_pluginManager; + bool m_enableCRIV1; +- bool m_enablePodEvents; + }; + + } +diff --git a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc +index bc5ab591..e2591ce0 100644 +--- a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc ++++ b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc +@@ -62,6 +62,7 @@ void RuntimeV1RuntimeServiceImpl::Init(std::string &podSandboxImage, + return; + } + ++ m_enablePodEvents = false; + if (enablePodEvents) { + if (mailbox_register_topic_handler(MAILBOX_TOPIC_CRI_CONTAINER, cri_container_topic_handler, + this, cri_container_topic_release, true) != 0) { +@@ -72,7 +73,6 @@ void RuntimeV1RuntimeServiceImpl::Init(std::string &podSandboxImage, + m_enablePodEvents = enablePodEvents; + } + +- + m_rService = std::unique_ptr(new CRIRuntimeServiceImpl(podSandboxImage, cb, networkPlugin, m_enablePodEvents)); + } + +-- +2.34.1 + diff --git a/0053-remove-container-root-path-in-rt_lcr_rm-if-lcr-runti.patch b/0053-remove-container-root-path-in-rt_lcr_rm-if-lcr-runti.patch new file mode 100644 index 0000000..1d9b3a3 --- /dev/null +++ b/0053-remove-container-root-path-in-rt_lcr_rm-if-lcr-runti.patch @@ -0,0 +1,33 @@ +From 0cb96d50302f9f3ad1c17e0bb650ac37db4d5206 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Wed, 10 Apr 2024 08:41:46 +0000 +Subject: [PATCH 53/69] remove container root path in rt_lcr_rm if lcr runtime + missing + +Signed-off-by: jikai +--- + src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c | 8 +++++++- + 1 file changed, 7 insertions(+), 1 deletion(-) + +diff --git a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c +index 6b862958..978da079 100644 +--- a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c ++++ b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c +@@ -238,7 +238,13 @@ int rt_lcr_rm(const char *name, const char *runtime, const rt_rm_params_t *param + + engine_ops = engines_get_handler(runtime); + if (engine_ops == NULL || engine_ops->engine_delete_op == NULL) { +- ERROR("Failed to get engine delete operations"); ++ // if engine_ops is NULL, container root path may have been corrupted, try to remove by daemon ++ // If user runs container with lcr but remove lcr runtime after, there might be resources remaining ++ ERROR("Failed to get engine delete operations, container %s root path may have been corrupted, try to remove by daemon", name); ++ if (remove_container_rootpath(name, params->rootpath) == 0) { ++ ret = 0; ++ goto out; ++ } + ret = -1; + goto out; + } +-- +2.34.1 + diff --git a/0054-ensure-sandbox-can-be-removed-if-sandbox-container-r.patch b/0054-ensure-sandbox-can-be-removed-if-sandbox-container-r.patch new file mode 100644 index 0000000..4fefbe1 --- /dev/null +++ b/0054-ensure-sandbox-can-be-removed-if-sandbox-container-r.patch @@ -0,0 +1,58 @@ +From 21afb41e02886df0b5251889cc443f28b7da274f Mon Sep 17 00:00:00 2001 +From: jikai +Date: Thu, 11 Apr 2024 01:21:34 +0000 +Subject: [PATCH 54/69] ensure sandbox can be removed if sandbox container + removed + +Signed-off-by: jikai +--- + .../sandbox/controller/shim/shim_controller.cc | 16 ++++++++++++---- + src/daemon/sandbox/sandbox.cc | 3 ++- + 2 files changed, 14 insertions(+), 5 deletions(-) + +diff --git a/src/daemon/sandbox/controller/shim/shim_controller.cc b/src/daemon/sandbox/controller/shim/shim_controller.cc +index 593fade9..4da637c7 100644 +--- a/src/daemon/sandbox/controller/shim/shim_controller.cc ++++ b/src/daemon/sandbox/controller/shim/shim_controller.cc +@@ -517,12 +517,20 @@ bool ShimController::Shutdown(const std::string &sandboxId, Errors &error) + container_delete_response *response {nullptr}; + int ret = m_cb->container.remove(request, &response); + auto responseWrapper = makeUniquePtrCStructWrapper(response, free_container_delete_response); ++ if (ret == 0) { ++ return true; ++ } + +- if (ret != 0) { +- std::string msg = (response != nullptr && response->errmsg != nullptr) ? response->errmsg : "internal"; +- ERROR("Failed to remove sandbox %s: %s", sandboxId.c_str(), msg.c_str()); +- error.SetError(msg); ++ std::string errMsg = "internal"; ++ if (response != nullptr && response->errmsg != nullptr) { ++ if (strstr(response->errmsg, "No such container") != nullptr) { ++ ERROR("Container for sandbox %s not found", sandboxId.c_str()); ++ return true; ++ } ++ errMsg = response->errmsg; + } ++ ERROR("Failed to remove sandbox %s: %s", sandboxId.c_str(), errMsg.c_str()); ++ error.SetError(errMsg); + return error.Empty(); + } + +diff --git a/src/daemon/sandbox/sandbox.cc b/src/daemon/sandbox/sandbox.cc +index c70116c1..bae5b8db 100644 +--- a/src/daemon/sandbox/sandbox.cc ++++ b/src/daemon/sandbox/sandbox.cc +@@ -757,7 +757,8 @@ auto Sandbox::Remove(Errors &error) -> bool + + WriteGuard lock(m_mutex); + +- if (!DoStop(DEFAULT_STOP_TIMEOUT, error)) { ++ // Only stop the sandbox when it is running ++ if (IsReady() && !DoStop(DEFAULT_STOP_TIMEOUT, error)) { + ERROR("Failed to stop Sandbox before removing, id='%s'", m_id.c_str()); + return false; + } +-- +2.34.1 + diff --git a/0055-bugfix-for-shim-timeout-exit-error-log-changes.patch b/0055-bugfix-for-shim-timeout-exit-error-log-changes.patch new file mode 100644 index 0000000..7009560 --- /dev/null +++ b/0055-bugfix-for-shim-timeout-exit-error-log-changes.patch @@ -0,0 +1,175 @@ +From 35ffb77f568124e6e7c8fd7b3d021878b92c13f7 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Tue, 9 Apr 2024 20:04:33 +0800 +Subject: [PATCH 55/69] bugfix for shim timeout exit error log changes + +Signed-off-by: zhongtao +--- + .../modules/runtime/isula/isula_rt_ops.c | 55 ++++++++++++------- + 1 file changed, 34 insertions(+), 21 deletions(-) + +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index bc3c36c8..1875cf5b 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -861,6 +861,8 @@ static int shim_create(shim_create_args *args) + pid_t pid = 0; + int shim_stderr_pipe[2] = { -1, -1 }; + int shim_stdout_pipe[2] = { -1, -1 }; ++ // used to accept exec error msg ++ int exec_err_pipe[2] = {-1, -1}; + int num = 0; + int ret = 0; + char exec_buff[BUFSIZ + 1] = { 0 }; +@@ -904,6 +906,11 @@ static int shim_create(shim_create_args *args) + return -1; + } + ++ if (pipe2(exec_err_pipe, O_CLOEXEC) != 0) { ++ ERROR("Failed to create pipe for exec err"); ++ return -1; ++ } ++ + pid = fork(); + if (pid < 0) { + SYSERROR("Failed fork for shim parent"); +@@ -911,30 +918,32 @@ static int shim_create(shim_create_args *args) + close(shim_stderr_pipe[1]); + close(shim_stdout_pipe[0]); + close(shim_stdout_pipe[1]); ++ close(exec_err_pipe[0]); ++ close(exec_err_pipe[1]); + return -1; + } + + if (pid == (pid_t)0) { + if (chdir(args->workdir) < 0) { +- (void)dprintf(shim_stderr_pipe[1], "%s: failed chdir to %s", args->id, args->workdir); ++ (void)dprintf(exec_err_pipe[1], "%s: failed chdir to %s", args->id, args->workdir); + exit(EXIT_FAILURE); + } + + //prevent the child process from having the same standard streams as the parent process + if (isula_null_stdfds() != 0) { +- (void)dprintf(shim_stderr_pipe[1], "failed to set std console to /dev/null"); ++ (void)dprintf(exec_err_pipe[1], "failed to set std console to /dev/null"); + exit(EXIT_FAILURE); + } + + if (args->fg) { + // child process, dup2 shim_stdout_pipe[1] to STDOUT, get container process exit_code in STDOUT + if (dup2(shim_stdout_pipe[1], STDOUT_FILENO) < 0) { +- (void)dprintf(shim_stderr_pipe[1], "Dup stdout fd error: %s", strerror(errno)); ++ (void)dprintf(exec_err_pipe[1], "Dup stdout fd error: %s", strerror(errno)); + exit(EXIT_FAILURE); + } + // child process, dup2 shim_stderr_pipe[1] to STDERR, get isulad-shim errmsg in STDERR + if (dup2(shim_stderr_pipe[1], STDERR_FILENO) < 0) { +- (void)dprintf(shim_stderr_pipe[1], "Dup stderr fd error: %s", strerror(errno)); ++ (void)dprintf(exec_err_pipe[1], "Dup stderr fd error: %s", strerror(errno)); + exit(EXIT_FAILURE); + } + goto realexec; +@@ -942,18 +951,18 @@ static int shim_create(shim_create_args *args) + + // clear NOTIFY_SOCKET from the env to adapt runc create + if (unsetenv("NOTIFY_SOCKET") != 0) { +- (void)dprintf(shim_stderr_pipe[1], "%s: unset env NOTIFY_SOCKET failed %s", args->id, strerror(errno)); ++ (void)dprintf(exec_err_pipe[1], "%s: unset env NOTIFY_SOCKET failed %s", args->id, strerror(errno)); + exit(EXIT_FAILURE); + } + + pid = fork(); + if (pid < 0) { +- (void)dprintf(shim_stderr_pipe[1], "%s: fork shim-process failed %s", args->id, strerror(errno)); ++ (void)dprintf(exec_err_pipe[1], "%s: fork shim-process failed %s", args->id, strerror(errno)); + _exit(EXIT_FAILURE); + } + if (pid != 0) { + if (file_write_int(fpid, pid) != 0) { +- (void)dprintf(shim_stderr_pipe[1], "%s: write %s with %d failed", args->id, fpid, pid); ++ (void)dprintf(exec_err_pipe[1], "%s: write %s with %d failed", args->id, fpid, pid); + } + _exit(EXIT_SUCCESS); + } +@@ -962,35 +971,38 @@ realexec: + /* real shim process. */ + close(shim_stderr_pipe[0]); + close(shim_stdout_pipe[0]); ++ close(exec_err_pipe[0]); + + if (setsid() < 0) { +- (void)dprintf(shim_stderr_pipe[1], "%s: failed setsid for process %d", args->id, getpid()); ++ (void)dprintf(exec_err_pipe[1], "%s: failed setsid for process %d", args->id, getpid()); + exit(EXIT_FAILURE); + } + + if (util_check_inherited(true, shim_stderr_pipe[1]) != 0) { +- (void)dprintf(shim_stderr_pipe[1], "close inherited fds failed"); ++ (void)dprintf(exec_err_pipe[1], "close inherited fds failed"); + exit(EXIT_FAILURE); + } + + if (setenv(SHIIM_LOG_PATH_ENV, engine_log_path, 1) != 0) { +- (void)dprintf(shim_stderr_pipe[1], "%s: failed to set SHIIM_LOG_PATH_ENV for process %d", args->id, getpid()); ++ (void)dprintf(exec_err_pipe[1], "%s: failed to set SHIIM_LOG_PATH_ENV for process %d", args->id, getpid()); + exit(EXIT_FAILURE); + } + + if (setenv(SHIIM_LOG_LEVEL_ENV, log_level, 1) != 0) { +- (void)dprintf(shim_stderr_pipe[1], "%s: failed to set SHIIM_LOG_LEVEL_ENV env for process %d", args->id, getpid()); ++ (void)dprintf(exec_err_pipe[1], "%s: failed to set SHIIM_LOG_LEVEL_ENV env for process %d", args->id, getpid()); + exit(EXIT_FAILURE); + } + + execvp(SHIM_BINARY, (char * const *)params); +- (void)dprintf(shim_stderr_pipe[1], "run process: %s failed: %s", SHIM_BINARY, strerror(errno)); ++ (void)dprintf(exec_err_pipe[1], "run process: %s failed: %s", SHIM_BINARY, strerror(errno)); + exit(EXIT_FAILURE); + } + + close(shim_stderr_pipe[1]); + close(shim_stdout_pipe[1]); +- num = util_read_nointr(shim_stderr_pipe[0], exec_buff, sizeof(exec_buff) - 1); ++ close(exec_err_pipe[1]); ++ num = util_read_nointr(exec_err_pipe[0], exec_buff, sizeof(exec_buff) - 1); ++ close(exec_err_pipe[0]); + + status = util_wait_for_pid_status(pid); + if (status < 0) { +@@ -1035,8 +1047,10 @@ realexec: + out: + close(shim_stdout_pipe[0]); + if (ret != 0) { +- show_runtime_errlog(args->workdir); + show_shim_errlog(shim_stderr_pipe[0]); ++ // Since users are more concerned about runtime error information, ++ // the runtime log will overwrite the shim log if it exists. ++ show_runtime_errlog(args->workdir); + if (args->timeout != NULL) { + kill(pid, SIGKILL); /* can kill other process? */ + } +@@ -1491,14 +1505,13 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p + args.exit_code = exit_code; + args.timeout = timeout; + ret = shim_create(&args); +- if (args.shim_exit_code == SHIM_EXIT_TIMEOUT) { +- ret = -1; +- isulad_set_error_message("Exec container error;exec timeout"); +- ERROR("isulad-shim %d exit for execing timeout", pid); +- goto errlog_out; +- } + if (ret != 0) { +- ERROR("%s: failed create shim process for exec %s", id, exec_id); ++ if (args.shim_exit_code == SHIM_EXIT_TIMEOUT) { ++ isulad_set_error_message("Exec container error;exec timeout"); ++ ERROR("isulad-shim %d exit for execing timeout", pid); ++ } else { ++ ERROR("%s: failed create shim process for exec %s", id, exec_id); ++ } + goto errlog_out; + } + +-- +2.34.1 + diff --git a/0056-bugfix-for-the-pre-created-pipe-was-not-closed-when-.patch b/0056-bugfix-for-the-pre-created-pipe-was-not-closed-when-.patch new file mode 100644 index 0000000..607f721 --- /dev/null +++ b/0056-bugfix-for-the-pre-created-pipe-was-not-closed-when-.patch @@ -0,0 +1,36 @@ +From e8ba4368f4be369f99d7da6fc04dcbe173985cd0 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Fri, 12 Apr 2024 14:42:05 +0800 +Subject: [PATCH 56/69] bugfix for the pre-created pipe was not closed when the + pipe creation failed + +Signed-off-by: zhongtao +--- + src/daemon/modules/runtime/isula/isula_rt_ops.c | 6 ++++++ + 1 file changed, 6 insertions(+) + +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 1875cf5b..47a14b1d 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -903,11 +903,17 @@ static int shim_create(shim_create_args *args) + + if (pipe2(shim_stdout_pipe, O_CLOEXEC) != 0) { + ERROR("Failed to create pipe for shim stdout"); ++ close(shim_stderr_pipe[0]); ++ close(shim_stderr_pipe[1]); + return -1; + } + + if (pipe2(exec_err_pipe, O_CLOEXEC) != 0) { + ERROR("Failed to create pipe for exec err"); ++ close(shim_stderr_pipe[0]); ++ close(shim_stderr_pipe[1]); ++ close(shim_stdout_pipe[0]); ++ close(shim_stdout_pipe[1]); + return -1; + } + +-- +2.34.1 + diff --git a/0057-add-debug-msg-info-in-image_load.sh.patch b/0057-add-debug-msg-info-in-image_load.sh.patch new file mode 100644 index 0000000..81f61c8 --- /dev/null +++ b/0057-add-debug-msg-info-in-image_load.sh.patch @@ -0,0 +1,26 @@ +From c3f7cf2a54188e5fc890a8d23b95254ac69cfa52 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Fri, 12 Apr 2024 15:23:07 +0800 +Subject: [PATCH 57/69] add debug msg info in image_load.sh + +Signed-off-by: zhongtao +--- + CI/test_cases/image_cases/image_load.sh | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/CI/test_cases/image_cases/image_load.sh b/CI/test_cases/image_cases/image_load.sh +index a2cada5f..d50b3203 100755 +--- a/CI/test_cases/image_cases/image_load.sh ++++ b/CI/test_cases/image_cases/image_load.sh +@@ -103,6 +103,8 @@ function test_concurrent_load() + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - fail to do isulad load $i" && ((ret++)) + done + ++ tail -n 50 /var/lib/isulad/isulad.log ++ + ubuntu_id=`isula inspect -f '{{.image.id}}' ubuntu` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - fail to inspect image: ubuntu" && ((ret++)) + +-- +2.34.1 + diff --git a/0058-empty-pointer-check-in-lcr_rt_ops.patch b/0058-empty-pointer-check-in-lcr_rt_ops.patch new file mode 100644 index 0000000..1cbe4b2 --- /dev/null +++ b/0058-empty-pointer-check-in-lcr_rt_ops.patch @@ -0,0 +1,214 @@ +From ea1bc00c894a3717ea375f5ff40c3eb05447ae17 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Sat, 13 Apr 2024 14:07:33 +0000 +Subject: [PATCH 58/69] empty pointer check in lcr_rt_ops + +Signed-off-by: jikai +--- + .../modules/runtime/engines/lcr/lcr_rt_ops.c | 84 ++++++++++++++++++- + 1 file changed, 81 insertions(+), 3 deletions(-) + +diff --git a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c +index 978da079..a89d0375 100644 +--- a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c ++++ b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c +@@ -53,6 +53,11 @@ int rt_lcr_create(const char *name, const char *runtime, const rt_create_params_ + char *runtime_root = NULL; + struct engine_operation *engine_ops = NULL; + ++ if (name == NULL || runtime == NULL || params == NULL) { ++ ERROR("Nullptr arguments not allowed"); ++ return -1; ++ } ++ + if (conf_get_systemd_cgroup()) { + ERROR("Systemd cgroup not supported for lcr runtime"); + isulad_set_error_message("Systemd cgroup not supported for lcr runtime"); +@@ -129,6 +134,11 @@ int rt_lcr_start(const char *name, const char *runtime, const rt_start_params_t + struct engine_operation *engine_ops = NULL; + engine_start_request_t request = { 0 }; + ++ if (name == NULL || runtime == NULL || params == NULL || pid_info == NULL) { ++ ERROR("Nullptr arguments not allowed"); ++ return -1; ++ } ++ + engine_ops = engines_get_handler(runtime); + if (engine_ops == NULL || engine_ops->engine_start_op == NULL) { + ERROR("Failed to get engine start operations"); +@@ -183,6 +193,11 @@ int rt_lcr_clean_resource(const char *name, const char *runtime, const rt_clean_ + int ret = 0; + struct engine_operation *engine_ops = NULL; + ++ if (name == NULL || runtime == NULL || params == NULL) { ++ ERROR("Nullptr arguments not allowed"); ++ return -1; ++ } ++ + engine_ops = engines_get_handler(runtime); + if (engine_ops == NULL || engine_ops->engine_clean_op == NULL) { + ERROR("Failed to get engine clean operations"); +@@ -236,6 +251,15 @@ int rt_lcr_rm(const char *name, const char *runtime, const rt_rm_params_t *param + int ret = 0; + struct engine_operation *engine_ops = NULL; + ++ if (name == NULL || runtime == NULL || params == NULL) { ++ ERROR("Nullptr arguments not allowed"); ++ return -1; ++ } ++ if (params->rootpath == NULL) { ++ ERROR("Missing root path"); ++ return -1; ++ } ++ + engine_ops = engines_get_handler(runtime); + if (engine_ops == NULL || engine_ops->engine_delete_op == NULL) { + // if engine_ops is NULL, container root path may have been corrupted, try to remove by daemon +@@ -284,6 +308,11 @@ int rt_lcr_status(const char *name, const char *runtime, const rt_status_params_ + int nret = 0; + struct engine_operation *engine_ops = NULL; + ++ if (name == NULL || runtime == NULL || params == NULL || status == NULL) { ++ ERROR("Nullptr arguments not allowed"); ++ return -1; ++ } ++ + engine_ops = engines_get_handler(runtime); + if (engine_ops == NULL || engine_ops->engine_get_container_status_op == NULL) { + ERROR("Failed to get engine status operations"); +@@ -322,6 +351,11 @@ int rt_lcr_resources_stats(const char *name, const char *runtime, const rt_stats + int nret = 0; + struct engine_operation *engine_ops = NULL; + ++ if (name == NULL || runtime == NULL || params == NULL || rs_stats == NULL) { ++ ERROR("Nullptr arguments not allowed"); ++ return -1; ++ } ++ + engine_ops = engines_get_handler(runtime); + if (engine_ops == NULL || engine_ops->engine_get_container_resources_stats_op == NULL) { + ERROR("Failed to get engine stats operations"); +@@ -451,6 +485,11 @@ int rt_lcr_exec(const char *id, const char *runtime, const rt_exec_params_t *par + char *user = NULL; + char *add_gids = NULL; + ++ if (id == NULL || runtime == NULL || params == NULL || exit_code == NULL) { ++ ERROR("Nullptr arguments not allowed"); ++ return -1; ++ } ++ + engine_ops = engines_get_handler(runtime); + if (engine_ops == NULL || engine_ops->engine_exec_op == NULL) { + DEBUG("Failed to get engine exec operations"); +@@ -519,6 +558,11 @@ int rt_lcr_pause(const char *name, const char *runtime, const rt_pause_params_t + int ret = 0; + struct engine_operation *engine_ops = NULL; + ++ if (name == NULL || runtime == NULL || params == NULL) { ++ ERROR("Nullptr arguments not allowed"); ++ return -1; ++ } ++ + engine_ops = engines_get_handler(runtime); + if (engine_ops == NULL || engine_ops->engine_pause_op == NULL) { + DEBUG("Failed to get engine pause operations"); +@@ -549,6 +593,11 @@ int rt_lcr_resume(const char *name, const char *runtime, const rt_resume_params_ + int ret = 0; + struct engine_operation *engine_ops = NULL; + ++ if (name == NULL || runtime == NULL || params == NULL) { ++ ERROR("Nullptr arguments not allowed"); ++ return -1; ++ } ++ + engine_ops = engines_get_handler(runtime); + if (engine_ops == NULL || engine_ops->engine_resume_op == NULL) { + DEBUG("Failed to get engine resume operations"); +@@ -579,6 +628,11 @@ int rt_lcr_attach(const char *name, const char *runtime, const rt_attach_params_ + int ret = 0; + struct engine_operation *engine_ops = NULL; + ++ if (name == NULL || runtime == NULL || params == NULL) { ++ ERROR("Null argument"); ++ return -1; ++ } ++ + engine_ops = engines_get_handler(runtime); + if (engine_ops == NULL || engine_ops->engine_console_op == NULL) { + DEBUG("Failed to get engine attach operations"); +@@ -641,6 +695,11 @@ int rt_lcr_update(const char *id, const char *runtime, const rt_update_params_t + struct engine_operation *engine_ops = NULL; + struct engine_cgroup_resources cr = { 0 }; + ++ if (id == NULL || runtime == NULL || params == NULL) { ++ ERROR("Nullptr arguments not allowed"); ++ return -1; ++ } ++ + engine_ops = engines_get_handler(runtime); + if (engine_ops == NULL || engine_ops->engine_update_op == NULL) { + DEBUG("Failed to get engine update operations"); +@@ -673,10 +732,9 @@ int rt_lcr_listpids(const char *name, const char *runtime, const rt_listpids_par + int ret = 0; + struct engine_operation *engine_ops = NULL; + +- if (out == NULL) { ++ if (name == NULL || runtime == NULL || params == NULL || out == NULL) { + ERROR("Invalid arguments"); +- ret = -1; +- goto out; ++ return -1; + } + + engine_ops = engines_get_handler(runtime); +@@ -709,6 +767,11 @@ int rt_lcr_resize(const char *id, const char *runtime, const rt_resize_params_t + int ret = 0; + struct engine_operation *engine_ops = NULL; + ++ if (id == NULL || runtime == NULL || params == NULL) { ++ ERROR("Invalid arguments"); ++ return -1; ++ } ++ + engine_ops = engines_get_handler(runtime); + if (engine_ops == NULL || engine_ops->engine_resize_op == NULL) { + DEBUG("Failed to get engine resume operations"); +@@ -740,6 +803,11 @@ int rt_lcr_exec_resize(const char *id, const char *runtime, const rt_exec_resize + int ret = 0; + struct engine_operation *engine_ops = NULL; + ++ if (id == NULL || runtime == NULL || params == NULL) { ++ ERROR("Nullptr arguments not allowed"); ++ return -1; ++ } ++ + engine_ops = engines_get_handler(runtime); + if (engine_ops == NULL || engine_ops->engine_resize_op == NULL) { + DEBUG("Failed to get engine resume operations"); +@@ -767,6 +835,11 @@ out: + + int rt_lcr_kill(const char *id, const char *runtime, const rt_kill_params_t *params) + { ++ if (id == NULL || runtime == NULL || params == NULL || params->pid < 0) { ++ ERROR("Invalid arguments not allowed"); ++ return -1; ++ } ++ + if (util_process_alive(params->pid, params->start_time) == false) { + if (params->signal == params->stop_signal || params->signal == SIGKILL) { + WARN("Process %d is not alive", params->pid); +@@ -798,6 +871,11 @@ int rt_lcr_rebuild_config(const char *name, const char *runtime, const rt_rebuil + oci_runtime_spec *oci_spec = NULL; + __isula_auto_free parser_error err = NULL; + ++ if (name == NULL || runtime == NULL || params == NULL) { ++ ERROR("Invalid arguments not allowed"); ++ return -1; ++ } ++ + engine_ops = engines_get_handler(runtime); + if (engine_ops == NULL || engine_ops->engine_create_op == NULL) { + ERROR("Failed to get engine rebuild config operations"); +-- +2.34.1 + diff --git a/0059-modify-some-grpc-status-codes-of-cri-in-case-of-erro.patch b/0059-modify-some-grpc-status-codes-of-cri-in-case-of-erro.patch new file mode 100644 index 0000000..1153418 --- /dev/null +++ b/0059-modify-some-grpc-status-codes-of-cri-in-case-of-erro.patch @@ -0,0 +1,315 @@ +From 628f4ceb329e16991ed33d3a460bcf8f5542ba99 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Mon, 8 Apr 2024 09:30:04 +0000 +Subject: [PATCH 59/69] modify some grpc status codes of cri in case of error + +Signed-off-by: jikai +--- + .../cri/v1/cri_v1_runtime_runtime_service.cc | 41 +++++++++++++------ + .../cri/v1/cri_v1_runtime_runtime_service.h | 3 ++ + .../v1alpha/cri_runtime_runtime_service.cc | 41 +++++++++++++------ + .../cri/v1alpha/cri_runtime_runtime_service.h | 2 + + 4 files changed, 63 insertions(+), 24 deletions(-) + +diff --git a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc +index e2591ce0..fb5aad3c 100644 +--- a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc ++++ b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc +@@ -50,6 +50,23 @@ static void cri_container_topic_release(void *arg) + delete resp; + } + ++grpc::Status RuntimeV1RuntimeServiceImpl::ToGRPCStatus(Errors &error) ++{ ++ if (error.Empty()) { ++ return grpc::Status::OK; ++ } ++ if (error.GetMessage().find("Failed to find") != std::string::npos) { ++ return grpc::Status(grpc::StatusCode::NOT_FOUND, error.GetMessage()); ++ } ++ ++ // Attach exceeded timeout for lxc and Exec container error;exec timeout for runc ++ if (error.GetMessage().find("Attach exceeded timeout") != std::string::npos ++ || error.GetMessage().find("Exec container error;exec timeout") != std::string::npos) { ++ return grpc::Status(grpc::StatusCode::DEADLINE_EXCEEDED, error.GetMessage()); ++ } ++ return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++} ++ + void RuntimeV1RuntimeServiceImpl::Init(std::string &podSandboxImage, + std::shared_ptr networkPlugin, + bool enablePodEvents, Errors &err) +@@ -167,7 +184,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::CreateContainer(grpc::ServerContext *c + m_rService->CreateContainer(request->pod_sandbox_id(), request->config(), request->sandbox_config(), error); + if (!error.Empty() || responseID.empty()) { + ERROR("Object: CRI, Type: Failed to create container"); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + reply->set_container_id(responseID); + +@@ -192,7 +209,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::StartContainer(grpc::ServerContext *co + m_rService->StartContainer(request->container_id(), error); + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to start container %s", request->container_id().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + EVENT("Event: {Object: CRI, Type: Started Container: %s}", request->container_id().c_str()); +@@ -216,7 +233,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::StopContainer(grpc::ServerContext *con + m_rService->StopContainer(request->container_id(), (int64_t)request->timeout(), error); + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to stop container %s", request->container_id().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + EVENT("Event: {Object: CRI, Type: Stopped Container: %s}", request->container_id().c_str()); +@@ -240,7 +257,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::RemoveContainer(grpc::ServerContext *c + m_rService->RemoveContainer(request->container_id(), error); + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to remove container %s", request->container_id().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + EVENT("Event: {Object: CRI, Type: Removed Container: %s}", request->container_id().c_str()); +@@ -359,7 +376,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::ContainerStatus(grpc::ServerContext *c + m_rService->ContainerStatus(request->container_id(), error); + if (!error.Empty() || !contStatus) { + ERROR("Object: CRI, Type: Failed to get container status %s", request->container_id().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + *(reply->mutable_status()) = *contStatus; + +@@ -384,7 +401,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::ExecSync(grpc::ServerContext *context, + m_rService->ExecSync(request->container_id(), request->cmd(), request->timeout(), reply, error); + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to sync exec container: %s", request->container_id().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + WARN("Event: {Object: CRI, Type: sync execed Container: %s}", request->container_id().c_str()); +@@ -437,7 +454,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::StopPodSandbox(grpc::ServerContext *co + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to stop pod:%s due to %s", request->pod_sandbox_id().c_str(), + error.GetMessage().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + EVENT("Event: {Object: CRI, Type: Stopped Pod: %s}", request->pod_sandbox_id().c_str()); +@@ -462,7 +479,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::RemovePodSandbox(grpc::ServerContext * + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to remove pod:%s due to %s", request->pod_sandbox_id().c_str(), + error.GetMessage().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + EVENT("Event: {Object: CRI, Type: Removed Pod: %s}", request->pod_sandbox_id().c_str()); +@@ -487,7 +504,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::PodSandboxStatus(grpc::ServerContext * + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to status pod:%s due to %s", request->pod_sandbox_id().c_str(), + error.GetMessage().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + INFO("Event: {Object: CRI, Type: Statused Pod: %s}", request->pod_sandbox_id().c_str()); +@@ -608,7 +625,7 @@ RuntimeV1RuntimeServiceImpl::UpdateContainerResources(grpc::ServerContext *conte + if (error.NotEmpty()) { + ERROR("Object: CRI, Type: Failed to update container:%s due to %s", request->container_id().c_str(), + error.GetMessage().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + WARN("Event: {Object: CRI, Type: Updated container resources: %s}", request->container_id().c_str()); +@@ -633,7 +650,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::Exec(grpc::ServerContext *context, + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to exec container:%s due to %s", request->container_id().c_str(), + error.GetMessage().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + EVENT("Event: {Object: CRI, Type: execed Container: %s}", request->container_id().c_str()); +@@ -658,7 +675,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::Attach(grpc::ServerContext *context, + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to attach container:%s due to %s", request->container_id().c_str(), + error.GetMessage().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + EVENT("Event: {Object: CRI, Type: attched Container: %s}", request->container_id().c_str()); +diff --git a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h +index 842d1811..1cf375a4 100644 +--- a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h ++++ b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h +@@ -114,6 +114,9 @@ public: + grpc::ServerWriter *writer) override; + + private: ++ ++ grpc::Status ToGRPCStatus(Errors &error); ++ + std::unique_ptr m_rService; + bool m_enablePodEvents; + }; +diff --git a/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.cc b/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.cc +index 5e85702c..1c83f4ca 100644 +--- a/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.cc ++++ b/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.cc +@@ -23,6 +23,23 @@ + + using namespace CRI; + ++grpc::Status RuntimeRuntimeServiceImpl::ToGRPCStatus(Errors &error) ++{ ++ if (error.Empty()) { ++ return grpc::Status::OK; ++ } ++ if (error.GetMessage().find("Failed to find") != std::string::npos) { ++ return grpc::Status(grpc::StatusCode::NOT_FOUND, error.GetMessage()); ++ } ++ ++ // Attach exceeded timeout for lxc and Exec container error;exec timeout for runc ++ if (error.GetMessage().find("Attach exceeded timeout") != std::string::npos ++ || error.GetMessage().find("Exec container error;exec timeout") != std::string::npos) { ++ return grpc::Status(grpc::StatusCode::DEADLINE_EXCEEDED, error.GetMessage()); ++ } ++ return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++} ++ + void RuntimeRuntimeServiceImpl::Init(std::string &podSandboxImage, + std::shared_ptr networkPlugin, Errors &err) + { +@@ -80,7 +97,7 @@ grpc::Status RuntimeRuntimeServiceImpl::CreateContainer(grpc::ServerContext *con + m_rService->CreateContainer(request->pod_sandbox_id(), request->config(), request->sandbox_config(), error); + if (!error.Empty() || responseID.empty()) { + ERROR("Object: CRI, Type: Failed to create container"); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + reply->set_container_id(responseID); + +@@ -105,7 +122,7 @@ grpc::Status RuntimeRuntimeServiceImpl::StartContainer(grpc::ServerContext *cont + m_rService->StartContainer(request->container_id(), error); + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to start container %s", request->container_id().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + EVENT("Event: {Object: CRI, Type: Started Container: %s}", request->container_id().c_str()); +@@ -129,7 +146,7 @@ grpc::Status RuntimeRuntimeServiceImpl::StopContainer(grpc::ServerContext *conte + m_rService->StopContainer(request->container_id(), (int64_t)request->timeout(), error); + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to stop container %s", request->container_id().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + EVENT("Event: {Object: CRI, Type: Stopped Container: %s}", request->container_id().c_str()); +@@ -153,7 +170,7 @@ grpc::Status RuntimeRuntimeServiceImpl::RemoveContainer(grpc::ServerContext *con + m_rService->RemoveContainer(request->container_id(), error); + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to remove container %s", request->container_id().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + EVENT("Event: {Object: CRI, Type: Removed Container: %s}", request->container_id().c_str()); +@@ -272,7 +289,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ContainerStatus(grpc::ServerContext *con + m_rService->ContainerStatus(request->container_id(), error); + if (!error.Empty() || !contStatus) { + ERROR("Object: CRI, Type: Failed to get container status %s", request->container_id().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + *(reply->mutable_status()) = *contStatus; + +@@ -297,7 +314,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ExecSync(grpc::ServerContext *context, + m_rService->ExecSync(request->container_id(), request->cmd(), request->timeout(), reply, error); + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to sync exec container: %s", request->container_id().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + WARN("Event: {Object: CRI, Type: sync execed Container: %s}", request->container_id().c_str()); +@@ -351,7 +368,7 @@ grpc::Status RuntimeRuntimeServiceImpl::StopPodSandbox(grpc::ServerContext *cont + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to stop pod:%s due to %s", request->pod_sandbox_id().c_str(), + error.GetMessage().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + EVENT("Event: {Object: CRI, Type: Stopped Pod: %s}", request->pod_sandbox_id().c_str()); +@@ -376,7 +393,7 @@ grpc::Status RuntimeRuntimeServiceImpl::RemovePodSandbox(grpc::ServerContext *co + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to remove pod:%s due to %s", request->pod_sandbox_id().c_str(), + error.GetMessage().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + EVENT("Event: {Object: CRI, Type: Removed Pod: %s}", request->pod_sandbox_id().c_str()); +@@ -402,7 +419,7 @@ grpc::Status RuntimeRuntimeServiceImpl::PodSandboxStatus(grpc::ServerContext *co + if (!error.Empty() || !podStatus) { + ERROR("Object: CRI, Type: Failed to status pod:%s due to %s", request->pod_sandbox_id().c_str(), + error.GetMessage().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + *(reply->mutable_status()) = *podStatus; + +@@ -523,7 +540,7 @@ RuntimeRuntimeServiceImpl::UpdateContainerResources(grpc::ServerContext *context + if (error.NotEmpty()) { + ERROR("Object: CRI, Type: Failed to update container:%s due to %s", request->container_id().c_str(), + error.GetMessage().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + WARN("Event: {Object: CRI, Type: Updated container resources: %s}", request->container_id().c_str()); +@@ -548,7 +565,7 @@ grpc::Status RuntimeRuntimeServiceImpl::Exec(grpc::ServerContext *context, + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to exec container:%s due to %s", request->container_id().c_str(), + error.GetMessage().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + EVENT("Event: {Object: CRI, Type: execed Container: %s}", request->container_id().c_str()); +@@ -573,7 +590,7 @@ grpc::Status RuntimeRuntimeServiceImpl::Attach(grpc::ServerContext *context, + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to attach container:%s due to %s", request->container_id().c_str(), + error.GetMessage().c_str()); +- return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ return ToGRPCStatus(error); + } + + EVENT("Event: {Object: CRI, Type: attched Container: %s}", request->container_id().c_str()); +diff --git a/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.h b/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.h +index e0f75897..210e67cc 100644 +--- a/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.h ++++ b/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.h +@@ -103,6 +103,8 @@ public: + runtime::v1alpha2::StatusResponse *reply) override; + + private: ++ grpc::Status ToGRPCStatus(Errors &err); ++ + std::unique_ptr m_rService; + }; + +-- +2.34.1 + diff --git a/0060-cdi-return-int-instead-of-error-string.patch b/0060-cdi-return-int-instead-of-error-string.patch new file mode 100644 index 0000000..e2ebc9b --- /dev/null +++ b/0060-cdi-return-int-instead-of-error-string.patch @@ -0,0 +1,363 @@ +From e7b94411b174c8445d9bdc84ec6c94b5d4343470 Mon Sep 17 00:00:00 2001 +From: liuxu +Date: Mon, 15 Apr 2024 15:38:57 +0800 +Subject: [PATCH 60/69] cdi:return int instead of error string + +--- + src/daemon/modules/api/cdi_operate_api.h | 7 ++++--- + .../device/cdi/behavior/cdi_container_edits.c | 12 ++++++------ + .../device/cdi/behavior/cdi_container_edits.h | 6 +++--- + .../modules/device/cdi/behavior/cdi_device.c | 2 +- + .../modules/device/cdi/behavior/cdi_device.h | 2 +- + .../modules/device/cdi/behavior/cdi_spec.c | 2 +- + .../modules/device/cdi/behavior/cdi_spec.h | 2 +- + .../modules/device/cdi/behavior/cdi_spec_dirs.c | 4 ++-- + .../modules/device/cdi/behavior/cdi_spec_dirs.h | 6 +++--- + .../device/cdi/behavior/parser/cdi_parser.c | 16 ++++++++-------- + .../device/cdi/behavior/parser/cdi_parser.h | 8 ++++---- + src/daemon/modules/device/cdi/cdi_annotations.c | 5 +++-- + src/daemon/modules/device/cdi/cdi_annotations.h | 3 ++- + src/daemon/modules/device/cdi/cdi_cache.c | 14 +++++++------- + src/daemon/modules/device/cdi/cdi_cache.h | 8 ++++---- + src/daemon/modules/device/cdi_operate.c | 13 +++++++------ + 16 files changed, 57 insertions(+), 53 deletions(-) + +diff --git a/src/daemon/modules/api/cdi_operate_api.h b/src/daemon/modules/api/cdi_operate_api.h +index 4f4c339e..49820ed7 100644 +--- a/src/daemon/modules/api/cdi_operate_api.h ++++ b/src/daemon/modules/api/cdi_operate_api.h +@@ -26,11 +26,12 @@ extern "C" { + + int cdi_operate_registry_init(char **specs_dirs, size_t specs_dirs_len); + +-char *cdi_operate_refresh(void); ++int cdi_operate_refresh(void); + +-string_array *cdi_operate_inject_devices(oci_runtime_spec *spec, string_array *devices, char **error); ++int cdi_operate_inject_devices(oci_runtime_spec *spec, string_array *devices); + +-char *cdi_operate_parse_annotations(json_map_string_string *annotations, string_array **keys, string_array **devices); ++int cdi_operate_parse_annotations(json_map_string_string *annotations, string_array **keys, ++ string_array **devices, char **error); + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c +index ce7b16db..590118b1 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c +@@ -27,19 +27,19 @@ + // POSTSTOP_HOOK is the name of the OCI "poststop" hook. + #define POSTSTOP_HOOK "poststop" + +-char *cdi_container_edits_apply(cdi_container_edits *e, oci_runtime_spec *spec) ++int cdi_container_edits_apply(cdi_container_edits *e, oci_runtime_spec *spec) + { +- return NULL; ++ return 0; + } + +-char *cdi_container_edits_validate(cdi_container_edits *e) ++int cdi_container_edits_validate(cdi_container_edits *e, char **error) + { +- return NULL; ++ return 0; + } + +-cdi_container_edits *cdi_container_edits_append(cdi_container_edits *e, cdi_container_edits *o) ++int cdi_container_edits_append(cdi_container_edits *e, cdi_container_edits *o) + { +- return NULL; ++ return 0; + } + + bool cdi_container_edits_is_empty(cdi_container_edits *e) +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_container_edits.h b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.h +index 7b16d2bc..ea921e37 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_container_edits.h ++++ b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.h +@@ -27,9 +27,9 @@ + extern "C" { + #endif + +-char *cdi_container_edits_apply(cdi_container_edits *e, oci_runtime_spec *spec); +-char *cdi_container_edits_validate(cdi_container_edits *e); +-cdi_container_edits *cdi_container_edits_append(cdi_container_edits *e, cdi_container_edits *o); ++int cdi_container_edits_apply(cdi_container_edits *e, oci_runtime_spec *spec); ++int cdi_container_edits_validate(cdi_container_edits *e, char **error); ++int cdi_container_edits_append(cdi_container_edits *e, cdi_container_edits *o); + bool cdi_container_edits_is_empty(cdi_container_edits *e); + + #ifdef __cplusplus +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_device.c b/src/daemon/modules/device/cdi/behavior/cdi_device.c +index 9904e9ee..0fef8f42 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_device.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_device.c +@@ -34,7 +34,7 @@ char *cdi_device_get_qualified_name(struct cdi_cache_device *d) + return NULL; + } + +-cdi_container_edits *cdi_device_edits(struct cdi_cache_device *d) ++cdi_container_edits *cdi_device_get_edits(struct cdi_cache_device *d) + { + return NULL; + } +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_device.h b/src/daemon/modules/device/cdi/behavior/cdi_device.h +index 3f460152..5d63a576 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_device.h ++++ b/src/daemon/modules/device/cdi/behavior/cdi_device.h +@@ -37,7 +37,7 @@ void free_cdi_cache_device(struct cdi_cache_device *d); + struct cdi_cache_device *cdi_device_new_device(struct cdi_cache_spec *spec, cdi_device *d, char **error); + struct cdi_cache_spec *cdi_device_get_spec(struct cdi_cache_device *d); + char *cdi_device_get_qualified_name(struct cdi_cache_device *d); +-cdi_container_edits *cdi_device_edits(struct cdi_cache_device *d); ++cdi_container_edits *cdi_device_get_edits(struct cdi_cache_device *d); + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec.c b/src/daemon/modules/device/cdi/behavior/cdi_spec.c +index 38fc9e38..f79b5a44 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_spec.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec.c +@@ -54,7 +54,7 @@ int cdi_spec_get_priority(struct cdi_cache_spec *s) + return 0; + } + +-cdi_container_edits *cdi_spec_edits(struct cdi_cache_spec *s) ++cdi_container_edits *cdi_spec_get_edits(struct cdi_cache_spec *s) + { + return NULL; + } +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec.h b/src/daemon/modules/device/cdi/behavior/cdi_spec.h +index bd4fc9d1..87248041 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_spec.h ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec.h +@@ -47,7 +47,7 @@ const char *cdi_spec_get_class(struct cdi_cache_spec *s); + struct cdi_cache_device *cdi_spec_get_cache_device(struct cdi_cache_spec *s, const char *name); + const char *cdi_spec_get_path(struct cdi_cache_spec *s); + int cdi_spec_get_priority(struct cdi_cache_spec *s); +-cdi_container_edits *cdi_spec_edits(struct cdi_cache_spec *s); ++cdi_container_edits *cdi_spec_get_edits(struct cdi_cache_spec *s); + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c +index 5df4c937..e340abc0 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c +@@ -23,7 +23,7 @@ string_array g_default_spec_dirs = { + .cap = DEFAULT_SPEC_DIRS_LEN, + }; + +-char *cdi_scan_spec_dirs(string_array *dirs, struct cdi_scan_fn_maps *scan_fn_maps, cdi_scan_spec_func scan_fn) ++int cdi_scan_spec_dirs(string_array *dirs, struct cdi_scan_fn_maps *scan_fn_maps, cdi_scan_spec_func scan_fn) + { +- return NULL; ++ return 0; + } +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h +index bd00e318..73d8c0f5 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h +@@ -35,10 +35,10 @@ struct cdi_scan_fn_maps { + map_t *spec_errors; + string_array *result; + }; +-typedef char *(*cdi_scan_spec_func)(struct cdi_scan_fn_maps *scan_fn_maps, const char *path, int priority, +- struct cdi_cache_spec *spec, char **error); ++typedef void(*cdi_scan_spec_func)(struct cdi_scan_fn_maps *scan_fn_maps, const char *path, int priority, ++ struct cdi_cache_spec *spec, char *error); + +-char *cdi_scan_spec_dirs(string_array *dirs, struct cdi_scan_fn_maps *scan_fn_maps, cdi_scan_spec_func scan_fn); ++int cdi_scan_spec_dirs(string_array *dirs, struct cdi_scan_fn_maps *scan_fn_maps, cdi_scan_spec_func scan_fn); + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c +index 45048f9a..14293c72 100644 +--- a/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c ++++ b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c +@@ -24,9 +24,9 @@ bool cdi_parser_is_qualified_name(const char *device) + return true; + } + +-char *cdi_parser_parse_qualified_name(const char *device, char **vendor, char **class, char **name) ++int cdi_parser_parse_qualified_name(const char *device, char **vendor, char **class, char **name) + { +- return NULL; ++ return 0; + } + + int cdi_parser_parse_device(const char *device, char **vendor, char **class, char **name) +@@ -39,17 +39,17 @@ int cdi_parser_parse_qualifier(const char *kind, char **vendor, char **class) + return 0; + } + +-char *cdi_parser_validate_vendor_name(const char *vendor) ++int cdi_parser_validate_vendor_name(const char *vendor, char **error) + { +- return NULL; ++ return 0; + } + +-char *cdi_parser_validate_class_name(const char *class) ++int cdi_parser_validate_class_name(const char *class, char **error) + { +- return NULL; ++ return 0; + } + +-char *cdi_parser_validate_device_name(const char *name) ++int cdi_parser_validate_device_name(const char *name, char **error) + { +- return NULL; ++ return 0; + } +diff --git a/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.h b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.h +index d9c057ea..467641a1 100644 +--- a/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.h ++++ b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.h +@@ -24,12 +24,12 @@ extern "C" { + + char *cdi_parser_qualified_name(const char *vendor, const char *class, const char *name); + bool cdi_parser_is_qualified_name(const char *device); +-char *cdi_parser_parse_qualified_name(const char *device, char **vendor, char **class, char **name); ++int cdi_parser_parse_qualified_name(const char *device, char **vendor, char **class, char **name); + int cdi_parser_parse_device(const char *device, char **vendor, char **class, char **name); + int cdi_parser_parse_qualifier(const char *kind, char **vendor, char **class); +-char *cdi_parser_validate_vendor_name(const char *vendor); +-char *cdi_parser_validate_class_name(const char *class); +-char *cdi_parser_validate_device_name(const char *name); ++int cdi_parser_validate_vendor_name(const char *vendor, char **error); ++int cdi_parser_validate_class_name(const char *class, char **error); ++int cdi_parser_validate_device_name(const char *name, char **error); + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/device/cdi/cdi_annotations.c b/src/daemon/modules/device/cdi/cdi_annotations.c +index 3cb9be84..cfe6e099 100644 +--- a/src/daemon/modules/device/cdi/cdi_annotations.c ++++ b/src/daemon/modules/device/cdi/cdi_annotations.c +@@ -25,7 +25,8 @@ + + #define CDI_ANNOTATIONS_PREFIX "cdi.k8s.io/" + +-char *cdi_parse_annotations(json_map_string_string *annotations, string_array **keys, string_array **devices) ++int cdi_parse_annotations(json_map_string_string *annotations, string_array **keys, ++ string_array **devices, char **error) + { +- return NULL; ++ return 0; + } +diff --git a/src/daemon/modules/device/cdi/cdi_annotations.h b/src/daemon/modules/device/cdi/cdi_annotations.h +index 52355099..49930963 100644 +--- a/src/daemon/modules/device/cdi/cdi_annotations.h ++++ b/src/daemon/modules/device/cdi/cdi_annotations.h +@@ -23,7 +23,8 @@ + extern "C" { + #endif + +-char *cdi_parse_annotations(json_map_string_string *annotations, string_array **keys, string_array **devices); ++int cdi_parse_annotations(json_map_string_string *annotations, string_array **keys, ++ string_array **devices, char **error); + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/device/cdi/cdi_cache.c b/src/daemon/modules/device/cdi/cdi_cache.c +index 9c54acbf..cfc23a1c 100644 +--- a/src/daemon/modules/device/cdi/cdi_cache.c ++++ b/src/daemon/modules/device/cdi/cdi_cache.c +@@ -19,24 +19,24 @@ void free_cdi_cache(struct cdi_cache *c) + (void)c; + } + +-struct cdi_cache *cdi_new_cache(string_array *spec_dirs, char **error) ++struct cdi_cache *cdi_new_cache(string_array *spec_dirs) + { + return NULL; + } + +-static string_array *cdi_inject_devices(struct cdi_cache *c, oci_runtime_spec *oci_spec, string_array *devices, char **error) ++static int cdi_inject_devices(struct cdi_cache *c, oci_runtime_spec *oci_spec, string_array *devices) + { +- return NULL; ++ return 0; + } + +-static char *cdi_configure(struct cdi_cache *c, string_array *spec_dirs) ++static int cdi_configure(struct cdi_cache *c, string_array *spec_dirs) + { +- return NULL; ++ return 0; + } + +-static char *cdi_refresh(struct cdi_cache *c) ++static int cdi_refresh(struct cdi_cache *c) + { +- return NULL; ++ return 0; + } + + static map_t *cdi_get_errors(struct cdi_cache *c) +diff --git a/src/daemon/modules/device/cdi/cdi_cache.h b/src/daemon/modules/device/cdi/cdi_cache.h +index 92fb64af..34c27471 100644 +--- a/src/daemon/modules/device/cdi/cdi_cache.h ++++ b/src/daemon/modules/device/cdi/cdi_cache.h +@@ -33,12 +33,12 @@ struct cdi_cache; + struct cdi_cache_ops { + // injecting CDI devices into an OCI Spec. + // Resolver +- string_array *(*inject_devices)(struct cdi_cache *c, oci_runtime_spec *spec, string_array *devices, char **error); ++ int (*inject_devices)(struct cdi_cache *c, oci_runtime_spec *spec, string_array *devices); + + // refreshing the cache of CDI Specs and devices. + // Refresher +- char *(*configure)(struct cdi_cache *c, string_array *spec_dirs); +- char *(*refresh)(struct cdi_cache *c); ++ int (*configure)(struct cdi_cache *c, string_array *spec_dirs); ++ int (*refresh)(struct cdi_cache *c); + map_t *(*get_errors)(struct cdi_cache *c); + string_array *(*get_spec_directories)(struct cdi_cache *c); + map_t *(*get_spec_dir_errors)(struct cdi_cache *c); +@@ -65,7 +65,7 @@ struct cdi_cache { + + void free_cdi_cache(struct cdi_cache *c); + +-struct cdi_cache *cdi_new_cache(string_array *spec_dirs, char **error); ++struct cdi_cache *cdi_new_cache(string_array *spec_dirs); + struct cdi_cache_ops *cdi_get_cache_ops(void); + + #ifdef __cplusplus +diff --git a/src/daemon/modules/device/cdi_operate.c b/src/daemon/modules/device/cdi_operate.c +index c7aa77d8..c5187ab1 100644 +--- a/src/daemon/modules/device/cdi_operate.c ++++ b/src/daemon/modules/device/cdi_operate.c +@@ -19,17 +19,18 @@ int cdi_operate_registry_init(char **specs_dirs, size_t specs_dirs_len) + return 0; + } + +-char *cdi_operate_refresh(void) ++int cdi_operate_refresh(void) + { +- return NULL; ++ return 0; + } + +-string_array *cdi_operate_inject_devices(oci_runtime_spec *spec, string_array *devices, char **error) ++int cdi_operate_inject_devices(oci_runtime_spec *spec, string_array *devices) + { +- return NULL; ++ return 0; + } + +-char *cdi_operate_parse_annotations(json_map_string_string *annotations, string_array **keys, string_array **devices) ++int cdi_operate_parse_annotations(json_map_string_string *annotations, string_array **keys, ++ string_array **devices, char **error) + { +- return NULL; ++ return 0; + } +\ No newline at end of file +-- +2.34.1 + diff --git a/0061-cdi-support-modules-operate-registry-annotations.patch b/0061-cdi-support-modules-operate-registry-annotations.patch new file mode 100644 index 0000000..aa26ca2 --- /dev/null +++ b/0061-cdi-support-modules-operate-registry-annotations.patch @@ -0,0 +1,516 @@ +From 4527dc8b6f7ab438742a8f7403b24420f646236d Mon Sep 17 00:00:00 2001 +From: liuxu +Date: Mon, 8 Apr 2024 11:57:16 +0800 +Subject: [PATCH 61/69] cdi:support modules operate/registry/annotations + +--- + .../modules/device/cdi/cdi_annotations.c | 72 +++++++ + src/daemon/modules/device/cdi/cdi_registry.c | 14 +- + src/daemon/modules/device/cdi_operate.c | 53 +++++- + src/utils/cutils/utils.c | 11 ++ + src/utils/cutils/utils.h | 2 + + src/utils/cutils/utils_array.c | 175 +++++++++++++++++- + src/utils/cutils/utils_array.h | 31 ++++ + 7 files changed, 350 insertions(+), 8 deletions(-) + +diff --git a/src/daemon/modules/device/cdi/cdi_annotations.c b/src/daemon/modules/device/cdi/cdi_annotations.c +index cfe6e099..020816d7 100644 +--- a/src/daemon/modules/device/cdi/cdi_annotations.c ++++ b/src/daemon/modules/device/cdi/cdi_annotations.c +@@ -21,12 +21,84 @@ + + #include "error.h" + #include "utils.h" ++#include "utils_array.h" + #include "cdi_parser.h" + + #define CDI_ANNOTATIONS_PREFIX "cdi.k8s.io/" + ++static int parse_devices(string_array *devices, const char *value, char **error) ++{ ++ __isula_auto_array_t char **parts = NULL; ++ char **pos; ++ ++ parts = util_string_split(value, ','); ++ if (parts == NULL) { ++ ERROR("Invalid CDI device value %s", value); ++ format_errorf(error, "Invalid CDI device value %s", value); ++ return -1; ++ } ++ for (pos = parts; pos != NULL && *pos != NULL; pos++) { ++ if (!cdi_parser_is_qualified_name(*pos)) { ++ ERROR("Invalid CDI device name %s", *pos); ++ format_errorf(error, "Invalid CDI device name %s", *pos); ++ return -1; ++ } ++ if (util_append_string_array(devices, *pos) != 0) { ++ ERROR("Out of memory"); ++ *error = util_strdup_s("Out of memory"); ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ + int cdi_parse_annotations(json_map_string_string *annotations, string_array **keys, + string_array **devices, char **error) + { ++ char *key = NULL; ++ char *value = NULL; ++ size_t i; ++ __isula_auto_string_array_t string_array *keys_array = NULL; ++ __isula_auto_string_array_t string_array *devices_array = NULL; ++ ++ if (annotations == NULL || keys == NULL || devices == NULL || error == NULL) { ++ ERROR("Invalid argument"); ++ return -1; ++ } ++ ++ keys_array = util_common_calloc_s(sizeof(*keys_array)); ++ if (keys_array == NULL) { ++ ERROR("Out of memory"); ++ *error = util_strdup_s("Out of memory"); ++ return -1; ++ } ++ devices_array = util_common_calloc_s(sizeof(*devices_array)); ++ if (devices_array == NULL) { ++ ERROR("Out of memory"); ++ *error = util_strdup_s("Out of memory"); ++ return -1; ++ } ++ ++ for (i = 0; i < annotations->len; i++) { ++ key = annotations->keys[i]; ++ value = annotations->values[i]; ++ if (!util_has_prefix(key, CDI_ANNOTATIONS_PREFIX)) { ++ continue; ++ } ++ if (parse_devices(devices_array, value, error) != 0) { ++ return -1; ++ } ++ if (util_append_string_array(keys_array, key) != 0) { ++ ERROR("Out of memory"); ++ *error = util_strdup_s("Out of memory"); ++ return -1; ++ } ++ } ++ ++ *keys = keys_array; ++ keys_array = NULL; ++ *devices = devices_array; ++ devices_array = NULL; + return 0; + } +diff --git a/src/daemon/modules/device/cdi/cdi_registry.c b/src/daemon/modules/device/cdi/cdi_registry.c +index 68767a5f..be381132 100644 +--- a/src/daemon/modules/device/cdi/cdi_registry.c ++++ b/src/daemon/modules/device/cdi/cdi_registry.c +@@ -14,12 +14,24 @@ + ******************************************************************************/ + #include "cdi_registry.h" + ++#include ++#include ++ ++static struct cdi_registry g_cdi_reg = { 0 }; ++ + int cdi_registry_init(string_array *spec_dirs) + { ++ // isulad will use default dirs when spec_dirs == NULL ++ g_cdi_reg.cdi_cache = cdi_new_cache(spec_dirs); ++ if (g_cdi_reg.cdi_cache == NULL) { ++ ERROR("Failed to init registry"); ++ return -1; ++ } ++ g_cdi_reg.ops = cdi_get_cache_ops(); + return 0; + } + + struct cdi_registry *cdi_get_registry(void) + { +- return NULL; ++ return &g_cdi_reg; + } +diff --git a/src/daemon/modules/device/cdi_operate.c b/src/daemon/modules/device/cdi_operate.c +index c5187ab1..f99bb7e4 100644 +--- a/src/daemon/modules/device/cdi_operate.c ++++ b/src/daemon/modules/device/cdi_operate.c +@@ -14,23 +14,66 @@ + ******************************************************************************/ + #include "cdi_operate_api.h" + ++#include ++ ++#include "utils.h" ++#include "error.h" ++#include "cdi_registry.h" ++#include "cdi_annotations.h" ++#include "cdi_spec_dirs.h" ++ + int cdi_operate_registry_init(char **specs_dirs, size_t specs_dirs_len) + { +- return 0; ++ string_array spec_dirs_array = { ++ .items = specs_dirs, ++ .len = specs_dirs_len, ++ .cap = specs_dirs_len, ++ }; ++ ++ return cdi_registry_init(&spec_dirs_array); + } + + int cdi_operate_refresh(void) + { +- return 0; ++ struct cdi_registry *registry = cdi_get_registry(); ++ if (registry == NULL || registry->ops == NULL || registry->ops->refresh == NULL) { ++ ERROR("Failed to get registry"); ++ return -1; ++ } ++ ++ return registry->ops->refresh(registry->cdi_cache); + } + + int cdi_operate_inject_devices(oci_runtime_spec *spec, string_array *devices) + { +- return 0; ++ struct cdi_registry *registry = NULL; ++ ++ if (spec == NULL || devices == NULL) { ++ ERROR("Invalid params"); ++ return -1; ++ } ++ ++ registry = cdi_get_registry(); ++ if (registry == NULL || registry->ops == NULL || registry->ops->inject_devices == NULL) { ++ ERROR("Failed to get registry"); ++ return -1; ++ } ++ ++ return registry->ops->inject_devices(registry->cdi_cache, spec, devices); + } + + int cdi_operate_parse_annotations(json_map_string_string *annotations, string_array **keys, + string_array **devices, char **error) + { +- return 0; +-} +\ No newline at end of file ++ if (error == NULL) { ++ ERROR("Invalid argument"); ++ return -1; ++ } ++ if (annotations == NULL || keys == NULL || devices == NULL) { ++ ERROR("Invalid params"); ++ *error = util_strdup_s("Invalid params"); ++ return -1; ++ } ++ ++ return cdi_parse_annotations(annotations, keys, devices, error); ++} +diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c +index 8da2cc60..9a33f935 100644 +--- a/src/utils/cutils/utils.c ++++ b/src/utils/cutils/utils.c +@@ -59,6 +59,17 @@ int malloc_trim(size_t pad) + } + #endif + ++void util_swap_ptr(void **p1, void **p2) ++{ ++ void *tmp; ++ if (p1 == NULL || p2 == NULL) { ++ return; ++ } ++ tmp = *p1; ++ *p1 = *p2; ++ *p2 = tmp; ++} ++ + int util_mem_realloc(void **newptr, size_t newsize, void *oldptr, size_t oldsize) + { + void *tmp = NULL; +diff --git a/src/utils/cutils/utils.h b/src/utils/cutils/utils.h +index 3acf0698..3671272a 100644 +--- a/src/utils/cutils/utils.h ++++ b/src/utils/cutils/utils.h +@@ -320,6 +320,8 @@ struct signame { + } \ + } while (0) + ++void util_swap_ptr(void **p1, void **p2); ++ + int util_mem_realloc(void **newptr, size_t newsize, void *oldptr, size_t oldsize); + + int util_check_inherited(bool closeall, int fd_to_ignore); +diff --git a/src/utils/cutils/utils_array.c b/src/utils/cutils/utils_array.c +index 25f19b8b..72294005 100644 +--- a/src/utils/cutils/utils_array.c ++++ b/src/utils/cutils/utils_array.c +@@ -86,6 +86,27 @@ void util_free_sensitive_array(char **array) + free(array); + } + ++char **util_copy_array_by_len(char **array, size_t len) ++{ ++ char **new_array = NULL; ++ size_t i; ++ ++ if (array == NULL || len == 0) { ++ return NULL; ++ } ++ ++ new_array = util_smart_calloc_s(sizeof(char *), len); ++ if (new_array == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ for (i = 0; i < len; i++) { ++ new_array[i] = util_strdup_s(array[i]); ++ } ++ return new_array; ++} ++ + int util_array_append(char ***array, const char *element) + { + size_t len; +@@ -166,7 +187,7 @@ bool util_array_contain(const char **array, const char *element) + return false; + } + +-static size_t get_string_array_scale_size(size_t old_size) ++static size_t get_array_scale_size(size_t old_size) + { + #define DOUBLE_THRESHOLD 1024 + const size_t max_threshold = MAX_MEMORY_SIZE / sizeof(char *); +@@ -188,7 +209,7 @@ static size_t get_string_array_scale_size(size_t old_size) + + static bool do_expand_array(string_array *array) + { +- size_t new_size = get_string_array_scale_size(array->cap); ++ size_t new_size = get_array_scale_size(array->cap); + char **new_items = NULL; + + // array capability sure less than MAX_MEMORY_SIZE +@@ -237,6 +258,29 @@ out: + return 0; + } + ++string_array *util_copy_string_array(string_array *sarr) ++{ ++ string_array *ptr = NULL; ++ size_t i; ++ ++ if (sarr == NULL) { ++ ERROR("Invalid string array"); ++ return NULL; ++ } ++ ++ ptr = util_string_array_new(sarr->cap); ++ if (ptr == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ for (i = 0; i < sarr->len; i++) { ++ ptr->items[i] = util_strdup_s(sarr->items[i]); ++ ptr->len += 1; ++ } ++ ++ return ptr; ++} ++ + bool util_string_array_contain(const string_array *sarr, const char *elem) + { + size_t i; +@@ -339,3 +383,130 @@ int util_common_array_append_pointer(void ***array, void *element) + + return 0; + } ++ ++void *util_clone_ptr(void *item) ++{ ++ return item; ++} ++ ++common_array *util_common_array_new(size_t len, free_common_array_item_cb free_item_cb, ++ clone_common_array_item_cb clone_item_cb) ++{ ++ common_array *ptr = NULL; ++ ++ if (len == 0 || free_item_cb == NULL || clone_item_cb == NULL) { ++ return NULL; ++ } ++ ++ ptr = (common_array *)util_common_calloc_s(sizeof(common_array)); ++ if (ptr == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ ptr->items = (void **)util_smart_calloc_s(sizeof(void *), len); ++ if (ptr->items == NULL) { ++ ERROR("Out of memory"); ++ free(ptr); ++ return NULL; ++ } ++ ++ ptr->len = 0; ++ ptr->cap = len; ++ ptr->free_item_cb = free_item_cb; ++ ptr->clone_item_cb = clone_item_cb; ++ ++ return ptr; ++} ++ ++void util_free_common_array(common_array *ptr) ++{ ++ size_t i; ++ ++ if (ptr == NULL || ptr->free_item_cb == NULL) { ++ return; ++ } ++ ++ for (i = 0; i < ptr->len; i++) { ++ ptr->free_item_cb(ptr->items[i]); ++ ptr->items[i] = NULL; ++ } ++ free(ptr->items); ++ ptr->items = NULL; ++ ptr->len = 0; ++ ptr->cap = 0; ++ ptr->free_item_cb = NULL; ++ ptr->clone_item_cb = NULL; ++ ++ free(ptr); ++} ++ ++static bool do_expand_common_array(common_array *array) ++{ ++ size_t new_size = get_array_scale_size(array->cap); ++ void **new_items = NULL; ++ ++ // array capability sure less than MAX_MEMORY_SIZE ++ // so we need to check Overflow: ++ if (new_size == array->cap) { ++ ERROR("Too large common array, overflow memory"); ++ return false; ++ } ++ ++ // new_size * sizeof(*new_items) and list->len * sizeof(*list->items) ++ if (util_mem_realloc((void **)&new_items, new_size * sizeof(void *), (void *)array->items, ++ array->len * sizeof(void *)) != 0) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ array->items = new_items; ++ array->cap = new_size; ++ ++ return true; ++} ++ ++int util_append_common_array(common_array *arr, void *val) ++{ ++ if (arr == NULL || arr->clone_item_cb == NULL) { ++ ERROR("Invalid common array"); ++ return -1; ++ } ++ ++ if (val == NULL) { ++ DEBUG("Empty new item, just ignore it"); ++ return 0; ++ } ++ ++ if (arr->len < arr->cap) { ++ goto out; ++ } ++ ++ // expand common array ++ if (!do_expand_common_array(arr)) { ++ return -1; ++ } ++ ++out: ++ arr->items[arr->len] = arr->clone_item_cb(val); ++ arr->len += 1; ++ return 0; ++} ++ ++int util_merge_common_array(common_array *dest_arr, common_array *src_arr) ++{ ++ size_t i; ++ ++ if (dest_arr == NULL || dest_arr->clone_item_cb == NULL || ++ src_arr == NULL || src_arr->clone_item_cb == NULL) { ++ ERROR("Invalid common array"); ++ return -1; ++ } ++ ++ for (i = 0; i < src_arr->len; i++) { ++ if (util_append_common_array(dest_arr, src_arr->items[i]) != 0) { ++ ERROR("Failed to append element"); ++ return -1; ++ } ++ } ++ return 0; ++} +diff --git a/src/utils/cutils/utils_array.h b/src/utils/cutils/utils_array.h +index 1c084595..0c4fd217 100644 +--- a/src/utils/cutils/utils_array.h ++++ b/src/utils/cutils/utils_array.h +@@ -30,6 +30,8 @@ void util_free_array_by_len(char **array, size_t len); + + void util_free_array(char **array); + ++char **util_copy_array_by_len(char **array, size_t len); ++ + int util_grow_array(char ***orig_array, size_t *orig_capacity, size_t size, + size_t increment); + +@@ -52,6 +54,8 @@ void util_free_string_array(string_array *ptr); + + int util_append_string_array(string_array *sarr, const char *val); + ++string_array *util_copy_string_array(string_array *sarr); ++ + bool util_string_array_contain(const string_array *sarr, const char *elem); + + void util_free_sensitive_array(char **array); +@@ -63,6 +67,33 @@ define_auto_cleanup_callback(util_free_array, char *); + // define auto free macro for char * + #define __isula_auto_array_t auto_cleanup_tag(util_free_array) + ++define_auto_cleanup_callback(util_free_string_array, string_array); ++#define __isula_auto_string_array_t auto_cleanup_tag(util_free_string_array) ++ ++typedef void (*free_common_array_item_cb)(void *item); ++typedef void *(*clone_common_array_item_cb)(void *item); ++typedef struct common_array_t { ++ void **items; ++ size_t len; ++ size_t cap; ++ free_common_array_item_cb free_item_cb; ++ clone_common_array_item_cb clone_item_cb; ++} common_array; ++ ++void *util_clone_ptr(void *item); ++ ++common_array *util_common_array_new(size_t len, free_common_array_item_cb free_item_cb, ++ clone_common_array_item_cb clone_item_cb); ++ ++void util_free_common_array(common_array *ptr); ++ ++int util_append_common_array(common_array *arr, void *val); ++ ++int util_merge_common_array(common_array *dest_arr, common_array *src_arr); ++ ++define_auto_cleanup_callback(util_free_common_array, common_array); ++#define __isula_auto_common_array_t auto_cleanup_tag(util_free_common_array) ++ + #ifdef __cplusplus + } + #endif +-- +2.34.1 + diff --git a/0062-do-not-umount-shmpath-for-sandbox-container.patch b/0062-do-not-umount-shmpath-for-sandbox-container.patch new file mode 100644 index 0000000..8b719a3 --- /dev/null +++ b/0062-do-not-umount-shmpath-for-sandbox-container.patch @@ -0,0 +1,30 @@ +From 8e35525073b52b5d161984015de641bd21570380 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Tue, 16 Apr 2024 10:32:50 +0000 +Subject: [PATCH 62/69] do not umount shmpath for sandbox container + +Signed-off-by: jikai +--- + src/daemon/modules/service/service_container.c | 6 ++++++ + 1 file changed, 6 insertions(+) + +diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c +index eb7ce4f4..a2322309 100644 +--- a/src/daemon/modules/service/service_container.c ++++ b/src/daemon/modules/service/service_container.c +@@ -1718,6 +1718,12 @@ void umount_share_shm(container_t *cont) + return; + } + if (cont->hostconfig->ipc_mode == NULL || namespace_is_shareable(cont->hostconfig->ipc_mode)) { ++#ifdef ENABLE_CRI_API_V1 ++ // For sandbox in cri v1, the shm path is created and umounted in CRI ++ if (is_sandbox_container(cont->common_config->sandbox_info)) { ++ return; ++ } ++#endif + if (cont->common_config == NULL || cont->common_config->shm_path == NULL) { + return; + } +-- +2.34.1 + diff --git a/0063-remove-default-systemd-cgroup-and-enable-cri-v1-valu.patch b/0063-remove-default-systemd-cgroup-and-enable-cri-v1-valu.patch new file mode 100644 index 0000000..b6a0e6b --- /dev/null +++ b/0063-remove-default-systemd-cgroup-and-enable-cri-v1-valu.patch @@ -0,0 +1,27 @@ +From c092597565e5f24b29ecd83b4b371a19a9c2db0d Mon Sep 17 00:00:00 2001 +From: jikai +Date: Wed, 17 Apr 2024 01:52:45 +0000 +Subject: [PATCH 63/69] remove default systemd-cgroup and enable-cri-v1 value + in daemon.json + +Signed-off-by: jikai +--- + src/contrib/config/daemon.json | 4 +--- + 1 file changed, 1 insertion(+), 3 deletions(-) + +diff --git a/src/contrib/config/daemon.json b/src/contrib/config/daemon.json +index 69362c26..711dda94 100644 +--- a/src/contrib/config/daemon.json ++++ b/src/contrib/config/daemon.json +@@ -35,7 +35,5 @@ + "insecure-skip-verify-enforce": false, + "cri-runtimes": { + "kata": "io.containerd.kata.v2" +- }, +- "enable-cri-v1": false, +- "systemd-cgroup": false ++ } + } +-- +2.34.1 + diff --git a/0064-cdi-support-module-cache.patch b/0064-cdi-support-module-cache.patch new file mode 100644 index 0000000..4ed9fb5 --- /dev/null +++ b/0064-cdi-support-module-cache.patch @@ -0,0 +1,862 @@ +From f87f0fddaec4b3aea72f3b9c91f2dc91b89683ce Mon Sep 17 00:00:00 2001 +From: liuxu +Date: Tue, 16 Apr 2024 11:44:22 +0800 +Subject: [PATCH 64/69] cdi:support module cache + +--- + .../device/cdi/behavior/cdi_spec_dirs.h | 3 +- + src/daemon/modules/device/cdi/cdi_cache.c | 760 +++++++++++++++++- + src/daemon/modules/device/cdi/cdi_cache.h | 9 +- + 3 files changed, 747 insertions(+), 25 deletions(-) + +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h +index 73d8c0f5..eedcabad 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h +@@ -32,8 +32,7 @@ struct cdi_scan_fn_maps { + map_t *specs; + map_t *devices; + map_t *conflicts; +- map_t *spec_errors; +- string_array *result; ++ bool *refresh_error_flag; + }; + typedef void(*cdi_scan_spec_func)(struct cdi_scan_fn_maps *scan_fn_maps, const char *path, int priority, + struct cdi_cache_spec *spec, char *error); +diff --git a/src/daemon/modules/device/cdi/cdi_cache.c b/src/daemon/modules/device/cdi/cdi_cache.c +index cfc23a1c..37767855 100644 +--- a/src/daemon/modules/device/cdi/cdi_cache.c ++++ b/src/daemon/modules/device/cdi/cdi_cache.c +@@ -14,56 +14,784 @@ + ******************************************************************************/ + #include "cdi_cache.h" + ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "utils.h" ++#include "utils_file.h" ++#include "path.h" ++#include "error.h" ++#include "cdi_device.h" ++#include "cdi_spec.h" ++#include "cdi_spec_dirs.h" ++#include "cdi_container_edits.h" ++ ++// cache ++static int cdi_set_spec_dirs(struct cdi_cache *c, string_array *spec_dirs); ++static int configure(struct cdi_cache *c, string_array *spec_dirs); ++static int refresh(struct cdi_cache *c); ++static bool refresh_if_required(struct cdi_cache *c, bool force, int *ret); ++ ++// watch ++static void free_cdi_watch(struct cdi_watch *watch); ++static void watch_setup(struct cdi_watch *watch, string_array *dirs); ++static void watch_start(struct cdi_cache *c); ++static void watch_stop(struct cdi_watch *w); ++static void *watch_thread_func(void *arg); ++static bool watch_update(struct cdi_watch *w, const char *removed, int wd); ++ ++static int cdi_set_spec_dirs(struct cdi_cache *c, string_array *spec_dirs) ++{ ++ __isula_auto_string_array_t string_array *new_spec_dirs = NULL; ++ char clean_path[PATH_MAX] = { 0 }; ++ size_t i; ++ ++ if (c == NULL || spec_dirs == NULL) { ++ ERROR("Invalid argument"); ++ return -1; ++ } ++ if (spec_dirs->len == 0) { ++ return 0; ++ } ++ ++ new_spec_dirs = util_string_array_new(spec_dirs->len); ++ if (new_spec_dirs == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ for (i = 0; i < spec_dirs->len; i++) { ++ if (util_clean_path(spec_dirs->items[i], clean_path, sizeof(clean_path)) == NULL) { ++ ERROR("Failed to get clean path %s", spec_dirs->items[i]); ++ return -1; ++ } ++ if (util_append_string_array(new_spec_dirs, clean_path) != 0) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ } ++ util_free_string_array(c->spec_dirs); ++ c->spec_dirs = new_spec_dirs; ++ new_spec_dirs = NULL; ++ ++ return 0; ++} ++ + void free_cdi_cache(struct cdi_cache *c) + { +- (void)c; ++ if (c == NULL) { ++ return; ++ } ++ ++ util_free_string_array(c->spec_dirs); ++ c->spec_dirs = NULL; ++ map_free(c->specs); ++ c->specs = NULL; ++ map_free(c->devices); ++ c->devices = NULL; ++ free_cdi_watch(c->watch); ++ c->watch = NULL; ++ ++ free(c); + } + + struct cdi_cache *cdi_new_cache(string_array *spec_dirs) + { ++ struct cdi_cache *c = NULL; ++ int ret = 0; ++ ++ c = util_common_calloc_s(sizeof(*c)); ++ if (c == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ c->refresh_error_flag = false; ++ c->auto_refresh = true; ++ c->watch = util_common_calloc_s(sizeof(struct cdi_watch)); ++ if (c->watch == NULL) { ++ ERROR("Out of memory"); ++ goto free_out; ++ } ++ c->watch->watcher_fd = -1; ++ ++ if (cdi_set_spec_dirs(c, &g_default_spec_dirs) != 0) { ++ ERROR("Failed to set spec dirs by default"); ++ goto free_out; ++ } ++ ++ (void)pthread_mutex_lock(&c->mutex); ++ ret = configure(c, spec_dirs); ++ (void)pthread_mutex_unlock(&c->mutex); ++ if (ret != 0) { ++ ERROR("Failed to configure"); ++ goto free_out; ++ } ++ ++ return c; ++ ++free_out: ++ free_cdi_cache(c); + return NULL; + } + +-static int cdi_inject_devices(struct cdi_cache *c, oci_runtime_spec *oci_spec, string_array *devices) ++static int cdi_configure(struct cdi_cache *c, string_array *spec_dirs) + { +- return 0; ++ int ret = 0; ++ ++ if (c == NULL) { ++ ERROR("Invalid arguments"); ++ return -1; ++ } ++ ++ (void)pthread_mutex_lock(&c->mutex); ++ ret = configure(c, spec_dirs); ++ (void)pthread_mutex_unlock(&c->mutex); ++ ++ return ret; + } + +-static int cdi_configure(struct cdi_cache *c, string_array *spec_dirs) ++static int configure(struct cdi_cache *c, string_array *spec_dirs) + { ++ int ret = 0; ++ ++ if (spec_dirs != NULL) { ++ ret = cdi_set_spec_dirs(c, spec_dirs); ++ if (ret != 0) { ++ ERROR("Failed to apply cache spec dirs"); ++ return -1; ++ } ++ } ++ ++ watch_stop(c->watch); ++ if (c->auto_refresh) { ++ watch_setup(c->watch, c->spec_dirs); ++ watch_start(c); ++ } ++ (void)refresh(c); + return 0; + } + + static int cdi_refresh(struct cdi_cache *c) + { +- return 0; ++ bool refreshed; ++ int ret = 0; ++ ++ if (c == NULL) { ++ ERROR("Invalid arguments"); ++ return -1; ++ } ++ ++ (void)pthread_mutex_lock(&c->mutex); ++ refreshed = refresh_if_required(c, !c->auto_refresh, &ret); ++ if (refreshed) { ++ goto unlock_out; ++ } ++ ++ ret = c->refresh_error_flag ? -1 : 0; ++unlock_out: ++ (void)pthread_mutex_unlock(&c->mutex); ++ return ret; + } + +-static map_t *cdi_get_errors(struct cdi_cache *c) ++static void map_cdi_cache_specs_kvfree(void *key, void *value) + { +- return NULL; ++ free(key); ++ util_free_common_array((common_array *)value); + } + +-static string_array *cdi_get_spec_directories(struct cdi_cache *c) ++static void map_cdi_cache_device_kvfree(void *key, void *value) + { +- return NULL; ++ free(key); ++ free_cdi_cache_device((struct cdi_cache_device *)value); + } + +-static map_t *cdi_get_spec_dir_errors(struct cdi_cache *c) ++static void set_refresh_error_flag(bool *refresh_error_flag, const char *error, const char *path) + { +- return NULL; ++ *refresh_error_flag = true; ++ ERROR("Cdi refresh error: %s, spec %s", error, path); ++} ++ ++static bool resolve_conflict(struct cdi_scan_fn_maps *scan_fn_maps, const char *name, ++ struct cdi_cache_device *dev, struct cdi_cache_device *old) ++{ ++ map_t *conflicts = scan_fn_maps->conflicts; ++ bool *refresh_error_flag = scan_fn_maps->refresh_error_flag; ++ struct cdi_cache_spec *dev_spec = NULL; ++ struct cdi_cache_spec *old_spec = NULL; ++ int dev_prio; ++ int old_prio; ++ bool val = true; ++ const char *dev_path = NULL; ++ const char *old_path = NULL; ++ ++ dev_spec = cdi_device_get_spec(dev); ++ old_spec = cdi_device_get_spec(old); ++ dev_prio = cdi_spec_get_priority(dev_spec); ++ old_prio = cdi_spec_get_priority(old_spec); ++ if (dev_prio > old_prio) { ++ return false; ++ } else if (dev_prio == old_prio) { ++ dev_path = cdi_spec_get_path(dev_spec); ++ old_path = cdi_spec_get_path(old_spec); ++ *refresh_error_flag = true; ++ ERROR("Conflicting device %s (specs %s, %s)", name, dev_path, old_path); ++ if (!map_replace(conflicts, (void *)name, (void *)&val)) { ++ ERROR("Failed to insert bool to conflicts by name %s", name); ++ return true; ++ } ++ } else { ++ // do nothing ++ } ++ ++ return true; ++} ++ ++static void refresh_scan_spec_func(struct cdi_scan_fn_maps *scan_fn_maps, const char *path, ++ int priority, struct cdi_cache_spec *spec, char *error) ++{ ++ map_t *specs = scan_fn_maps->specs; ++ map_t *devices = scan_fn_maps->devices; ++ bool *refresh_error_flag = scan_fn_maps->refresh_error_flag; ++ char clean_path[PATH_MAX] = { 0 }; ++ __isula_auto_free char *tmp_error = NULL; ++ const char *vendor = NULL; ++ __isula_auto_common_array_t common_array *spec_array = NULL; ++ map_itor *itor = NULL; ++ __isula_auto_free char *qualified = NULL; ++ struct cdi_cache_device *dev = NULL; ++ struct cdi_cache_device *other = NULL; ++ ++ if (util_clean_path(path, clean_path, sizeof(clean_path)) == NULL) { ++ ERROR("Failed to get clean path %s", path); ++ format_errorf(&tmp_error, "Failed to get clean path %s", path); ++ return; ++ } ++ if (error != NULL) { ++ ERROR("Failed to load CDI Spec %s", error); ++ format_errorf(&tmp_error, "Failed to load CDI Spec %s", error); ++ goto error_out; ++ } ++ ++ vendor = cdi_spec_get_vendor(spec); ++ spec_array = map_search(specs, (void *)vendor); ++ if (spec_array == NULL) { ++ spec_array = util_common_array_new(1, (free_common_array_item_cb)free_cdi_cache_spec, util_clone_ptr); ++ if (spec_array == NULL) { ++ ERROR("Out of memory"); ++ tmp_error = util_strdup_s("Out of memory"); ++ goto error_out; ++ } ++ if (!map_insert(specs, (void *)vendor, spec_array)) { ++ ERROR("Failed to insert spec array to specs"); ++ tmp_error = util_strdup_s("Failed to insert spec array to specs"); ++ goto error_out; ++ } ++ } ++ if (util_append_common_array(spec_array, spec) != 0) { ++ ERROR("Failed to append spec"); ++ tmp_error = util_strdup_s("Failed to append spec"); ++ goto error_out; ++ } ++ spec_array = NULL; ++ ++ itor = map_itor_new(spec->devices); ++ if (itor == NULL) { ++ ERROR("Out of memory, create new map itor failed"); ++ tmp_error = util_strdup_s("Out of memory, create new map itor failed"); ++ goto error_out; ++ } ++ for (; map_itor_valid(itor); map_itor_next(itor)) { ++ dev = map_itor_value(itor); ++ qualified = cdi_device_get_qualified_name(dev); ++ other = map_search(devices, (void *)qualified); ++ if (other != NULL) { ++ if (resolve_conflict(scan_fn_maps, qualified, dev, other)) { ++ continue; ++ } ++ } ++ if (!map_replace(devices, (void *)qualified, dev)) { ++ ERROR("Failed to insert device to devices by name %s", qualified); ++ format_errorf(&tmp_error, "Failed to insert device to devices by name %s", qualified); ++ goto error_out; ++ } ++ free(qualified); ++ qualified = NULL; ++ } ++ goto out; ++ ++error_out: ++ set_refresh_error_flag(refresh_error_flag, tmp_error, path); ++out: ++ map_itor_free(itor); ++ return; ++} ++ ++static int refresh(struct cdi_cache *c) ++{ ++ int ret = 0; ++ __isula_auto_free char *error = NULL; ++ map_t *specs = NULL; ++ map_t *devices = NULL; ++ map_t *conflicts = NULL; ++ struct cdi_scan_fn_maps scan_fn_maps = { 0 }; ++ map_itor *itor = NULL; ++ char *conflict = NULL; ++ ++ c->refresh_error_flag = false; ++ specs = map_new(MAP_STR_PTR, MAP_DEFAULT_CMP_FUNC, map_cdi_cache_specs_kvfree); ++ if (specs == NULL) { ++ ERROR("Out of memory"); ++ ret = -1; ++ goto free_out; ++ } ++ devices = map_new(MAP_STR_PTR, MAP_DEFAULT_CMP_FUNC, map_cdi_cache_device_kvfree); ++ if (devices == NULL) { ++ ERROR("Out of memory"); ++ ret = -1; ++ goto free_out; ++ } ++ conflicts = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ if (conflicts == NULL) { ++ ERROR("Out of memory"); ++ ret = -1; ++ goto free_out; ++ } ++ ++ scan_fn_maps.specs = specs; ++ scan_fn_maps.devices = devices; ++ scan_fn_maps.conflicts = conflicts; ++ scan_fn_maps.refresh_error_flag = &c->refresh_error_flag; ++ // ignore error when scan spec dirs ++ (void)cdi_scan_spec_dirs(c->spec_dirs, &scan_fn_maps, refresh_scan_spec_func); ++ ++ itor = map_itor_new(conflicts); ++ if (itor == NULL) { ++ ERROR("Out of memory, create new map itor failed"); ++ ret = -1; ++ goto free_out; ++ } ++ for (; map_itor_valid(itor); map_itor_next(itor)) { ++ conflict = map_itor_key(itor); ++ if ((map_search(devices, conflict) != NULL) && ++ !map_remove(devices, conflict)) { ++ ERROR("Failed to remove conflict device from devices"); ++ ret = -1; ++ goto free_out; ++ } ++ } ++ ++ util_swap_ptr((void **)&c->specs, (void **)&specs); ++ util_swap_ptr((void **)&c->devices, (void **)&devices); ++ ++ ret = c->refresh_error_flag ? -1 : 0; ++ ++free_out: ++ map_itor_free(itor); ++ map_free(specs); ++ map_free(devices); ++ map_free(conflicts); ++ return ret; ++} ++ ++static bool refresh_if_required(struct cdi_cache *c, bool force, int *ret) ++{ ++ if (force || (c->auto_refresh && watch_update(c->watch, NULL, -1))) { ++ *ret = refresh(c); ++ return true; ++ } ++ return false; ++} ++ ++static void map_spec_ptr_kvfree(void *key, void *value) ++{ ++ // do not need free spec* ++ (void)key; ++ free(value); ++} ++ ++static int cdi_inject_devices(struct cdi_cache *c, oci_runtime_spec *oci_spec, string_array *devices) ++{ ++ int ret = 0; ++ __isula_auto_string_array_t string_array *unresolved = NULL; ++ cdi_container_edits *edits = NULL; ++ map_t *specs = NULL; ++ size_t i; ++ const char *device = NULL; ++ struct cdi_cache_device *d = NULL; ++ int tmp_val = 0; ++ __isula_auto_free char *unresolved_str = NULL; ++ ++ if (c == NULL || devices == NULL) { ++ ERROR("Can't inject devices"); ++ return -1; ++ } ++ if (oci_spec == NULL) { ++ ERROR("Can't inject devices, nil OCI Spec"); ++ return -1; ++ } ++ ++ unresolved = util_common_calloc_s(sizeof(*unresolved)); ++ if (unresolved == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ specs = map_new(MAP_PTR_INT, MAP_DEFAULT_CMP_FUNC, map_spec_ptr_kvfree); ++ if (specs == NULL) { ++ ERROR("Out of memory"); ++ ret = -1; ++ goto out; ++ } ++ edits = util_common_calloc_s(sizeof(*edits)); ++ if (edits == NULL) { ++ ERROR("Out of memory"); ++ ret = -1; ++ goto out; ++ } ++ ++ (void)pthread_mutex_lock(&c->mutex); ++ ++ (void)refresh_if_required(c, false, &ret); ++ ++ for(i = 0; i < devices->len; i++) { ++ device = devices->items[i]; ++ d = map_search(c->devices, (void *)device); ++ if (d == NULL) { ++ if (util_append_string_array(unresolved, device) != 0) { ++ ERROR("Out of memory"); ++ ret = -1; ++ goto out; ++ } ++ continue; ++ } ++ if (map_search(specs, (void *)cdi_device_get_spec(d)) == NULL) { ++ if (!map_insert(specs, (void *)cdi_device_get_spec(d), (void *)&tmp_val)) { ++ ERROR("Failed to insert spec ptr to specs when find device %s", device); ++ ret = -1; ++ goto out; ++ } ++ if (cdi_container_edits_append(edits, cdi_spec_get_edits(cdi_device_get_spec(d))) != 0) { ++ ERROR("Failed to append edits when find device %s", device); ++ ret = -1; ++ goto out; ++ } ++ } ++ if (cdi_container_edits_append(edits, cdi_device_get_edits(d)) != 0) { ++ ERROR("Failed to append edits when find device %s", device); ++ ret = -1; ++ goto out; ++ } ++ } ++ ++ if (unresolved->len != 0) { ++ unresolved_str = util_string_join(", ", (const char **)unresolved->items, unresolved->len); ++ ERROR("Unresolvable CDI devices %s", unresolved_str); ++ ret = -1; ++ goto out; ++ } ++ ++ ret = cdi_container_edits_apply(edits, oci_spec); ++ if (ret != 0) { ++ ERROR("Failed to apply edits when inject devices"); ++ ret = -1; ++ } ++ ++out: ++ (void)pthread_mutex_unlock(&c->mutex); ++ map_free(specs); ++ free_cdi_container_edits(edits); ++ return ret; + } + + static struct cdi_cache_ops g_cdi_cache_ops = { + .inject_devices = cdi_inject_devices, + .configure = cdi_configure, +- .refresh = cdi_refresh, +- .get_errors = cdi_get_errors, +- .get_spec_directories = cdi_get_spec_directories, +- .get_spec_dir_errors = cdi_get_spec_dir_errors ++ .refresh = cdi_refresh + }; + + struct cdi_cache_ops *cdi_get_cache_ops(void) + { + return &g_cdi_cache_ops; +-} +\ No newline at end of file ++} ++ ++static void free_cdi_watch(struct cdi_watch *w) ++{ ++ if (w == NULL) { ++ return; ++ } ++ ++ watch_stop(w); ++ free(w); ++} ++ ++static int init_tracked(struct cdi_watch *w, string_array *dirs) ++{ ++ size_t i; ++ bool tmp_value = false; ++ ++ w->tracked = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ if (w->tracked == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ for(i = 0; i < dirs->len; i++) { ++ if (!map_replace(w->tracked, (void *)dirs->items[i], (void *)&tmp_value)) { ++ ERROR("Failed to insert tracked by dir %s", dirs->items[i]); ++ goto error_out; ++ } ++ } ++ w->wd_dirs = map_new(MAP_INT_STR, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ if (w->wd_dirs == NULL) { ++ ERROR("Out of memory"); ++ goto error_out; ++ } ++ ++ return 0; ++ ++error_out: ++ map_free(w->tracked); ++ w->tracked = NULL; ++ return -1; ++} ++ ++static void watch_setup(struct cdi_watch *w, string_array *dirs) ++{ ++ __isula_auto_free char *error = NULL; ++ ++ if (w == NULL || dirs == NULL || dirs->len == 0) { ++ ERROR("Invalid param"); ++ return; ++ } ++ ++ if (init_tracked(w, dirs) != 0) { ++ ERROR("Failed to initialize tracked"); ++ return; ++ } ++ ++ w->watcher_fd = inotify_init(); ++ if (w->watcher_fd < 0) { ++ ERROR("Failed to initialize inotify fd"); ++ map_free(w->tracked); ++ w->tracked = NULL; ++ map_free(w->wd_dirs); ++ w->wd_dirs = NULL; ++ return; ++ } ++ ++ (void)watch_update(w, NULL, -1); ++} ++ ++static void watch_start(struct cdi_cache *c) ++{ ++ pthread_t thread = 0; ++ int ret = 0; ++ ++ ret = pthread_create(&thread, NULL, watch_thread_func, c); ++ if (ret != 0) { ++ ERROR("Cdi watch thread create failed"); ++ return; ++ } ++} ++ ++static void watch_stop(struct cdi_watch *w) ++{ ++ if (w == NULL) { ++ return; ++ } ++ ++ if (w->watcher_fd >= 0) { ++ close(w->watcher_fd); ++ w->watcher_fd = -1; ++ } ++ map_free(w->tracked); ++ w->tracked = NULL; ++ map_free(w->wd_dirs); ++ w->wd_dirs = NULL; ++} ++ ++// wait_events wait until inotify ++static int wait_events(int watcher_fd) ++{ ++ fd_set rfds; ++ FD_ZERO(&rfds); ++ FD_SET(watcher_fd, &rfds); ++ return select(FD_SETSIZE, &rfds, NULL, NULL, NULL); ++} ++ ++#define CDI_WATCH_EVENTS (IN_MOVED_TO | IN_MOVED_FROM | IN_DELETE | IN_MODIFY | IN_MOVE_SELF | IN_DELETE_SELF) ++ ++static int process_cdi_events(int watcher_fd, struct cdi_cache *c) ++{ ++ ssize_t events_length = 0; ++ ssize_t events_index = 0; ++ struct inotify_event *cdi_event = NULL; ++ char buffer[MAXLINE] __attribute__((aligned(__alignof__(struct inotify_event)))) = { 0 }; ++ int update_cnt = 0; ++ __isula_auto_free char *event_dir = NULL; ++ ++ events_length = util_read_nointr(watcher_fd, buffer, sizeof(buffer)); ++ if (events_length <= 0) { ++ ERROR("Failed to wait events"); ++ return -1; ++ } ++ ++ (void)pthread_mutex_lock(&c->mutex); ++ ++ while (events_index < events_length) { ++ cdi_event = (struct inotify_event *)(&buffer[events_index]); ++ ssize_t event_size = (ssize_t)(cdi_event->len) + (ssize_t)offsetof(struct inotify_event, name); ++ if (event_size == 0 || event_size > (events_length - events_index)) { ++ break; ++ } ++ events_index += event_size; ++ ++ /* ++ * file: ++ * Rename: mask == IN_MOVED_TO | IN_MOVED_FROM ++ * Remove: mask == IN_MOVED_FROM || mask == IN_DELETE ++ * Write: mask == IN_MODIFY ++ * dir: ++ * Remove: mask == IN_MOVE_SELF || mask == IN_DELETE_SELF ++ */ ++ if ((cdi_event->mask & CDI_WATCH_EVENTS) == 0) { ++ continue; ++ } ++ DEBUG("Cdi spec file %s is changed", cdi_event->name); ++ if (cdi_event->mask == IN_MODIFY) { ++ if (!util_has_suffix(cdi_event->name, ".json")) { ++ WARN("Invalid spec %s ext", cdi_event->name); ++ continue; ++ } ++ } ++ event_dir = util_strdup_s(map_search(c->watch->wd_dirs, &(cdi_event->wd))); ++ if (!(cdi_event->mask == IN_DELETE_SELF || cdi_event->mask == IN_MOVE_SELF)) { ++ free(event_dir); ++ event_dir = NULL; ++ } ++ watch_update(c->watch, event_dir, cdi_event->wd); ++ update_cnt++; ++ } ++ if (update_cnt > 0) { ++ (void)refresh(c); ++ } ++ ++ (void)pthread_mutex_unlock(&c->mutex); ++ return 0; ++} ++ ++// Watch Spec directory changes, triggering a refresh if necessary. ++static void *watch_thread_func(void *arg) ++{ ++ struct cdi_cache *c = (struct cdi_cache *)arg; ++ int errcode = 0; ++ int watcher_fd = -1; ++ ++ errcode = pthread_detach(pthread_self()); ++ if (errcode != 0) { ++ errno = errcode; ++ SYSERROR("Detach thread failed"); ++ return NULL; ++ } ++ ++ prctl(PR_SET_NAME, "cdi-watcher"); ++ ++ watcher_fd = c->watch->watcher_fd; ++ if (watcher_fd < 0) { ++ ERROR("Invalid inotify fd"); ++ return NULL; ++ } ++ ++ for (;;) { ++ if (wait_events(watcher_fd) < 0) { ++ ERROR("Failed to wait events"); ++ break; ++ } ++ if (process_cdi_events(watcher_fd, c) != 0) { ++ break; ++ } ++ } ++ return NULL; ++} ++ ++static void update_remove_watch_dir(struct cdi_watch *w, const char *dir, int wd) ++{ ++ bool tmp_value = false; ++ if (wd >= 0) { ++ (void)inotify_rm_watch(w->watcher_fd, wd); ++ if ((map_search(w->wd_dirs, &wd) != NULL) && ++ !map_remove(w->wd_dirs, &wd)) { ++ ERROR("Failed to remove watch fd of %s", dir); ++ } ++ } ++ if (!map_replace(w->tracked, (void *)dir, (void *)&tmp_value)) { ++ ERROR("Failed to insert tracked by dir %s", dir); ++ } ++} ++ ++static void update_add_watch_dir(struct cdi_watch *w, const char *dir, bool *update) ++{ ++ int wd = -1; ++ bool tmp_value = true; ++ __isula_auto_free char *error = NULL; ++ ++ wd = inotify_add_watch(w->watcher_fd, dir, CDI_WATCH_EVENTS); ++ if (wd < 0) { ++ if (errno == ENOENT) { ++ SYSINFO("Watch device dir %s", dir); ++ } else { ++ SYSERROR("Failed to watch device dir %s", dir); ++ } ++ return; ++ } else { ++ DEBUG("Watching %s for device disovery", dir); ++ tmp_value = true; ++ if (!map_replace(w->tracked, (void *)dir, (void *)&tmp_value)) { ++ ERROR("Failed to insert tracked by dir %s", dir); ++ goto error_out; ++ } ++ if (!map_replace(w->wd_dirs, (void *)&wd, (void *)dir)) { ++ ERROR("Failed to insert dir %s by wd", dir); ++ goto error_out; ++ } ++ *update = true; ++ } ++ return; ++ ++error_out: ++ update_remove_watch_dir(w, dir, wd); ++} ++ ++static bool watch_update(struct cdi_watch *w, const char *removed, int wd) ++{ ++ const char *dir = NULL; ++ bool *ok = NULL; ++ bool update = false; ++ map_itor *itor = NULL; ++ __isula_auto_free char *error = NULL; ++ ++ itor = map_itor_new(w->tracked); ++ if (itor == NULL) { ++ ERROR("Out of memory, create new map itor failed"); ++ return false; ++ } ++ for (; map_itor_valid(itor); map_itor_next(itor)) { ++ dir = map_itor_key(itor); ++ ok = map_itor_value(itor); ++ if (ok == NULL || *ok) { ++ continue; ++ } ++ update_add_watch_dir(w, dir, &update); ++ } ++ ++ if (removed != NULL) { ++ update_remove_watch_dir(w, removed, wd); ++ WARN("Directory removed: %s", removed); ++ update = true; ++ } ++ ++ map_itor_free(itor); ++ return update; ++} +diff --git a/src/daemon/modules/device/cdi/cdi_cache.h b/src/daemon/modules/device/cdi/cdi_cache.h +index 34c27471..da315de2 100644 +--- a/src/daemon/modules/device/cdi/cdi_cache.h ++++ b/src/daemon/modules/device/cdi/cdi_cache.h +@@ -39,9 +39,6 @@ struct cdi_cache_ops { + // Refresher + int (*configure)(struct cdi_cache *c, string_array *spec_dirs); + int (*refresh)(struct cdi_cache *c); +- map_t *(*get_errors)(struct cdi_cache *c); +- string_array *(*get_spec_directories)(struct cdi_cache *c); +- map_t *(*get_spec_dir_errors)(struct cdi_cache *c); + }; + + struct cdi_watch { +@@ -54,11 +51,9 @@ struct cdi_watch { + struct cdi_cache { + pthread_mutex_t mutex; + string_array *spec_dirs; // cdi-spec-dirs will scan for CDI Spec files +- map_t *specs; // MAP_STR_PTR specs[vendor] = cdi_cache_spec** ++ map_t *specs; // MAP_STR_PTR specs[vendor] = common_array of cdi_cache_spec* + map_t *devices; // MAP_STR_PTR devices[cdi_device.name] = cdi_cache_device* +- map_t *errors; // MAP_STR_PTR errors[cdi_cache_spec.path] = string_array *errors +- map_t *dir_errors; // MAP_STR_STR dir_errors[spec_dirs[i]] = error +- ++ bool refresh_error_flag; + bool auto_refresh; + struct cdi_watch *watch; + }; +-- +2.34.1 + diff --git a/0065-change-default-subscribe-timeout-to-5min.patch b/0065-change-default-subscribe-timeout-to-5min.patch new file mode 100644 index 0000000..7e4d067 --- /dev/null +++ b/0065-change-default-subscribe-timeout-to-5min.patch @@ -0,0 +1,76 @@ +From 5c89c23f5e0de06a17a9263114430674221a1ee0 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Wed, 17 Apr 2024 06:59:08 +0000 +Subject: [PATCH 65/69] change default subscribe timeout to 5min + +Signed-off-by: jikai +--- + src/daemon/mailbox/message_queue.c | 4 ++-- + src/utils/cutils/blocking_queue.c | 13 +++++++------ + src/utils/cutils/blocking_queue.h | 2 +- + 3 files changed, 10 insertions(+), 9 deletions(-) + +diff --git a/src/daemon/mailbox/message_queue.c b/src/daemon/mailbox/message_queue.c +index 7e53301e..699ea0bc 100644 +--- a/src/daemon/mailbox/message_queue.c ++++ b/src/daemon/mailbox/message_queue.c +@@ -20,8 +20,8 @@ + + #include "utils.h" + +-// default set subscriber timeout to 1000ms, maybe could be configured later +-const int64_t subscribe_timeout = 1000; ++// default set subscriber timeout to 300s, maybe could be configured later ++const int64_t subscribe_timeout = 300; + + static void message_queue_subscriber_free(void *key, void *val) + { +diff --git a/src/utils/cutils/blocking_queue.c b/src/utils/cutils/blocking_queue.c +index 7c9c5f50..9bdb2ca3 100644 +--- a/src/utils/cutils/blocking_queue.c ++++ b/src/utils/cutils/blocking_queue.c +@@ -55,12 +55,11 @@ blocking_queue *blocking_queue_create(int64_t timeout, void (*release)(void *)) + queue->release = release; + + if (timeout >= 0) { +- queue->timeout.tv_sec = timeout / (Time_Second / Time_Milli); +- queue->timeout.tv_nsec = (timeout % (Time_Second / Time_Milli) ) * Time_Milli; ++ queue->timeout = timeout; + } else { +- queue->timeout.tv_sec = -1; ++ queue->timeout = -1; + } +- ++ + return isula_transfer_ptr(queue); + } + +@@ -112,8 +111,10 @@ int blocking_queue_pop(blocking_queue *queue, void **data) { + lock = &queue->lock; + + while (queue->head->next == NULL) { +- if (queue->timeout.tv_sec >= 0) { +- int ret = pthread_cond_timedwait(&queue->not_empty, &queue->lock, &queue->timeout); ++ if (queue->timeout >= 0) { ++ struct timespec timeout = { 0 }; ++ timeout.tv_sec = queue->timeout + time(NULL); ++ int ret = pthread_cond_timedwait(&queue->not_empty, &queue->lock, &timeout); + if (ret != 0) { + if (ret != ETIMEDOUT) { + ERROR("Failed to wait cond"); +diff --git a/src/utils/cutils/blocking_queue.h b/src/utils/cutils/blocking_queue.h +index 1c52a9d3..257779c3 100644 +--- a/src/utils/cutils/blocking_queue.h ++++ b/src/utils/cutils/blocking_queue.h +@@ -37,7 +37,7 @@ typedef struct blocking_queue { + blocking_node *head; + blocking_node *tail; + pthread_mutex_t lock; +- struct timespec timeout; ++ int64_t timeout; + pthread_cond_t not_empty; + void (*release)(void *); + } blocking_queue; +-- +2.34.1 + diff --git a/0066-cdi-support-modules-version-spec-spec_dirs-device.patch b/0066-cdi-support-modules-version-spec-spec_dirs-device.patch new file mode 100644 index 0000000..6aff31b --- /dev/null +++ b/0066-cdi-support-modules-version-spec-spec_dirs-device.patch @@ -0,0 +1,1254 @@ +From 684f0aa648061a13e777153e297a7064ba33ea17 Mon Sep 17 00:00:00 2001 +From: liuxu +Date: Thu, 11 Apr 2024 11:00:44 +0800 +Subject: [PATCH 66/69] cdi:support modules version/spec/spec_dirs/device + +--- + .../modules/device/cdi/behavior/cdi_device.c | 100 +++++++- + .../modules/device/cdi/behavior/cdi_device.h | 12 +- + .../modules/device/cdi/behavior/cdi_spec.c | 220 ++++++++++++++++-- + .../modules/device/cdi/behavior/cdi_spec.h | 16 +- + .../device/cdi/behavior/cdi_spec_dirs.c | 80 +++++++ + .../device/cdi/behavior/cdi_spec_dirs.h | 2 +- + .../modules/device/cdi/behavior/cdi_version.c | 173 ++++++++++++-- + .../modules/device/cdi/behavior/cdi_version.h | 1 - + src/daemon/modules/device/cdi/cdi_cache.c | 37 +-- + src/daemon/modules/device/cdi/cdi_cache.h | 1 + + .../network/cni_operator/libcni/libcni_api.c | 98 +------- + src/utils/cutils/utils_version.c | 147 ++++++++++++ + src/utils/cutils/utils_version.h | 36 +++ + 13 files changed, 748 insertions(+), 175 deletions(-) + create mode 100644 src/utils/cutils/utils_version.c + create mode 100644 src/utils/cutils/utils_version.h + +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_device.c b/src/daemon/modules/device/cdi/behavior/cdi_device.c +index 0fef8f42..aec3d7c0 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_device.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_device.c +@@ -14,27 +14,109 @@ + ******************************************************************************/ + #include "cdi_device.h" + ++#include ++#include ++ ++#include "error.h" ++#include "cdi_parser.h" ++#include "cdi_spec.h" ++ ++static int cdi_device_validate(struct cdi_cache_device *d); ++ + void free_cdi_cache_device(struct cdi_cache_device *d) + { +- (void)d; ++ if (d == NULL) { ++ return; ++ } ++ ++ /* ++ * free_cdi_cache_device should not be recursively free raw_device. ++ * Otherwise, the function conflicts with the raw_spec free raw_device ++ * when cdi_cache_spec free raw_spec, triggering double free. ++ */ ++ d->raw_device = NULL; ++ ++ /* ++ * free_cdi_cache_device should not be recursively free cache_spec. ++ * Otherwise, the function conflicts with the cache free specs, ++ * triggering double free. ++ */ ++ d->cache_spec = NULL; ++ ++ free(d); ++} ++ ++struct cdi_cache_device *cdi_device_new_device(struct cdi_cache_spec *spec, cdi_device *d) ++{ ++ struct cdi_cache_device *dev = NULL; ++ ++ if (spec == NULL || d == NULL) { ++ ERROR("Invalid params"); ++ return NULL; ++ } ++ ++ dev = util_common_calloc_s(sizeof(*dev)); ++ if (dev == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ dev->raw_device = d; ++ dev->cache_spec = spec; ++ ++ if (cdi_device_validate(dev) != 0) { ++ free_cdi_cache_device(dev); ++ return NULL; ++ } ++ return dev; + } + +-struct cdi_cache_device *cdi_device_new_device(struct cdi_cache_spec *spec, cdi_device *d, char **error) ++const struct cdi_cache_spec *cdi_device_get_spec(const struct cdi_cache_device *d) + { +- return NULL; ++ if (d == NULL) { ++ ERROR("Invalid params"); ++ return NULL; ++ } ++ return d->cache_spec; + } + +-struct cdi_cache_spec *cdi_device_get_spec(struct cdi_cache_device *d) ++char *cdi_device_get_qualified_name(const struct cdi_cache_device *d) + { +- return NULL; ++ if (d == NULL || d->raw_device == NULL) { ++ ERROR("Invalid params"); ++ return NULL; ++ } ++ return cdi_parser_qualified_name(cdi_spec_get_vendor(d->cache_spec), ++ cdi_spec_get_class(d->cache_spec), d->raw_device->name); + } + +-char *cdi_device_get_qualified_name(struct cdi_cache_device *d) ++cdi_container_edits *cdi_device_get_edits(const struct cdi_cache_device *d) + { +- return NULL; ++ if (d == NULL || d->raw_device == NULL) { ++ ERROR("Invalid params"); ++ return NULL; ++ } ++ return d->raw_device->container_edits; + } + +-cdi_container_edits *cdi_device_get_edits(struct cdi_cache_device *d) ++static int cdi_device_validate(struct cdi_cache_device *d) + { +- return NULL; ++ cdi_container_edits *edits = NULL; ++ ++ if (cdi_parser_validate_device_name(d->raw_device->name) != 0) { ++ ERROR("Failed to validate device name"); ++ return -1; ++ } ++ ++ // ignore validate annotations ++ ++ edits = cdi_device_get_edits(d); ++ if (cdi_container_edits_is_empty(edits)) { ++ ERROR("Invalid device, empty device edits"); ++ return -1; ++ } ++ if (cdi_container_edits_validate(edits) != 0) { ++ ERROR("Invalid device %s", d->raw_device->name); ++ return -1; ++ } ++ return 0; + } +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_device.h b/src/daemon/modules/device/cdi/behavior/cdi_device.h +index 5d63a576..9b0a5eab 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_device.h ++++ b/src/daemon/modules/device/cdi/behavior/cdi_device.h +@@ -28,16 +28,16 @@ extern "C" { + struct cdi_cache_spec; + + struct cdi_cache_device { +- cdi_device *raw_device; +- struct cdi_cache_spec *cache_spec; ++ const cdi_device *raw_device; ++ const struct cdi_cache_spec *cache_spec; + }; + + void free_cdi_cache_device(struct cdi_cache_device *d); + +-struct cdi_cache_device *cdi_device_new_device(struct cdi_cache_spec *spec, cdi_device *d, char **error); +-struct cdi_cache_spec *cdi_device_get_spec(struct cdi_cache_device *d); +-char *cdi_device_get_qualified_name(struct cdi_cache_device *d); +-cdi_container_edits *cdi_device_get_edits(struct cdi_cache_device *d); ++struct cdi_cache_device *cdi_device_new_device(struct cdi_cache_spec *spec, cdi_device *d); ++const struct cdi_cache_spec *cdi_device_get_spec(const struct cdi_cache_device *d); ++char *cdi_device_get_qualified_name(const struct cdi_cache_device *d); ++cdi_container_edits *cdi_device_get_edits(const struct cdi_cache_device *d); + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec.c b/src/daemon/modules/device/cdi/behavior/cdi_spec.c +index f79b5a44..235b1863 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_spec.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec.c +@@ -14,47 +14,235 @@ + ******************************************************************************/ + #include "cdi_spec.h" + ++#include ++#include ++#include ++ ++#include "utils.h" ++#include "utils_version.h" ++#include "error.h" ++#include "path.h" ++#include "cdi_version.h" ++#include "cdi_parser.h" ++#include "cdi_device.h" ++ ++static int cdi_spec_init(struct cdi_cache_spec *s); ++ + void free_cdi_cache_spec(struct cdi_cache_spec *s) + { +- (void)s; ++ if (s == NULL) { ++ return; ++ } ++ ++ free_cdi_spec(s->raw_spec); ++ s->raw_spec = NULL; ++ free(s->vendor); ++ s->vendor = NULL; ++ free(s->class); ++ s->class = NULL; ++ free(s->path); ++ s->path = NULL; ++ map_free(s->devices); ++ s->devices = NULL; ++ ++ free(s); + } + +-struct cdi_cache_spec *cdi_spec_read_spec(const char *path, int priority, char **error) ++struct cdi_cache_spec *cdi_spec_read_spec(const char *path, int priority) + { +- return NULL; ++ cdi_spec *raw_spec = NULL; ++ __isula_auto_free parser_error err = NULL; ++ char cleanpath[PATH_MAX] = { 0 }; ++ ++ if (util_clean_path(path, cleanpath, sizeof(cleanpath)) == NULL) { ++ ERROR("Failed to get clean path %s", path); ++ return NULL; ++ } ++ ++ raw_spec = cdi_spec_parse_file(cleanpath, NULL, &err); ++ if (raw_spec == NULL) { ++ ERROR("Failed to read CDI Spec %s: %s", cleanpath, err); ++ return NULL; ++ } ++ DEBUG("Read cdi spec %s", cleanpath); ++ ++ return cdi_spec_new_spec(raw_spec, cleanpath, priority); + } + +-struct cdi_cache_spec *cdi_spec_new_spec(cdi_spec *raw, const char *path, int priority, char **error) ++struct cdi_cache_spec *cdi_spec_new_spec(cdi_spec *raw, const char *path, int priority) + { ++ struct cdi_cache_spec *spec = NULL; ++ __isula_auto_free char *checked_path = NULL; ++ ++ if (raw == NULL) { ++ ERROR("Invalid param"); ++ return NULL; ++ } ++ ++ if (!util_has_suffix(path, ".json")) { ++ checked_path = util_string_append(path, CDI_DEFAULT_SPEC_EXT); ++ if (checked_path == NULL) { ++ ERROR("Failed to append %s to path %s", CDI_DEFAULT_SPEC_EXT, path); ++ return NULL; ++ } ++ } else { ++ checked_path = util_strdup_s(path); ++ } ++ spec = util_common_calloc_s(sizeof(*spec)); ++ if (spec == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ spec->raw_spec = raw; ++ spec->path = checked_path; ++ checked_path = NULL; ++ spec->priority = priority; ++ ++ if (cdi_parser_parse_qualifier(raw->kind, &spec->vendor, &spec->class) != 0) { ++ ERROR("Failed to parse kind %s", raw->kind); ++ goto error_out; ++ } ++ if (cdi_spec_init(spec) != 0) { ++ ERROR("Invalid CDI Spec"); ++ goto error_out; ++ } ++ ++ return spec; ++ ++error_out: ++ free_cdi_cache_spec(spec); + return NULL; + } + +-const char *cdi_spec_get_vendor(struct cdi_cache_spec *s) ++const char *cdi_spec_get_vendor(const struct cdi_cache_spec *s) + { +- return NULL; ++ if (s == NULL) { ++ ERROR("Invalid params"); ++ return NULL; ++ } ++ return s->vendor; + } + +-const char *cdi_spec_get_class(struct cdi_cache_spec *s) ++const char *cdi_spec_get_class(const struct cdi_cache_spec *s) + { +- return NULL; ++ if (s == NULL) { ++ ERROR("Invalid params"); ++ return NULL; ++ } ++ return s->class; + } + +-struct cdi_cache_device *cdi_spec_get_cache_device(struct cdi_cache_spec *s, const char *name) ++struct cdi_cache_device *cdi_spec_get_cache_device(const struct cdi_cache_spec *s, const char *name) + { +- return NULL; ++ if (s == NULL) { ++ ERROR("Invalid params"); ++ return NULL; ++ } ++ return map_search(s->devices, (void *)name);; + } + +-const char *cdi_spec_get_path(struct cdi_cache_spec *s) ++const char *cdi_spec_get_path(const struct cdi_cache_spec *s) + { +- return NULL; ++ if (s == NULL) { ++ ERROR("Invalid params"); ++ return NULL; ++ } ++ return s->path; + } + +-int cdi_spec_get_priority(struct cdi_cache_spec *s) ++int cdi_spec_get_priority(const struct cdi_cache_spec *s) + { +- return 0; ++ if (s == NULL) { ++ ERROR("Invalid params"); ++ return -1; ++ } ++ return s->priority; + } + +-cdi_container_edits *cdi_spec_get_edits(struct cdi_cache_spec *s) ++cdi_container_edits *cdi_spec_get_edits(const struct cdi_cache_spec *s) + { +- return NULL; ++ if (s == NULL || s->raw_spec == NULL) { ++ ERROR("Invalid params"); ++ return NULL; ++ } ++ return s->raw_spec->container_edits; ++} ++ ++static void map_cdi_cache_device_kvfree(void *key, void *value) ++{ ++ free(key); ++ free_cdi_cache_device((struct cdi_cache_device *)value); ++} ++ ++static int cdi_spec_init(struct cdi_cache_spec *s) ++{ ++ const char *min_version = NULL; ++ __isula_auto_free char *spec_version = NULL; ++ cdi_container_edits *edits = NULL; ++ struct cdi_cache_device *dev = NULL; ++ cdi_device *d = NULL; ++ size_t i; ++ bool version_result = true; ++ ++ if (!cdi_is_valid_version(s->raw_spec->cdi_version)) { ++ ERROR("Failed to validate cdi spec version: %s", s->raw_spec->cdi_version); ++ return -1; ++ } ++ ++ min_version = cdi_minimum_required_version(s->raw_spec); ++ if (min_version == NULL) { ++ ERROR("Could not determine minimum required version"); ++ return -1; ++ } ++ if (util_version_greater_than(min_version, s->raw_spec->cdi_version, &version_result) != 0) { ++ ERROR("Failed to compare version %s and %s", min_version, s->raw_spec->cdi_version); ++ return -1; ++ } ++ if (version_result) { ++ ERROR("The spec version must be at least v%s", min_version); ++ return -1; ++ } ++ ++ if (cdi_parser_validate_vendor_name(s->vendor) != 0) { ++ return -1; ++ } ++ if (cdi_parser_validate_class_name(s->class) != 0) { ++ return -1; ++ } ++ ++ // ignore validate annotations ++ ++ edits = cdi_spec_get_edits(s); ++ if (cdi_container_edits_validate(edits) != 0) { ++ return -1; ++ } ++ ++ s->devices = map_new(MAP_STR_PTR, MAP_DEFAULT_CMP_FUNC, map_cdi_cache_device_kvfree); ++ if (s->devices == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ for (i = 0; i < s->raw_spec->devices_len; i++) { ++ d = s->raw_spec->devices[i]; ++ dev = cdi_device_new_device(s, d); ++ if (dev == NULL) { ++ ERROR("Could not determine minimum required version"); ++ goto error_out; ++ } ++ if (map_search(s->devices, (void *)d->name) != NULL) { ++ ERROR("Invalid spec, multiple device %s", d->name); ++ goto error_out; ++ } ++ if (!map_insert(s->devices, (void *)d->name, dev)) { ++ ERROR("Failed to insert device %s", d->name); ++ goto error_out; ++ } ++ } ++ ++ return 0; ++ ++error_out: ++ map_free(s->devices); ++ s->devices = NULL; ++ return -1; + } +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec.h b/src/daemon/modules/device/cdi/behavior/cdi_spec.h +index 87248041..ca7b2cfa 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_spec.h ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec.h +@@ -40,14 +40,14 @@ struct cdi_cache_spec { + + void free_cdi_cache_spec(struct cdi_cache_spec *s); + +-struct cdi_cache_spec *cdi_spec_read_spec(const char *path, int priority, char **error); +-struct cdi_cache_spec *cdi_spec_new_spec(cdi_spec *raw, const char *path, int priority, char **error); +-const char *cdi_spec_get_vendor(struct cdi_cache_spec *s); +-const char *cdi_spec_get_class(struct cdi_cache_spec *s); +-struct cdi_cache_device *cdi_spec_get_cache_device(struct cdi_cache_spec *s, const char *name); +-const char *cdi_spec_get_path(struct cdi_cache_spec *s); +-int cdi_spec_get_priority(struct cdi_cache_spec *s); +-cdi_container_edits *cdi_spec_get_edits(struct cdi_cache_spec *s); ++struct cdi_cache_spec *cdi_spec_read_spec(const char *path, int priority); ++struct cdi_cache_spec *cdi_spec_new_spec(cdi_spec *raw, const char *path, int priority); ++const char *cdi_spec_get_vendor(const struct cdi_cache_spec *s); ++const char *cdi_spec_get_class(const struct cdi_cache_spec *s); ++struct cdi_cache_device *cdi_spec_get_cache_device(const struct cdi_cache_spec *s, const char *name); ++const char *cdi_spec_get_path(const struct cdi_cache_spec *s); ++int cdi_spec_get_priority(const struct cdi_cache_spec *s); ++cdi_container_edits *cdi_spec_get_edits(const struct cdi_cache_spec *s); + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c +index e340abc0..cafb52b8 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c +@@ -14,6 +14,17 @@ + ******************************************************************************/ + #include "cdi_spec_dirs.h" + ++#include ++#include ++#include ++ ++#include "utils.h" ++#include "path.h" ++#include "error.h" ++#include "utils_file.h" ++#include "utils_array.h" ++#include "cdi_spec.h" ++ + #define DEFAULT_SPEC_DIRS_LEN 2 + static char *default_spec_dirs_items[DEFAULT_SPEC_DIRS_LEN] = {CDI_DEFAULT_STATIC_DIR, CDI_DEFAULT_DYNAMIC_DIR}; + +@@ -23,7 +34,76 @@ string_array g_default_spec_dirs = { + .cap = DEFAULT_SPEC_DIRS_LEN, + }; + ++struct scan_spec_dir_cb_args { ++ struct cdi_scan_fn_maps *scan_fn_maps; ++ cdi_scan_spec_func scan_fn; ++ int priority; ++}; ++ ++static bool scan_spec_dir_cb(const char *dir, const struct dirent *pdirent, void *context) ++{ ++ struct scan_spec_dir_cb_args *args = (struct scan_spec_dir_cb_args *)context; ++ struct cdi_scan_fn_maps *scan_fn_maps = args->scan_fn_maps; ++ cdi_scan_spec_func scan_fn = args->scan_fn; ++ int priority = args->priority; ++ struct stat st = { 0 }; ++ __isula_auto_free char *file_path = NULL; ++ struct cdi_cache_spec *cache_spec = NULL; ++ ++ file_path = util_path_join(dir, pdirent->d_name); ++ if (file_path == NULL) { ++ ERROR("Failed to get path %s/%s", dir, pdirent->d_name); ++ goto error_out; ++ } ++ ++ if (lstat(file_path, &st) != 0) { ++ ERROR("Failed to lstat %s", file_path); ++ goto error_out; ++ } ++ if (S_ISDIR(st.st_mode)) { ++ DEBUG("Skip dir %s", file_path); ++ return true; ++ } ++ ++ if (!util_has_suffix(file_path, ".json")) { ++ DEBUG("Skip file %s", file_path); ++ return true; ++ } ++ ++ cache_spec = cdi_spec_read_spec(file_path, priority); ++ if (cache_spec == NULL) { ++ ERROR("Failed to read spec %s", file_path); ++ goto error_out; ++ } ++ scan_fn(scan_fn_maps, file_path, priority, cache_spec); ++ return true; ++ ++error_out: ++ *(scan_fn_maps->refresh_error_flag) = true; ++ return true; ++} ++ + int cdi_scan_spec_dirs(string_array *dirs, struct cdi_scan_fn_maps *scan_fn_maps, cdi_scan_spec_func scan_fn) + { ++ size_t i; ++ int nret = 0; ++ ++ for (i = 0; i < dirs->len; i++) { ++ struct scan_spec_dir_cb_args args = { ++ .scan_fn_maps = scan_fn_maps, ++ .scan_fn = scan_fn, ++ .priority = i, ++ }; ++ if (!util_dir_exists(dirs->items[i])) { ++ WARN("Cdi dir %s not exists", dirs->items[i]); ++ continue; ++ } ++ nret = util_scan_subdirs(dirs->items[i], scan_spec_dir_cb, &args); ++ if (nret != 0) { ++ ERROR("Failed to scan dir %s", dirs->items[i]); ++ return -1; ++ } ++ } ++ + return 0; + } +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h +index eedcabad..b17a0cd8 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h +@@ -35,7 +35,7 @@ struct cdi_scan_fn_maps { + bool *refresh_error_flag; + }; + typedef void(*cdi_scan_spec_func)(struct cdi_scan_fn_maps *scan_fn_maps, const char *path, int priority, +- struct cdi_cache_spec *spec, char *error); ++ struct cdi_cache_spec *spec); + + int cdi_scan_spec_dirs(string_array *dirs, struct cdi_scan_fn_maps *scan_fn_maps, cdi_scan_spec_func scan_fn); + +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_version.c b/src/daemon/modules/device/cdi/behavior/cdi_version.c +index 3e87c111..882a965e 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_version.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_version.c +@@ -14,27 +14,174 @@ + ******************************************************************************/ + #include "cdi_version.h" + +-#define CDI_V_CURRENT_VERSION "v"##CDI_CURRENT_VERSION +- +-#define CDI_V010 "v0.1.0" +-#define CDI_V020 "v0.2.0" +-#define CDI_V030 "v0.3.0" +-#define CDI_V040 "v0.4.0" +-#define CDI_V050 "v0.5.0" +-#define CDI_V060 "v0.6.0" ++#include ++#include ++#include ++ ++#include "error.h" ++#include "utils_version.h" ++#include "utils_string.h" ++#include "cdi_container_edits.h" ++#include "cdi_parser.h" ++ ++#define CDI_V010 "0.1.0" ++#define CDI_V020 "0.2.0" ++#define CDI_V030 "0.3.0" ++#define CDI_V040 "0.4.0" ++#define CDI_V050 "0.5.0" ++#define CDI_V060 "0.6.0" + #define CDI_V_EARLIEST CDI_V030 + +-const char *cdi_minimum_required_version(cdi_spec *spec) ++typedef bool (*required_version_cb)(cdi_spec *spec); ++ ++struct required_version_map { ++ char *version; ++ required_version_cb cb; ++}; ++ ++static bool requires_v060(cdi_spec *spec) ++{ ++ size_t i; ++ int ret = 0; ++ __isula_auto_free char *vendor = NULL; ++ __isula_auto_free char *class = NULL; ++ ++ // The 0.6.0 spec allows annotations to be specified at a spec level ++ if (spec->annotations != NULL) { ++ return true; ++ } ++ ++ // The 0.6.0 spec allows annotations to be specified at a device level ++ if (spec->devices != NULL) { ++ for (i = 0; i < spec->devices_len; i++) { ++ if (spec->devices[i]->annotations != NULL) { ++ return true; ++ } ++ } ++ } ++ ++ // The 0.6.0 spec allows dots "." in Kind name label (class) ++ ret = cdi_parser_parse_qualifier(spec->kind, &vendor, &class); ++ if (ret == 0 && vendor != NULL) { ++ if (util_strings_count(class, '.') > 0) { ++ return true; ++ } ++ } ++ return false; ++} ++ ++static bool check_host_path(cdi_container_edits *e) + { +- return NULL; ++ size_t i; ++ ++ if (e == NULL) { ++ return false; ++ } ++ for (i = 0; i < e->device_nodes_len; i++) { ++ // The HostPath field was added in 0.5.0 ++ if (e->device_nodes[i]->host_path != NULL) { ++ return true; ++ } ++ } ++ return false; + } + +-bool cdi_is_greater_than_version(const char *v, const char *o) ++static bool requires_v050(cdi_spec *spec) + { +- return true; ++ size_t i; ++ ++ for (i = 0; i < spec->devices_len; i++) { ++ // The 0.5.0 spec allowed device names to start with a digit instead of requiring a letter ++ if (spec->devices[i]->name != NULL && strlen(spec->devices[i]->name) > 0 && ++ !isalpha(spec->devices[i]->name[0])) { ++ return true; ++ } ++ if (check_host_path(spec->devices[i]->container_edits)) { ++ return true; ++ } ++ } ++ ++ return check_host_path(spec->container_edits); ++} ++ ++static bool check_mount_type(cdi_container_edits *e) ++{ ++ size_t i; ++ ++ if (e == NULL) { ++ return false; ++ } ++ for (i = 0; i < e->mounts_len; i++) { ++ // The Type field was added in 0.4.0 ++ if (e->mounts[i]->type != NULL) { ++ return true; ++ } ++ } ++ return false; ++} ++ ++static bool requires_v040(cdi_spec *spec) ++{ ++ size_t i; ++ ++ for (i = 0; i < spec->devices_len; i++) { ++ if (check_mount_type(spec->devices[i]->container_edits)) { ++ return true; ++ } ++ } ++ ++ return check_mount_type(spec->container_edits); ++} ++ ++#define VALID_SPEC_VERSIONS_LEN 6 ++static struct required_version_map g_valid_spec_versions[VALID_SPEC_VERSIONS_LEN] = { ++ {CDI_V010, NULL}, ++ {CDI_V020, NULL}, ++ {CDI_V030, NULL}, ++ {CDI_V040, requires_v060}, ++ {CDI_V050, requires_v050}, ++ {CDI_V060, requires_v040} ++}; ++ ++const char *cdi_minimum_required_version(cdi_spec *spec) ++{ ++ const char *min_version = CDI_V_EARLIEST; ++ int i; ++ bool result = true; ++ ++ if (spec == NULL) { ++ return NULL; ++ } ++ ++ for (i = 0; i < VALID_SPEC_VERSIONS_LEN; i++) { ++ if (g_valid_spec_versions[i].cb == NULL) { ++ continue; ++ } ++ if (g_valid_spec_versions[i].cb(spec)) { ++ if (util_version_greater_than(g_valid_spec_versions[i].version, min_version, &result) != 0) { ++ ERROR("Failed to compare version %s and %s", g_valid_spec_versions[i].version, min_version); ++ return NULL; ++ } ++ if (result) { ++ min_version = g_valid_spec_versions[i].version; ++ } ++ } ++ if (strcmp(min_version, CDI_CURRENT_VERSION)) { ++ break; ++ } ++ } ++ ++ return min_version; + } + + bool cdi_is_valid_version(const char *spec_version) + { +- return true; ++ int i; ++ ++ for (i = 0; i < VALID_SPEC_VERSIONS_LEN; i++) { ++ if (strcmp(g_valid_spec_versions[i].version, spec_version) == 0) { ++ return true; ++ } ++ } ++ return false; + } +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_version.h b/src/daemon/modules/device/cdi/behavior/cdi_version.h +index 99b7e692..6bd86340 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_version.h ++++ b/src/daemon/modules/device/cdi/behavior/cdi_version.h +@@ -24,7 +24,6 @@ extern "C" { + #define CDI_CURRENT_VERSION "0.6.0" + + const char *cdi_minimum_required_version(cdi_spec *spec); +-bool cdi_is_greater_than_version(const char *v, const char *o); + bool cdi_is_valid_version(const char *spec_version); + + #ifdef __cplusplus +diff --git a/src/daemon/modules/device/cdi/cdi_cache.c b/src/daemon/modules/device/cdi/cdi_cache.c +index 37767855..e637f7cd 100644 +--- a/src/daemon/modules/device/cdi/cdi_cache.c ++++ b/src/daemon/modules/device/cdi/cdi_cache.c +@@ -206,13 +206,12 @@ static void map_cdi_cache_specs_kvfree(void *key, void *value) + static void map_cdi_cache_device_kvfree(void *key, void *value) + { + free(key); +- free_cdi_cache_device((struct cdi_cache_device *)value); +-} +- +-static void set_refresh_error_flag(bool *refresh_error_flag, const char *error, const char *path) +-{ +- *refresh_error_flag = true; +- ERROR("Cdi refresh error: %s, spec %s", error, path); ++ /* ++ * map_cdi_cache_device_kvfree should not be recursively free cdi_cache_device. ++ * Otherwise, the function conflicts with the cdi_cache_specs free devices, ++ * triggering double free. ++ */ ++ (void)value; + } + + static bool resolve_conflict(struct cdi_scan_fn_maps *scan_fn_maps, const char *name, +@@ -220,8 +219,8 @@ static bool resolve_conflict(struct cdi_scan_fn_maps *scan_fn_maps, const char * + { + map_t *conflicts = scan_fn_maps->conflicts; + bool *refresh_error_flag = scan_fn_maps->refresh_error_flag; +- struct cdi_cache_spec *dev_spec = NULL; +- struct cdi_cache_spec *old_spec = NULL; ++ const struct cdi_cache_spec *dev_spec = NULL; ++ const struct cdi_cache_spec *old_spec = NULL; + int dev_prio; + int old_prio; + bool val = true; +@@ -251,7 +250,7 @@ static bool resolve_conflict(struct cdi_scan_fn_maps *scan_fn_maps, const char * + } + + static void refresh_scan_spec_func(struct cdi_scan_fn_maps *scan_fn_maps, const char *path, +- int priority, struct cdi_cache_spec *spec, char *error) ++ int priority, struct cdi_cache_spec *spec) + { + map_t *specs = scan_fn_maps->specs; + map_t *devices = scan_fn_maps->devices; +@@ -267,12 +266,6 @@ static void refresh_scan_spec_func(struct cdi_scan_fn_maps *scan_fn_maps, const + + if (util_clean_path(path, clean_path, sizeof(clean_path)) == NULL) { + ERROR("Failed to get clean path %s", path); +- format_errorf(&tmp_error, "Failed to get clean path %s", path); +- return; +- } +- if (error != NULL) { +- ERROR("Failed to load CDI Spec %s", error); +- format_errorf(&tmp_error, "Failed to load CDI Spec %s", error); + goto error_out; + } + +@@ -282,18 +275,15 @@ static void refresh_scan_spec_func(struct cdi_scan_fn_maps *scan_fn_maps, const + spec_array = util_common_array_new(1, (free_common_array_item_cb)free_cdi_cache_spec, util_clone_ptr); + if (spec_array == NULL) { + ERROR("Out of memory"); +- tmp_error = util_strdup_s("Out of memory"); + goto error_out; + } + if (!map_insert(specs, (void *)vendor, spec_array)) { + ERROR("Failed to insert spec array to specs"); +- tmp_error = util_strdup_s("Failed to insert spec array to specs"); + goto error_out; + } + } + if (util_append_common_array(spec_array, spec) != 0) { + ERROR("Failed to append spec"); +- tmp_error = util_strdup_s("Failed to append spec"); + goto error_out; + } + spec_array = NULL; +@@ -301,7 +291,6 @@ static void refresh_scan_spec_func(struct cdi_scan_fn_maps *scan_fn_maps, const + itor = map_itor_new(spec->devices); + if (itor == NULL) { + ERROR("Out of memory, create new map itor failed"); +- tmp_error = util_strdup_s("Out of memory, create new map itor failed"); + goto error_out; + } + for (; map_itor_valid(itor); map_itor_next(itor)) { +@@ -315,7 +304,6 @@ static void refresh_scan_spec_func(struct cdi_scan_fn_maps *scan_fn_maps, const + } + if (!map_replace(devices, (void *)qualified, dev)) { + ERROR("Failed to insert device to devices by name %s", qualified); +- format_errorf(&tmp_error, "Failed to insert device to devices by name %s", qualified); + goto error_out; + } + free(qualified); +@@ -324,7 +312,7 @@ static void refresh_scan_spec_func(struct cdi_scan_fn_maps *scan_fn_maps, const + goto out; + + error_out: +- set_refresh_error_flag(refresh_error_flag, tmp_error, path); ++ *refresh_error_flag = true; + out: + map_itor_free(itor); + return; +@@ -333,7 +321,6 @@ out: + static int refresh(struct cdi_cache *c) + { + int ret = 0; +- __isula_auto_free char *error = NULL; + map_t *specs = NULL; + map_t *devices = NULL; + map_t *conflicts = NULL; +@@ -559,8 +546,6 @@ error_out: + + static void watch_setup(struct cdi_watch *w, string_array *dirs) + { +- __isula_auto_free char *error = NULL; +- + if (w == NULL || dirs == NULL || dirs->len == 0) { + ERROR("Invalid param"); + return; +@@ -735,7 +720,6 @@ static void update_add_watch_dir(struct cdi_watch *w, const char *dir, bool *upd + { + int wd = -1; + bool tmp_value = true; +- __isula_auto_free char *error = NULL; + + wd = inotify_add_watch(w->watcher_fd, dir, CDI_WATCH_EVENTS); + if (wd < 0) { +@@ -770,7 +754,6 @@ static bool watch_update(struct cdi_watch *w, const char *removed, int wd) + bool *ok = NULL; + bool update = false; + map_itor *itor = NULL; +- __isula_auto_free char *error = NULL; + + itor = map_itor_new(w->tracked); + if (itor == NULL) { +diff --git a/src/daemon/modules/device/cdi/cdi_cache.h b/src/daemon/modules/device/cdi/cdi_cache.h +index da315de2..638e954e 100644 +--- a/src/daemon/modules/device/cdi/cdi_cache.h ++++ b/src/daemon/modules/device/cdi/cdi_cache.h +@@ -52,6 +52,7 @@ struct cdi_cache { + pthread_mutex_t mutex; + string_array *spec_dirs; // cdi-spec-dirs will scan for CDI Spec files + map_t *specs; // MAP_STR_PTR specs[vendor] = common_array of cdi_cache_spec* ++ // This map holding the reference to cdi device, the devices will not released when the map is freed. + map_t *devices; // MAP_STR_PTR devices[cdi_device.name] = cdi_cache_device* + bool refresh_error_flag; + bool auto_refresh; +diff --git a/src/daemon/modules/network/cni_operator/libcni/libcni_api.c b/src/daemon/modules/network/cni_operator/libcni/libcni_api.c +index 068881be..7ba983af 100644 +--- a/src/daemon/modules/network/cni_operator/libcni/libcni_api.c ++++ b/src/daemon/modules/network/cni_operator/libcni/libcni_api.c +@@ -28,6 +28,7 @@ + #include + + #include "utils.h" ++#include "utils_version.h" + #include "utils_network.h" + #include "libcni_cached.h" + #include "libcni_conf.h" +@@ -544,97 +545,6 @@ free_out: + return ret; + } + +-struct parse_version { +- int major; +- int minor; +- int micro; +-}; +- +-static bool do_parse_version(const char **splits, size_t splits_len, struct parse_version *ret) +-{ +- if (util_safe_int(splits[0], &ret->major) != 0) { +- ERROR("failed to convert major version part: %s", splits[0]); +- return false; +- } +- +- if (splits_len >= 2 && util_safe_int(splits[1], &ret->minor) != 0) { +- ERROR("failed to convert minor version part: %s", splits[1]); +- return false; +- } +- +- if (splits_len >= 3 && util_safe_int(splits[2], &ret->micro) != 0) { +- ERROR("failed to convert micro version part: %s", splits[2]); +- return false; +- } +- +- return true; +-} +- +-static bool parse_version_from_str(const char *src_version, struct parse_version *result) +-{ +- char **splits = NULL; +- const size_t max_len = 4; +- size_t tlen = 0; +- bool ret = false; +- +- splits = util_string_split(src_version, '.'); +- if (splits == NULL) { +- ERROR("Split version: \"%s\" failed", src_version); +- return false; +- } +- tlen = util_array_len((const char **)splits); +- if (tlen < 1 || tlen >= max_len) { +- ERROR("Invalid version: \"%s\"", src_version); +- goto out; +- } +- +- ret = do_parse_version((const char **)splits, tlen, result); +- +-out: +- util_free_array(splits); +- return ret; +-} +- +-static bool do_compare_version(const struct parse_version *p_first, const struct parse_version *p_second) +-{ +- bool ret = false; +- +- if (p_first->major > p_second->major) { +- ret = true; +- } else if (p_first->major == p_second->major) { +- if (p_first->minor > p_second->minor) { +- ret = true; +- } else if (p_first->minor == p_second->minor && p_first->micro >= p_second->micro) { +- ret = true; +- } +- } +- +- return ret; +-} +- +-static int version_greater_than_or_equal_to(const char *first, const char *second, bool *result) +-{ +- struct parse_version first_parsed = { 0 }; +- struct parse_version second_parsed = { 0 }; +- +- if (result == NULL) { +- ERROR("Invalid argument"); +- return -1; +- } +- +- if (!parse_version_from_str(first, &first_parsed)) { +- return -1; +- } +- +- if (!parse_version_from_str(second, &second_parsed)) { +- return -1; +- } +- +- *result = do_compare_version(&first_parsed, &second_parsed); +- +- return 0; +-} +- + static inline bool check_add_network_args(const cni_net_conf *net, const struct runtime_conf *rc) + { + return (net == NULL || rc == NULL); +@@ -690,7 +600,7 @@ int cni_add_network_list(const struct cni_network_list_conf *list, const struct + } + + if (*pret != NULL && +- version_greater_than_or_equal_to((*pret)->cniversion, SUPPORT_CACHE_AND_CHECK_VERSION, &greater) != 0) { ++ util_version_greater_than_or_equal_to((*pret)->cniversion, SUPPORT_CACHE_AND_CHECK_VERSION, &greater) != 0) { + return 0; + } + +@@ -741,7 +651,7 @@ int cni_del_network_list(const struct cni_network_list_conf *list, const struct + return -1; + } + +- if (version_greater_than_or_equal_to(list->list->cni_version, SUPPORT_CACHE_AND_CHECK_VERSION, &greater) != 0) { ++ if (util_version_greater_than_or_equal_to(list->list->cni_version, SUPPORT_CACHE_AND_CHECK_VERSION, &greater) != 0) { + return -1; + } + +@@ -807,7 +717,7 @@ int cni_check_network_list(const struct cni_network_list_conf *list, const struc + return -1; + } + +- if (version_greater_than_or_equal_to(list->list->cni_version, SUPPORT_CACHE_AND_CHECK_VERSION, &greater) != 0) { ++ if (util_version_greater_than_or_equal_to(list->list->cni_version, SUPPORT_CACHE_AND_CHECK_VERSION, &greater) != 0) { + return -1; + } + +diff --git a/src/utils/cutils/utils_version.c b/src/utils/cutils/utils_version.c +new file mode 100644 +index 00000000..9dea5b09 +--- /dev/null ++++ b/src/utils/cutils/utils_version.c +@@ -0,0 +1,147 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2022. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-4-7 ++ * Description: provide version functions ++ ********************************************************************************/ ++ ++#define _GNU_SOURCE ++#include "utils_version.h" ++ ++#include ++ ++#include "utils.h" ++#include "utils_string.h" ++ ++struct parse_version { ++ int major; ++ int minor; ++ int micro; ++}; ++ ++static bool do_parse_version(const char **splits, size_t splits_len, struct parse_version *ret) ++{ ++ if (util_safe_int(splits[0], &ret->major) != 0) { ++ ERROR("Failed to convert major version part: %s", splits[0]); ++ return false; ++ } ++ ++ if (splits_len >= 2 && util_safe_int(splits[1], &ret->minor) != 0) { ++ ERROR("Failed to convert minor version part: %s", splits[1]); ++ return false; ++ } ++ ++ if (splits_len >= 3 && util_safe_int(splits[2], &ret->micro) != 0) { ++ ERROR("Failed to convert micro version part: %s", splits[2]); ++ return false; ++ } ++ ++ return true; ++} ++ ++static bool parse_version_from_str(const char *src_version, struct parse_version *result) ++{ ++ __isula_auto_array_t char **splits = NULL; ++ const size_t max_len = 4; ++ size_t tlen = 0; ++ bool ret = false; ++ ++ splits = util_string_split(src_version, '.'); ++ if (splits == NULL) { ++ ERROR("Split version: \"%s\" failed", src_version); ++ return false; ++ } ++ tlen = util_array_len((const char **)splits); ++ if (tlen < 1 || tlen >= max_len) { ++ ERROR("Invalid version: \"%s\"", src_version); ++ return false; ++ } ++ ++ ret = do_parse_version((const char **)splits, tlen, result); ++ ++ return ret; ++} ++ ++static int do_compare_version(const struct parse_version *p_first, const struct parse_version *p_second) ++{ ++ if (p_first->major != p_second->major) { ++ return p_first->major - p_second->major; ++ } ++ if (p_first->minor != p_second->minor) { ++ return p_first->minor - p_second->minor; ++ } ++ if (p_first->micro != p_second->micro) { ++ return p_first->micro - p_second->micro; ++ } ++ ++ return 0; ++} ++ ++int util_version_compare(const char *first, const char *second, int *diff_value) ++{ ++ struct parse_version first_parsed = { 0 }; ++ struct parse_version second_parsed = { 0 }; ++ ++ if (first == NULL || second == NULL || diff_value == NULL) { ++ ERROR("Invalid argument"); ++ return -1; ++ } ++ ++ if (!parse_version_from_str(first, &first_parsed)) { ++ return -1; ++ } ++ ++ if (!parse_version_from_str(second, &second_parsed)) { ++ return -1; ++ } ++ ++ *diff_value = do_compare_version(&first_parsed, &second_parsed); ++ ++ return 0; ++} ++ ++int util_version_greater_than(const char *first, const char *second, bool *result) ++{ ++ int ret; ++ int diff_value = 0; ++ ++ if (result == NULL) { ++ ERROR("Invalid argument"); ++ return -1; ++ } ++ ++ ret = util_version_compare(first, second, &diff_value); ++ if (ret != 0) { ++ return ret; ++ } ++ ++ *result = (diff_value > 0); ++ return ret; ++} ++ ++int util_version_greater_than_or_equal_to(const char *first, const char *second, bool *result) ++{ ++ int ret; ++ int diff_value = 0; ++ ++ if (result == NULL) { ++ ERROR("Invalid argument"); ++ return -1; ++ } ++ ++ ret = util_version_compare(first, second, &diff_value); ++ if (ret != 0) { ++ return ret; ++ } ++ ++ *result = (diff_value >= 0); ++ return ret; ++} +diff --git a/src/utils/cutils/utils_version.h b/src/utils/cutils/utils_version.h +new file mode 100644 +index 00000000..f9b543b8 +--- /dev/null ++++ b/src/utils/cutils/utils_version.h +@@ -0,0 +1,36 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2022. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-4-7 ++ * Description: provide version functions ++ ********************************************************************************/ ++ ++#ifndef UTILS_CUTILS_UTILS_VERSION_H ++#define UTILS_CUTILS_UTILS_VERSION_H ++ ++#include ++#include ++#include ++#include ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++int util_version_compare(const char *first, const char *second, int *diff_value); ++int util_version_greater_than(const char *first, const char *second, bool *result); ++int util_version_greater_than_or_equal_to(const char *first, const char *second, bool *result); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif // UTILS_CUTILS_UTILS_VERSION_H +\ No newline at end of file +-- +2.34.1 + diff --git a/0067-cdi-support-modules-container_edits-parser.patch b/0067-cdi-support-modules-container_edits-parser.patch new file mode 100644 index 0000000..901de2b --- /dev/null +++ b/0067-cdi-support-modules-container_edits-parser.patch @@ -0,0 +1,1210 @@ +From d0442316761717849deb248b5da45240f7bb1d38 Mon Sep 17 00:00:00 2001 +From: liuxu +Date: Thu, 11 Apr 2024 11:01:44 +0800 +Subject: [PATCH 67/69] cdi:support modules container_edits/parser + +--- + src/daemon/modules/api/specs_api.h | 13 + + .../device/cdi/behavior/cdi_container_edits.c | 622 +++++++++++++++++- + .../device/cdi/behavior/cdi_container_edits.h | 2 +- + .../device/cdi/behavior/parser/cdi_parser.c | 196 +++++- + .../device/cdi/behavior/parser/cdi_parser.h | 6 +- + src/daemon/modules/spec/specs.c | 230 +++++++ + 6 files changed, 1052 insertions(+), 17 deletions(-) + +diff --git a/src/daemon/modules/api/specs_api.h b/src/daemon/modules/api/specs_api.h +index f54c0d31..0999836b 100644 +--- a/src/daemon/modules/api/specs_api.h ++++ b/src/daemon/modules/api/specs_api.h +@@ -55,6 +55,19 @@ const oci_runtime_spec *get_readonly_default_oci_spec(bool system_container); + + int spec_module_init(void); + ++#ifdef ENABLE_CDI ++int defs_process_add_multiple_env(defs_process *dp, const char **envs, size_t env_len); ++int spec_add_multiple_process_env(oci_runtime_spec *oci_spec, const char **envs, size_t env_len); ++int spec_add_device(oci_runtime_spec *oci_spec, defs_device *device); ++int spec_add_linux_resources_device(oci_runtime_spec *oci_spec, bool allow, const char *dev_type, ++ int64_t major, int64_t minor, const char *access); ++void spec_remove_mount(oci_runtime_spec *oci_spec, const char *dest); ++int spec_add_mount(oci_runtime_spec *oci_spec, defs_mount *mnt); ++int spec_add_prestart_hook(oci_runtime_spec *oci_spec, defs_hook *prestart_hook); ++int spec_add_poststart_hook(oci_runtime_spec *oci_spec, defs_hook *poststart_hook); ++int spec_add_poststop_hook(oci_runtime_spec *oci_spec, defs_hook *poststop_hook); ++#endif /* ENABLE_CDI */ ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c +index 590118b1..816b9c2d 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c +@@ -14,36 +14,642 @@ + ******************************************************************************/ + #include "cdi_container_edits.h" + ++#include ++#include ++#include ++#include ++#include ++ ++#include "error.h" ++#include "path.h" ++#include "specs_extend.h" ++#include "utils.h" ++#include "utils_array.h" ++#include "specs_api.h" ++ ++/* ++ * The OCI being used by the iSulad not supportes ++ * createRuntime/createContainer/startContainer currently. ++ */ + // PRESTART_HOOK is the name of the OCI "prestart" hook. + #define PRESTART_HOOK "prestart" +-// CREATE_RUNTIME_HOOK is the name of the OCI "createRuntime" hook. +-#define CREATE_RUNTIME_HOOK "createRuntime" +-// CREATE_CONTAINER_HOOK is the name of the OCI "createContainer" hook. +-#define CREATE_CONTAINER_HOOK "createContainer" +-// START_CONTAINER_HOOK is the name of the OCI "startContainer" hook. +-#define START_CONTAINER_HOOK "startContainer" + // POSTSTART_HOOK is the name of the OCI "poststart" hook. + #define POSTSTART_HOOK "poststart" + // POSTSTOP_HOOK is the name of the OCI "poststop" hook. + #define POSTSTOP_HOOK "poststop" + ++#define VALID_HOOK_NAME_LEN 3 ++static const char* g_valid_hook_names[VALID_HOOK_NAME_LEN] = { ++ PRESTART_HOOK, POSTSTART_HOOK, POSTSTOP_HOOK ++}; ++ ++static int cdi_validate_env(char **envs, size_t envs_len); ++static int cdi_validate_device_node(cdi_device_node *d); ++static int cdi_validate_hook(cdi_hook *h); ++static int cdi_validate_mount(cdi_mount *m); ++ ++#define BLOCK_DEVICE "b" ++#define CHAR_DEVICE "c" ++#define FIFO_DEVICE "p" ++static int device_info_from_path(const char *path, char **dev_type, int64_t *major, int64_t *minor) ++{ ++ struct stat stat = { 0 }; ++ int ret = 0; ++ ++ ret = lstat(path, &stat); ++ if (ret != 0) { ++ ERROR("Failed to stat %s", path); ++ return -1; ++ } ++ ++ if (S_ISBLK(stat.st_mode)) { ++ *dev_type = util_strdup_s(BLOCK_DEVICE); ++ } else if (S_ISCHR(stat.st_mode)) { ++ *dev_type = util_strdup_s(CHAR_DEVICE); ++ } else if (S_ISFIFO(stat.st_mode)) { ++ *dev_type = util_strdup_s(FIFO_DEVICE); ++ } else { ++ *dev_type = NULL; ++ *major = 0; ++ *minor = 0; ++ ERROR("Not a device node"); ++ return -1; ++ } ++ ++ *major = (int64_t)major(stat.st_rdev); ++ *minor = (int64_t)minor(stat.st_rdev); ++ return 0; ++} ++ ++static int fill_device_node_info(cdi_device_node *d) ++{ ++ __isula_auto_free char *dev_type = NULL; ++ int64_t major; ++ int64_t minor; ++ ++ if (d->host_path == NULL) { ++ d->host_path = util_strdup_s(d->path); ++ } ++ ++ if (d->type != NULL && (d->major != 0 || strcmp(d->type, FIFO_DEVICE) == 0)) { ++ return 0; ++ } ++ ++ if (device_info_from_path(d->host_path, &dev_type, &major, &minor) != 0) { ++ ERROR("Failed to stat CDI host device %s", d->host_path); ++ return -1; ++ } ++ ++ if (d->type == NULL) { ++ d->type = dev_type; ++ dev_type = NULL; ++ } else { ++ if (strcmp(d->type, dev_type) != 0) { ++ ERROR("CDI device (%s, %s), host type mismatch (%s, %s)", ++ d->path, d->host_path, d->type, dev_type); ++ return -1; ++ } ++ } ++ if (d->major == 0 && strcmp(d->type, FIFO_DEVICE) != 0) { ++ d->major = major; ++ d->minor = minor; ++ } ++ return 0; ++} ++ ++static cdi_device_node *clone_cdi_device_node(cdi_device_node *d) ++{ ++ cdi_device_node *device_node = NULL; ++ ++ device_node = util_common_calloc_s(sizeof(*device_node)); ++ if (device_node == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ device_node->path = util_strdup_s(d->path); ++ device_node->host_path = util_strdup_s(d->host_path); ++ device_node->type = util_strdup_s(d->type); ++ device_node->major = d->major; ++ device_node->minor = d->minor; ++ device_node->file_mode = d->file_mode; ++ device_node->permissions = util_strdup_s(d->permissions); ++ device_node->uid = d->uid; ++ device_node->gid = d->gid; ++ return device_node; ++} ++ ++static cdi_hook *clone_cdi_hook(cdi_hook *h) ++{ ++ cdi_hook *hook = NULL; ++ ++ hook = util_common_calloc_s(sizeof(*hook)); ++ if (hook == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ hook->hook_name = util_strdup_s(h->hook_name); ++ hook->path = util_strdup_s(h->path); ++ if (h->args_len != 0) { ++ hook->args = util_copy_array_by_len(h->args, h->args_len); ++ if (hook->args == NULL) { ++ ERROR("Failed to copy args"); ++ goto error_out; ++ } ++ hook->args_len = h->args_len; ++ } ++ if (h->env_len != 0) { ++ hook->env = util_copy_array_by_len(h->env, h->env_len); ++ if (hook->env == NULL) { ++ ERROR("Failed to copy env"); ++ goto error_out; ++ } ++ hook->env_len = h->env_len; ++ } ++ hook->timeout = h->timeout; ++ ++ return hook; ++ ++error_out: ++ free_cdi_hook(hook); ++ return NULL; ++} ++ ++static cdi_mount *clone_cdi_mount(cdi_mount *m) ++{ ++ cdi_mount *mount = NULL; ++ ++ mount = util_common_calloc_s(sizeof(*mount)); ++ if (mount == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ mount->host_path = util_strdup_s(m->host_path); ++ mount->container_path = util_strdup_s(m->container_path); ++ if (m->options_len != 0) { ++ mount->options = util_copy_array_by_len(m->options, m->options_len); ++ if (mount->options == NULL) { ++ ERROR("Failed to copy options"); ++ free_cdi_mount(mount); ++ return NULL; ++ } ++ mount->options_len = m->options_len; ++ } ++ mount->type = util_strdup_s(m->type); ++ ++ return mount; ++} ++ ++static defs_hook *cdi_hook_to_oci(cdi_hook *h) ++{ ++ defs_hook *oci_hook = NULL; ++ ++ oci_hook = util_common_calloc_s(sizeof(*oci_hook)); ++ if (oci_hook == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ oci_hook->path = util_strdup_s(h->path); ++ if (h->args_len != 0) { ++ oci_hook->args = util_copy_array_by_len(h->args, h->args_len); ++ if (oci_hook->args == NULL) { ++ ERROR("Failed to copy args"); ++ goto error_out; ++ } ++ oci_hook->args_len = h->args_len; ++ } ++ if (h->env_len != 0) { ++ oci_hook->env = util_copy_array_by_len(h->env, h->env_len); ++ if (oci_hook->env == NULL) { ++ ERROR("Failed to copy env"); ++ goto error_out; ++ } ++ oci_hook->env_len = h->env_len; ++ } ++ oci_hook->timeout = h->timeout; ++ return oci_hook; ++ ++error_out: ++ free_defs_hook(oci_hook); ++ return NULL; ++} ++ ++static defs_mount *cdi_mount_to_oci(cdi_mount *m) ++{ ++ defs_mount *oci_mount = NULL; ++ ++ oci_mount = util_common_calloc_s(sizeof(*oci_mount)); ++ if (oci_mount == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ oci_mount->source = util_strdup_s(m->host_path); ++ oci_mount->destination = util_strdup_s(m->container_path); ++ if (m->options_len != 0) { ++ oci_mount->options = util_copy_array_by_len(m->options, m->options_len); ++ if (oci_mount->options == NULL) { ++ ERROR("Failed to copy options"); ++ free_defs_mount(oci_mount); ++ return NULL; ++ } ++ oci_mount->options_len = m->options_len; ++ } ++ oci_mount->type = util_strdup_s(m->type); ++ ++ return oci_mount; ++} ++ ++static defs_device *cdi_device_node_to_oci(cdi_device_node *d) ++{ ++ defs_device *oci_device = NULL; ++ ++ oci_device = util_common_calloc_s(sizeof(*oci_device)); ++ if (oci_device == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ oci_device->path = util_strdup_s(d->path); ++ oci_device->type = util_strdup_s(d->type); ++ oci_device->major = d->major; ++ oci_device->minor = d->minor; ++ oci_device->file_mode = d->file_mode; ++ oci_device->uid = d->uid; ++ oci_device->gid = d->gid; ++ ++ return oci_device; ++} ++ ++static int apply_cdi_device_nodes(cdi_container_edits *e, oci_runtime_spec *spec) ++{ ++ size_t i; ++ defs_device *dev = NULL; ++ cdi_device_node *dn = NULL; ++ const char *access = NULL; ++ ++ for (i = 0; i < e->device_nodes_len; i++) { ++ dn = clone_cdi_device_node(e->device_nodes[i]); ++ if (dn == NULL) { ++ ERROR("Failed to copy device node"); ++ goto error_out; ++ } ++ ++ if (fill_device_node_info(dn) != 0) { ++ goto error_out; ++ } ++ dev = cdi_device_node_to_oci(dn); ++ if (dev == NULL) { ++ ERROR("Failed to generate oci device"); ++ goto error_out; ++ } ++ /* Currently, for uid and gid, isulad cannot distinguish ++ * 0 and unspecified. Here, 0 is processed as unspecified. ++ */ ++ if (dev->uid == 0 && spec->process != NULL) { ++ if (spec->process->user->uid > 0) { ++ dev->uid = spec->process->user->uid; ++ } ++ } ++ if (dev->gid == 0 && spec->process != NULL) { ++ if (spec->process->user->gid > 0) { ++ dev->gid = spec->process->user->gid; ++ } ++ } ++ ++ if (spec_add_device(spec, dev) != 0) { ++ goto error_out; ++ } ++ ++ if (strcmp(dev->type, BLOCK_DEVICE) == 0 || strcmp(dev->type, CHAR_DEVICE) == 0) { ++ if (e->device_nodes[i]->permissions != NULL) { ++ access = e->device_nodes[i]->permissions; ++ } else { ++ access = "rwm"; ++ } ++ if (spec_add_linux_resources_device(spec, true, dev->type, ++ dev->major, dev->minor, access)) { ++ dev = NULL; ++ goto error_out; ++ } ++ } ++ free_cdi_device_node(dn); ++ dn = NULL; ++ dev = NULL; ++ } ++ ++ return 0; ++ ++error_out: ++ free_cdi_device_node(dn); ++ free_defs_device(dev); ++ return -1; ++} ++ ++static int defs_mount_parts(defs_mount *m) ++{ ++ char cleanpath[PATH_MAX] = { 0 }; ++ if (util_clean_path(m->destination, cleanpath, sizeof(cleanpath)) == NULL) { ++ return -1; ++ } ++ ++ return util_strings_count(cleanpath, '/'); ++} ++ ++static inline int defs_mount_cmp(defs_mount **first, defs_mount **second) ++{ ++ int first_part = defs_mount_parts(*first); ++ int second_part = defs_mount_parts(*second); ++ ++ if (first_part < second_part) { ++ return -1; ++ } ++ if (first_part > second_part) { ++ return 1; ++ } ++ ++ return strcmp((*first)->destination, (*second)->destination); ++} ++ ++static int apply_cdi_mounts(cdi_container_edits *e, oci_runtime_spec *spec) ++{ ++ size_t i; ++ defs_mount *mnt = NULL; ++ ++ if (e->mounts_len == 0) { ++ return 0; ++ } ++ ++ for (i = 0; i < e->mounts_len; i++) { ++ spec_remove_mount(spec, e->mounts[i]->container_path); ++ mnt = cdi_mount_to_oci(e->mounts[i]); ++ if (spec_add_mount(spec, mnt) != 0) { ++ free_defs_mount(mnt); ++ return -1; ++ } ++ } ++ ++ qsort(spec->mounts, spec->mounts_len, ++ sizeof(defs_mount *), (int (*)(const void *, const void *))defs_mount_cmp); ++ return 0; ++} ++ ++static int apply_cdi_hooks(cdi_container_edits *e, oci_runtime_spec *spec) ++{ ++ size_t i; ++ int ret = 0; ++ ++ for (i = 0; i < e->hooks_len; i++) { ++ defs_hook *oci_hook = cdi_hook_to_oci(e->hooks[i]); ++ if (strcmp(e->hooks[i]->hook_name, PRESTART_HOOK)) { ++ ret = spec_add_prestart_hook(spec, oci_hook); ++ } else if (strcmp(e->hooks[i]->hook_name, POSTSTART_HOOK)) { ++ ret = spec_add_poststart_hook(spec, oci_hook); ++ } else if (strcmp(e->hooks[i]->hook_name, POSTSTOP_HOOK)) { ++ ret = spec_add_poststop_hook(spec, oci_hook); ++ } else { ++ /* ++ * The OCI being used by the iSulad not supportes ++ * createRuntime/createContainer/startContainer currently. ++ */ ++ ERROR("Unknown hook name %s", e->hooks[i]->hook_name); ++ free_defs_hook(oci_hook); ++ return -1; ++ } ++ if (ret != 0) { ++ ERROR("Failed add hook %s", e->hooks[i]->hook_name); ++ free_defs_hook(oci_hook); ++ return -1; ++ } ++ } ++ return ret; ++} ++ + int cdi_container_edits_apply(cdi_container_edits *e, oci_runtime_spec *spec) + { ++ if (spec == NULL) { ++ ERROR("Can't edit nil OCI Spec"); ++ return -1; ++ } ++ if (e == NULL) { ++ WARN("Cdi container edits is nil"); ++ return 0; ++ } ++ ++ if (e->env_len > 0) { ++ if (spec_add_multiple_process_env(spec, (const char **)e->env, e->env_len) != 0) { ++ ERROR("Failed to merge envs"); ++ return -1; ++ } ++ } ++ ++ if (apply_cdi_device_nodes(e, spec) != 0) { ++ ERROR("Failed to apply device nodes"); ++ return -1; ++ } ++ ++ if (apply_cdi_mounts(e, spec) != 0) { ++ ERROR("Failed to apply mounts"); ++ return -1; ++ } ++ ++ if (apply_cdi_hooks(e, spec) != 0) { ++ ERROR("Failed to apply hooks"); ++ return -1; ++ } ++ + return 0; + } + +-int cdi_container_edits_validate(cdi_container_edits *e, char **error) ++int cdi_container_edits_validate(cdi_container_edits *e) + { ++ size_t i; ++ ++ if (e == NULL) { ++ WARN("Cdi container edits is nil"); ++ return 0; ++ } ++ ++ if (cdi_validate_env(e->env, e->env_len) != 0) { ++ ERROR("Invalid container edits"); ++ return -1; ++ } ++ for (i = 0; i < e->device_nodes_len; i++) { ++ if (cdi_validate_device_node(e->device_nodes[i]) != 0) { ++ ERROR("Invalid container device node"); ++ return -1; ++ } ++ } ++ for (i = 0; i < e->hooks_len; i++) { ++ if (cdi_validate_hook(e->hooks[i]) != 0) { ++ ERROR("Invalid container hook"); ++ return -1; ++ } ++ } ++ for (i = 0; i < e->mounts_len; i++) { ++ if (cdi_validate_mount(e->mounts[i]) != 0) { ++ ERROR("Invalid container mount"); ++ return -1; ++ } ++ } ++ + return 0; + } + ++#define EDITS_APPEND_ITEM_DEF(item) \ ++ static int append_##item(cdi_container_edits *e, cdi_container_edits *o, clone_common_array_item_cb cb) \ ++ { \ ++ common_array e_array = { \ ++ .items = (void **)e->item, \ ++ .len = e->item##_len, \ ++ .cap = e->item##_len, \ ++ .free_item_cb = NULL, \ ++ .clone_item_cb = cb \ ++ }; \ ++ common_array o_array = { \ ++ .items = (void **)o->item, \ ++ .len = o->item##_len, \ ++ .cap = o->item##_len, \ ++ .free_item_cb = NULL, \ ++ .clone_item_cb = cb \ ++ }; \ ++ if (util_merge_common_array(&e_array, &o_array) != 0) { \ ++ ERROR("Out of memory"); \ ++ return -1; \ ++ } \ ++ e->item = (void *)e_array.items; \ ++ e->item##_len += o->item##_len; \ ++ return 0; \ ++ } ++ ++EDITS_APPEND_ITEM_DEF(env) ++EDITS_APPEND_ITEM_DEF(device_nodes) ++EDITS_APPEND_ITEM_DEF(hooks) ++EDITS_APPEND_ITEM_DEF(mounts) ++ + int cdi_container_edits_append(cdi_container_edits *e, cdi_container_edits *o) + { ++ if (o == NULL) { ++ return 0; ++ } ++ if (e == NULL) { ++ ERROR("Invalid params"); ++ return -1; ++ } ++ ++ if (append_env(e, o, (clone_common_array_item_cb)util_strdup_s) != 0) { ++ return -1; ++ } ++ if (append_device_nodes(e, o, (clone_common_array_item_cb)clone_cdi_device_node) != 0) { ++ return -1; ++ } ++ if (append_hooks(e, o, (clone_common_array_item_cb)clone_cdi_hook) != 0) { ++ return -1; ++ } ++ if (append_mounts(e, o, (clone_common_array_item_cb)clone_cdi_mount) != 0) { ++ return -1; ++ } ++ + return 0; + } + + bool cdi_container_edits_is_empty(cdi_container_edits *e) + { +- return true; ++ if (e == NULL) { ++ return false; ++ } ++ return e->env_len + e->device_nodes_len + e->hooks_len + e->mounts_len == 0; + } + ++static int cdi_validate_env(char **envs, size_t envs_len) ++{ ++ size_t i; ++ char *ptr = NULL; ++ ++ for (i = 0; i < envs_len; i++) { ++ ptr = strchr(envs[i], '='); ++ if (ptr == NULL || ptr == envs[i]) { ++ ERROR("Invalid environment variable %s", envs[i]); ++ return -1; ++ } ++ } ++ return 0; ++} ++ ++static int cdi_validate_device_node(cdi_device_node *d) ++{ ++ char *p = NULL; ++ ++ if (d == NULL) { ++ ERROR("Device node is nil"); ++ return -1; ++ } ++ ++ if (d->path == NULL) { ++ ERROR("Invalid (empty) device path"); ++ return -1; ++ } ++ if (d->type != NULL && strcmp(d->type, BLOCK_DEVICE) != 0 && ++ strcmp(d->type, CHAR_DEVICE) != 0 && strcmp(d->type, FIFO_DEVICE) != 0) { ++ ERROR("Device %s: invalid type %s", d->path, d->type); ++ return -1; ++ } ++ for (p = d->permissions; p != NULL && *p != '\0'; p++) { ++ if (*p != 'r' && *p != 'w' && *p != 'm') { ++ ERROR("Device %s: invalid permissions %s", d->path, d->permissions); ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ ++static int cdi_validate_hook(cdi_hook *h) ++{ ++ size_t i; ++ ++ if (h == NULL) { ++ ERROR("Hook is nil"); ++ return -1; ++ } ++ ++ for (i = 0; i < VALID_HOOK_NAME_LEN; i++) { ++ if (strcmp(h->hook_name, g_valid_hook_names[i]) == 0) { ++ break; ++ } ++ } ++ if (i == VALID_HOOK_NAME_LEN) { ++ ERROR("Invalid hook name %s", h->hook_name); ++ return -1; ++ } ++ if (h->path == NULL) { ++ ERROR("Invalid hook %s with empty path", h->hook_name); ++ return -1; ++ } ++ if (cdi_validate_env(h->env, h->env_len) != 0) { ++ ERROR("Invalid hook %s", h->hook_name); ++ return -1; ++ } ++ return 0; ++} ++ ++static int cdi_validate_mount(cdi_mount *m) ++{ ++ if (m == NULL) { ++ ERROR("Mount is nil"); ++ return -1; ++ } ++ ++ if (m->host_path == NULL) { ++ ERROR("Invalid mount, empty host path"); ++ return -1; ++ } ++ if (m->container_path == NULL) { ++ ERROR("Invalid mount, empty container path"); ++ return -1; ++ } ++ return 0; ++} +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_container_edits.h b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.h +index ea921e37..ddceec25 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_container_edits.h ++++ b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.h +@@ -28,7 +28,7 @@ extern "C" { + #endif + + int cdi_container_edits_apply(cdi_container_edits *e, oci_runtime_spec *spec); +-int cdi_container_edits_validate(cdi_container_edits *e, char **error); ++int cdi_container_edits_validate(cdi_container_edits *e); + int cdi_container_edits_append(cdi_container_edits *e, cdi_container_edits *o); + bool cdi_container_edits_is_empty(cdi_container_edits *e); + +diff --git a/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c +index 14293c72..8824d29c 100644 +--- a/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c ++++ b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c +@@ -14,42 +14,228 @@ + ******************************************************************************/ + #include "cdi_parser.h" + ++#include ++#include ++#include ++#include ++#include ++ ++#include "error.h" ++#include "utils_string.h" ++ ++/* cdi_parser_qualified_name returns the qualified name for a device. ++ * The syntax for a qualified device names is ++ * ++ * "/=". ++ * ++ * A valid vendor and class name may contain the following runes: ++ * ++ * 'A'-'Z', 'a'-'z', '0'-'9', '.', '-', '_'. ++ * ++ * A valid device name may contain the following runes: ++ * ++ * 'A'-'Z', 'a'-'z', '0'-'9', '-', '_', '.', ':' ++ */ + char *cdi_parser_qualified_name(const char *vendor, const char *class, const char *name) + { +- return NULL; ++ char device_name[PATH_MAX] = { 0 }; ++ int nret; ++ ++ if (vendor == NULL || class == NULL || name == NULL) { ++ ERROR("Invalid argument"); ++ return NULL; ++ } ++ ++ nret = snprintf(device_name, sizeof(device_name), "%s/%s=%s", ++ vendor, class, name); ++ if (nret < 0 || (size_t)nret >= sizeof(device_name)) { ++ ERROR("Device name is too long"); ++ return NULL; ++ } ++ return util_strdup_s(device_name); + } + ++// cdi_parser_is_qualified_name tests if a device name is qualified. + bool cdi_parser_is_qualified_name(const char *device) + { +- return true; ++ __isula_auto_free char *vendor = NULL; ++ __isula_auto_free char *class = NULL; ++ __isula_auto_free char *name = NULL; ++ ++ return cdi_parser_parse_qualified_name(device, &vendor, &class, &name) == 0; + } + ++// cdi_parser_parse_qualified_name splits a qualified name into device vendor, class, and name. + int cdi_parser_parse_qualified_name(const char *device, char **vendor, char **class, char **name) + { ++ int ret = 0; ++ ++ ret = cdi_parser_parse_device(device, vendor, class, name); ++ if (ret != 0) { ++ if (*vendor == NULL) { ++ ERROR("Unqualified device %s, missing vendor", device); ++ return -1; ++ } ++ if (*class == NULL) { ++ ERROR("Unqualified device %s, missing class", device); ++ return -1; ++ } ++ if (*name == NULL) { ++ ERROR("Unqualified device %s, missing name", device); ++ return -1; ++ } ++ ERROR("Unqualified device %s", device); ++ return -1; ++ } ++ ++ if (cdi_parser_validate_vendor_name(*vendor) != 0) { ++ ERROR("Invalid device %s", device); ++ goto err_out; ++ } ++ if (cdi_parser_validate_class_name(*class) != 0) { ++ ERROR("Invalid device %s", device); ++ goto err_out; ++ } ++ if (cdi_parser_validate_device_name(*name) != 0) { ++ ERROR("Invalid device %s", device); ++ goto err_out; ++ } ++ + return 0; ++ ++err_out: ++ free(*vendor); ++ *vendor = NULL; ++ free(*class); ++ *class = NULL; ++ free(*name); ++ *name = NULL; ++ return -1; + } + ++// cdi_parser_parse_device tries to split a device name into vendor, class, and name. + int cdi_parser_parse_device(const char *device, char **vendor, char **class, char **name) + { ++ __isula_auto_array_t char **parts = NULL; ++ ++ if (vendor == NULL || class == NULL || name == NULL || ++ device == NULL || device[0] == '/') { ++ ERROR("Invalid argument"); ++ return -1; ++ } ++ ++ parts = util_string_split_n(device, '=', 2); ++ if (parts == NULL || util_array_len((const char **)parts) != 2 || parts[0] == NULL || parts[1] == NULL) { ++ return -1; ++ } ++ ++ *name = parts[1]; ++ parts[1] = NULL; ++ (void)cdi_parser_parse_qualifier(parts[0], vendor, class); ++ if (*vendor == NULL) { ++ ERROR("Failed to parse device qualifier: %s", parts[0]); ++ return -1; ++ } ++ + return 0; + } + ++/* cdi_parser_parse_qualifier splits a device qualifier into vendor and class. ++ * The syntax for a device qualifier is ++ * ++ * "/" ++ */ + int cdi_parser_parse_qualifier(const char *kind, char **vendor, char **class) + { ++ __isula_auto_array_t char **parts = NULL; ++ ++ if (kind == NULL || vendor == NULL || class == NULL) { ++ ERROR("Invalid argument"); ++ return -1; ++ } ++ ++ parts = util_string_split_n(kind, '/', 2); ++ if (parts == NULL || util_array_len((const char **)parts) != 2 || parts[0] == NULL || parts[1] == NULL) { ++ return -1; ++ } ++ *vendor = parts[0]; ++ parts[0] = NULL; ++ *class = parts[1]; ++ parts[1] = NULL; ++ ++ return 0; ++} ++ ++static int validate_vendor_or_class_name(const char *name) ++{ ++ int i = 0; ++ ++ if (name == NULL) { ++ ERROR("Empty name"); ++ return -1; ++ } ++ ++ if (!isalpha(name[0])) { ++ ERROR("%s, should start with letter", name); ++ return -1; ++ } ++ for (i = 1; name[i] != '\0'; i++) { ++ if (!(isalnum(name[i]) || name[i] == '_' || name[i] == '-' || name[i] == '.')) { ++ ERROR("Invalid character '%c' in name %s", name[i], name); ++ return -1; ++ } ++ } ++ if (!isalnum(name[i - 1])) { ++ ERROR("%s, should end with a letter or digit", name); ++ return -1; ++ } ++ + return 0; + } + +-int cdi_parser_validate_vendor_name(const char *vendor, char **error) ++int cdi_parser_validate_vendor_name(const char *vendor) + { ++ if (validate_vendor_or_class_name(vendor) != 0) { ++ ERROR("Invalid vendor"); ++ return -1; ++ } + return 0; + } + +-int cdi_parser_validate_class_name(const char *class, char **error) ++int cdi_parser_validate_class_name(const char *class) + { ++ if (validate_vendor_or_class_name(class) != 0) { ++ ERROR("Invalid class."); ++ return -1; ++ } + return 0; + } + +-int cdi_parser_validate_device_name(const char *name, char **error) ++int cdi_parser_validate_device_name(const char *name) + { ++ size_t i; ++ ++ if (name == NULL) { ++ ERROR("Invalid (empty) device name"); ++ return -1; ++ } ++ if (!isalnum(name[0])) { ++ ERROR("Invalid class %s, should start with a letter or digit", name); ++ return -1; ++ } ++ if (strlen(name) == 1) { ++ return 0; ++ } ++ for (i = 1; name[i] != '\0'; i++) { ++ if (!(isalnum(name[i]) || name[i] == '_' || name[i] == '-' || name[i] == '.' || name[i] == ':')) { ++ ERROR("Invalid character '%c' in device name %s", name[i], name); ++ return -1; ++ } ++ } ++ if (!isalnum(name[i - 1])) { ++ ERROR("Invalid name %s, should end with a letter or digit", name); ++ return -1; ++ } ++ + return 0; + } +diff --git a/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.h b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.h +index 467641a1..3658e29b 100644 +--- a/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.h ++++ b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.h +@@ -27,9 +27,9 @@ bool cdi_parser_is_qualified_name(const char *device); + int cdi_parser_parse_qualified_name(const char *device, char **vendor, char **class, char **name); + int cdi_parser_parse_device(const char *device, char **vendor, char **class, char **name); + int cdi_parser_parse_qualifier(const char *kind, char **vendor, char **class); +-int cdi_parser_validate_vendor_name(const char *vendor, char **error); +-int cdi_parser_validate_class_name(const char *class, char **error); +-int cdi_parser_validate_device_name(const char *name, char **error); ++int cdi_parser_validate_vendor_name(const char *vendor); ++int cdi_parser_validate_class_name(const char *class); ++int cdi_parser_validate_device_name(const char *name); + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index b4d2b0f6..77ca70f9 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -29,6 +29,7 @@ + #include + #include + #include ++#include + + #include "specs_api.h" + #include "utils.h" +@@ -2595,3 +2596,232 @@ int spec_module_init(void) + return 0; + } + ++#ifdef ENABLE_CDI ++static int add_env(defs_process *dp, const char *env, const char *key) ++{ ++ size_t i; ++ char *oci_key = NULL; ++ char *oci_value = NULL; ++ char *saveptr = NULL; ++ __isula_auto_free char *tmp_env = NULL; ++ ++ for (i = 0; i < dp->env_len; i++) { ++ tmp_env = util_strdup_s(dp->env[i]); ++ oci_key = strtok_r(tmp_env, "=", &saveptr); ++ oci_value = strtok_r(NULL, "=", &saveptr); ++ if (oci_key == NULL || oci_value == NULL) { ++ ERROR("Bad env format"); ++ return -1; ++ } ++ if (strcmp(key, oci_key) == 0) { ++ free(dp->env[i]); ++ dp->env[i] = util_strdup_s(env); ++ return 0; ++ } ++ free(tmp_env); ++ tmp_env = NULL; ++ } ++ if (util_mem_realloc((void **)&dp->env, (dp->env_len + 1) * sizeof(char *), ++ (void *)dp->env, dp->env_len * sizeof(char *)) != 0) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ dp->env[dp->env_len] = util_strdup_s(env); ++ dp->env_len++; ++ return 0; ++} ++ ++int defs_process_add_multiple_env(defs_process *dp, const char **envs, size_t env_len) ++{ ++ size_t i; ++ char *key = NULL; ++ char *value = NULL; ++ char *saveptr = NULL; ++ __isula_auto_free char *tmp_env = NULL; ++ ++ if (envs == NULL || env_len == 0) { ++ DEBUG("empty envs"); ++ return 0; ++ } ++ if (dp == NULL) { ++ ERROR("Invalid params"); ++ return -1; ++ } ++ ++ for (i = 0; i < env_len; i++) { ++ tmp_env = util_strdup_s(envs[i]); ++ key = strtok_r(tmp_env, "=", &saveptr); ++ value = strtok_r(NULL, "=", &saveptr); ++ if (key == NULL || value == NULL) { ++ ERROR("Bad env format: %s", tmp_env); ++ return -1; ++ } ++ if (add_env(dp, envs[i], key) != 0) { ++ return -1; ++ } ++ free(tmp_env); ++ tmp_env = NULL; ++ } ++ ++ return 0; ++} ++ ++int spec_add_multiple_process_env(oci_runtime_spec *oci_spec, const char **envs, size_t env_len) ++{ ++ int ret = 0; ++ ++ if (envs == NULL || env_len == 0) { ++ DEBUG("empty envs"); ++ return 0; ++ } ++ if (oci_spec == NULL) { ++ ERROR("Invalid params"); ++ return -1; ++ } ++ ++ ret = make_sure_oci_spec_process(oci_spec); ++ if (ret < 0) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ ++ ret = defs_process_add_multiple_env(oci_spec->process, envs, env_len); ++ if (ret < 0) { ++ ERROR("Failed to add envs"); ++ } ++ ++ return ret; ++} ++ ++int spec_add_device(oci_runtime_spec *oci_spec, defs_device *device) ++{ ++ int ret = 0; ++ size_t i; ++ ++ if (device == NULL) { ++ return -1; ++ } ++ ret = make_sure_oci_spec_linux(oci_spec); ++ if (ret < 0) { ++ return -1; ++ } ++ ++ for (i = 0; i < oci_spec->linux->devices_len; i++) { ++ if (strcmp(oci_spec->linux->devices[i]->path, device->path) == 0) { ++ free_defs_device(oci_spec->linux->devices[i]); ++ oci_spec->linux->devices[i] = device; ++ return 0; ++ } ++ } ++ ++ if (util_mem_realloc((void **)&oci_spec->linux->devices, (oci_spec->linux->devices_len + 1) * sizeof(char *), ++ (void *)oci_spec->linux->devices, oci_spec->linux->devices_len * sizeof(char *)) != 0) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ oci_spec->linux->devices[oci_spec->linux->devices_len] = device; ++ oci_spec->linux->devices_len++; ++ ++ return 0; ++} ++ ++int spec_add_linux_resources_device(oci_runtime_spec *oci_spec, bool allow, const char *dev_type, ++ int64_t major, int64_t minor, const char *access) ++{ ++ int ret = 0; ++ defs_device_cgroup *device = NULL; ++ ++ ret = make_sure_oci_spec_linux_resources(oci_spec); ++ if (ret < 0) { ++ return -1; ++ } ++ ++ device = util_common_calloc_s(sizeof(*device)); ++ if (device == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ device->allow = allow; ++ device->type = util_strdup_s(dev_type); ++ device->access = util_strdup_s(access); ++ device->major = major; ++ device->minor = minor; ++ ++ if (util_mem_realloc((void **)&oci_spec->linux->resources->devices, (oci_spec->linux->resources->devices_len + 1) * sizeof(char *), ++ (void *)oci_spec->linux->resources->devices, oci_spec->linux->resources->devices_len * sizeof(char *)) != 0) { ++ ERROR("Out of memory"); ++ free_defs_device_cgroup(device); ++ return -1; ++ } ++ oci_spec->linux->resources->devices[oci_spec->linux->resources->devices_len] = device; ++ oci_spec->linux->resources->devices_len++; ++ ++ return 0; ++} ++ ++void spec_remove_mount(oci_runtime_spec *oci_spec, const char *dest) ++{ ++ size_t i; ++ ++ if (oci_spec == NULL || oci_spec->mounts == NULL || dest == NULL) { ++ return; ++ } ++ ++ for (i = 0; i < oci_spec->mounts_len; i++) { ++ if (strcmp(oci_spec->mounts[i]->destination, dest) == 0) { ++ free_defs_mount(oci_spec->mounts[i]); ++ (void)memcpy((void **)&oci_spec->mounts[i], (void **)&oci_spec->mounts[i + 1], ++ (oci_spec->mounts_len - i - 1) * sizeof(void *)); ++ oci_spec->mounts_len--; ++ return; ++ } ++ } ++} ++ ++int spec_add_mount(oci_runtime_spec *oci_spec, defs_mount *mnt) ++{ ++ if (oci_spec == NULL || mnt == NULL) { ++ return -1; ++ } ++ ++ if (util_mem_realloc((void **)&oci_spec->mounts, (oci_spec->mounts_len + 1) * sizeof(char *), ++ (void *)oci_spec->mounts, oci_spec->mounts_len * sizeof(char *)) != 0) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ oci_spec->mounts[oci_spec->mounts_len] = mnt; ++ oci_spec->mounts_len++; ++ ++ return 0; ++} ++ ++#define SPEC_ADD_HOOKS_ITEM_DEF(hooktype) \ ++ int spec_add_##hooktype##_hook(oci_runtime_spec *oci_spec, defs_hook *hooktype##_hook) \ ++ { \ ++ int ret = 0; \ ++ if (oci_spec == NULL || hooktype##_hook == NULL) { \ ++ return -1; \ ++ } \ ++ ret = make_sure_oci_spec_hooks(oci_spec); \ ++ if (ret < 0) { \ ++ return -1; \ ++ } \ ++ if (util_mem_realloc((void **)&oci_spec->hooks->hooktype, (oci_spec->hooks->hooktype##_len + 1) * sizeof(char *), \ ++ (void *)oci_spec->hooks->hooktype, oci_spec->hooks->hooktype##_len * sizeof(char *)) != 0) { \ ++ ERROR("Out of memory"); \ ++ return -1; \ ++ } \ ++ oci_spec->hooks->hooktype[oci_spec->hooks->hooktype##_len] = hooktype##_hook; \ ++ oci_spec->hooks->hooktype##_len++; \ ++ return 0; \ ++ } ++ ++/* ++* The OCI being used by the iSulad not supportes ++* createRuntime/createContainer/startContainer currently. ++*/ ++SPEC_ADD_HOOKS_ITEM_DEF(prestart) ++SPEC_ADD_HOOKS_ITEM_DEF(poststart) ++SPEC_ADD_HOOKS_ITEM_DEF(poststop) ++ ++#endif /* ENABLE_CDI */ +\ No newline at end of file +-- +2.34.1 + diff --git a/0068-cdi-invoke-cdi-operate-when-init-isulad-and-create-c.patch b/0068-cdi-invoke-cdi-operate-when-init-isulad-and-create-c.patch new file mode 100644 index 0000000..3633616 --- /dev/null +++ b/0068-cdi-invoke-cdi-operate-when-init-isulad-and-create-c.patch @@ -0,0 +1,371 @@ +From 0674bfac4dd1ab812432334c779ab718dc54bc8b Mon Sep 17 00:00:00 2001 +From: liuxu +Date: Thu, 11 Apr 2024 11:02:19 +0800 +Subject: [PATCH 68/69] cdi:invoke cdi operate when init isulad and create + container + +--- + src/cmd/isulad/main.c | 11 +++ + src/daemon/common/cri/v1/v1_cri_helpers.cc | 79 +++++++++++++++++++ + src/daemon/common/cri/v1/v1_cri_helpers.h | 3 + + src/daemon/config/daemon_arguments.c | 4 + + src/daemon/config/isulad_config.c | 8 ++ + .../v1/v1_cri_container_manager_service.cc | 8 ++ + .../executor/container_cb/execution_create.c | 9 +++ + .../modules/service/service_container.c | 10 +++ + src/daemon/modules/spec/specs_mount.c | 43 +++++++++- + src/daemon/modules/spec/specs_mount.h | 4 + + src/daemon/modules/spec/verify.c | 2 +- + 11 files changed, 179 insertions(+), 2 deletions(-) + +diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c +index 9fa87bdb..3e2249d7 100644 +--- a/src/cmd/isulad/main.c ++++ b/src/cmd/isulad/main.c +@@ -83,6 +83,9 @@ + #endif + #include "id_name_manager.h" + #include "cgroup.h" ++#ifdef ENABLE_CDI ++#include "cdi_operate_api.h" ++#endif /* ENABLE_CDI */ + + sem_t g_daemon_shutdown_sem; + sem_t g_daemon_wait_shutdown_sem; +@@ -1400,6 +1403,14 @@ static int isulad_server_init_common() + } + #endif + ++#ifdef ENABLE_CDI ++ if (args->json_confs->enable_cdi && ++ cdi_operate_registry_init(args->json_confs->cdi_spec_dirs, args->json_confs->cdi_spec_dirs_len) != 0) { ++ ERROR("Failed to init CDI module"); ++ goto out; ++ } ++#endif /* ENABLE_CDI */ ++ + if (spec_module_init() != 0) { + ERROR("Failed to init spec module"); + goto out; +diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.cc b/src/daemon/common/cri/v1/v1_cri_helpers.cc +index ea5c8bb5..520d23d4 100644 +--- a/src/daemon/common/cri/v1/v1_cri_helpers.cc ++++ b/src/daemon/common/cri/v1/v1_cri_helpers.cc +@@ -22,6 +22,7 @@ + + #include + #include ++#include + + #include "v1_cri_security_context.h" + #include "cri_helpers.h" +@@ -33,6 +34,9 @@ + #include "isulad_config.h" + #include "sha256.h" + #include "v1_naming.h" ++#ifdef ENABLE_CDI ++#include "cdi_operate_api.h" ++#endif /* ENABLE_CDI */ + + namespace CRIHelpersV1 { + +@@ -666,4 +670,79 @@ std::unique_ptr GetContainerStatus(service_executo + return contStatus; + } + ++#ifdef ENABLE_CDI ++static int InsertCDIDevices(std::unordered_set &fromCRI, const std::string &devName, ++ string_array *requested, Errors &err) ++{ ++ if (fromCRI.find(devName) == fromCRI.end()) { ++ fromCRI.insert(devName); ++ if (util_append_string_array(requested, devName.c_str()) != 0) { ++ ERROR("Out of memory"); ++ err.Errorf("Out of memory"); ++ return -1; ++ } ++ DEBUG("Appended device: %s", devName.c_str()); ++ } else { ++ INFO("Skipping duplicate CDI device %s", devName.c_str()); ++ } ++ return 0; ++} ++ ++void GenerateCDIRequestedDevices(const runtime::v1::ContainerConfig &config, host_config *hostconfig, Errors &err) ++{ ++ std::unordered_set fromCRI; ++ __isula_auto_string_array_t string_array *requested = nullptr; ++ __isula_auto_string_array_t string_array *keys = nullptr; ++ __isula_auto_string_array_t string_array *devices = nullptr; ++ json_map_string_string *annotations = nullptr; ++ __isula_auto_free char *error = nullptr; ++ ++ if (hostconfig == nullptr) { ++ ERROR("Invalid input arguments"); ++ err.Errorf("Invalid input arguments"); ++ return; ++ } ++ ++ if (config.cdi_devices().empty() && config.annotations().empty()) { ++ return; ++ } ++ requested = (string_array *)util_common_calloc_s(sizeof(*requested)); ++ if (requested == nullptr) { ++ ERROR("Out of memory"); ++ err.Errorf("Out of memory"); ++ return; ++ } ++ if (!config.cdi_devices().empty()) { ++ for (int i = 0; i < config.cdi_devices().size(); i++) { ++ if (InsertCDIDevices(fromCRI, config.cdi_devices(i).name(), requested, err) != 0) { ++ goto free_out; ++ } ++ } ++ } ++ if (!config.annotations().empty()) { ++ annotations = CRIHelpers::MakeAnnotations(config.annotations(), err); ++ if (err.NotEmpty()) { ++ goto free_out; ++ } ++ if (cdi_operate_parse_annotations(annotations, &keys, &devices, &error) != 0) { ++ ERROR("Failed to parse CDI annotations: %s", error); ++ err.Errorf("Failed to parse CDI annotations: %s", error); ++ goto free_out; ++ } ++ for (size_t i = 0; i < devices->len; i++) { ++ if (InsertCDIDevices(fromCRI, std::string(devices->items[i]), requested, err) != 0) { ++ goto free_out; ++ } ++ } ++ } ++ hostconfig->cdi_requested_devices = requested->items; ++ requested->items = nullptr; ++ hostconfig->cdi_requested_devices_len = requested->len; ++ requested->len = 0; ++ ++free_out: ++ free_json_map_string_string(annotations); ++} ++#endif /* ENABLE_CDI */ ++ + } // v1 namespace CRIHelpers +diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.h b/src/daemon/common/cri/v1/v1_cri_helpers.h +index 1578c428..22cffd0d 100644 +--- a/src/daemon/common/cri/v1/v1_cri_helpers.h ++++ b/src/daemon/common/cri/v1/v1_cri_helpers.h +@@ -79,6 +79,9 @@ std::string CRISandboxerConvert(const std::string &runtime); + + void ApplySandboxSecurityContextToHostConfig(const runtime::v1::LinuxSandboxSecurityContext &context, host_config *hc, + Errors &error); ++#ifdef ENABLE_CDI ++void GenerateCDIRequestedDevices(const runtime::v1::ContainerConfig &config, host_config *hostconfig, Errors &err); ++#endif /* ENABLE_CDI */ + + auto GetContainerStatus(service_executor_t *m_cb, const std::string &containerID, Errors &error) + -> std::unique_ptr; +diff --git a/src/daemon/config/daemon_arguments.c b/src/daemon/config/daemon_arguments.c +index 0ae6268a..ef15934a 100644 +--- a/src/daemon/config/daemon_arguments.c ++++ b/src/daemon/config/daemon_arguments.c +@@ -173,6 +173,10 @@ int service_arguments_init(struct service_arguments *args) + goto free_out; + } + ++#ifdef ENABLE_CDI ++ args->json_confs->enable_cdi = false; ++#endif /* ENABLE_CDI */ ++ + ret = 0; + + free_out: +diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c +index 778ff921..695a0d95 100644 +--- a/src/daemon/config/isulad_config.c ++++ b/src/daemon/config/isulad_config.c +@@ -1830,6 +1830,14 @@ int merge_json_confs_into_global(struct service_arguments *args) + args->json_confs->metrics_port = tmp_json_confs->metrics_port; + #endif + ++#ifdef ENABLE_CDI ++ args->json_confs->enable_cdi = tmp_json_confs->enable_cdi; ++ args->json_confs->cdi_spec_dirs = tmp_json_confs->cdi_spec_dirs; ++ tmp_json_confs->cdi_spec_dirs = NULL; ++ args->json_confs->cdi_spec_dirs_len = tmp_json_confs->cdi_spec_dirs_len; ++ tmp_json_confs->cdi_spec_dirs_len = 0; ++#endif /* ENABLE_CDI */ ++ + out: + free(err); + free_isulad_daemon_configs(tmp_json_confs); +diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +index e86dafae..1097c32c 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +@@ -199,6 +199,14 @@ auto ContainerManagerService::GenerateCreateContainerHostConfig( + } + } + ++#ifdef ENABLE_CDI ++ CRIHelpersV1::GenerateCDIRequestedDevices(containerConfig, hostconfig, error); ++ if (error.NotEmpty()) { ++ ERROR("Failed to generate CDI requested devices"); ++ goto cleanup; ++ } ++#endif /* ENABLE_CDI */ ++ + return hostconfig; + + cleanup: +diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c +index a9102226..785b4e27 100644 +--- a/src/daemon/executor/container_cb/execution_create.c ++++ b/src/daemon/executor/container_cb/execution_create.c +@@ -63,6 +63,7 @@ + #include "runtime_api.h" + #include "id_name_manager.h" + #include "mailbox.h" ++#include "specs_mount.h" + + #ifdef ENABLE_CRI_API_V1 + static bool validate_sandbox_info(const container_sandbox_info *sandbox) +@@ -512,6 +513,14 @@ static oci_runtime_spec *generate_oci_config(host_config *host_spec, const char + goto error_out; + } + ++#ifdef ENABLE_CDI ++ ret = inject_CDI_devcies_for_oci_spec(oci_spec, host_spec); ++ if (ret != 0) { ++ ERROR("Failed to inject CDI devices"); ++ goto error_out; ++ } ++#endif /* ENABLE_CDI */ ++ + return oci_spec; + + error_out: +diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c +index eb7ce4f4..b19a134a 100644 +--- a/src/daemon/modules/service/service_container.c ++++ b/src/daemon/modules/service/service_container.c +@@ -2003,6 +2003,16 @@ static defs_process *make_exec_process_spec(const container_config *container_sp + } + + spec->no_new_privileges = oci_spec->process->no_new_privileges; ++ ++#ifdef ENABLE_CDI ++ // extend step: merge env from oci_spec which comes from injected devices ++ ret = defs_process_add_multiple_env(spec, (const char **)oci_spec->process->env, ++ oci_spec->process->env_len); ++ if (ret != 0) { ++ ERROR("Failed to dup oci env for exec process spec"); ++ goto err_out; ++ } ++#endif /* ENABLE_CDI */ + } + + // for oci runtime: +diff --git a/src/daemon/modules/spec/specs_mount.c b/src/daemon/modules/spec/specs_mount.c +index 50ee9a85..12bd261b 100644 +--- a/src/daemon/modules/spec/specs_mount.c ++++ b/src/daemon/modules/spec/specs_mount.c +@@ -28,6 +28,7 @@ + #include + #include + #include ++#include + #include + #include + +@@ -54,6 +55,9 @@ + #include "volume_api.h" + #include "parse_volume.h" + #include "specs_api.h" ++#ifdef ENABLE_CDI ++#include "cdi_operate_api.h" ++#endif /* ENABLE_CDI */ + + enum update_rw { + update_rw_untouch, +@@ -3582,6 +3586,15 @@ int update_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostcon + oci_spec->linux->resources->devices_len += 1; + } + ++ // extend step: inject CDI devcies ++#ifdef ENABLE_CDI ++ ret = inject_CDI_devcies_for_oci_spec(oci_spec, hostconfig); ++ if (ret != 0) { ++ ERROR("Failed to inject CDI devices"); ++ return -1; ++ } ++#endif /* ENABLE_CDI */ ++ + // Step8: do update devices and cgroup device rules at here + if (hostconfig->privileged) { + // Step8.1: for priviledged container, we should merge all devices under /dev +@@ -3592,4 +3605,32 @@ int update_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostcon + } + + return ret; +-} +\ No newline at end of file ++} ++ ++#ifdef ENABLE_CDI ++int inject_CDI_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostconfig) ++{ ++ int ret = 0; ++ string_array devices_array = { 0 }; ++ __isula_auto_free char *error = NULL; ++ ++ if (oci_spec == NULL || hostconfig == NULL) { ++ ERROR("Invalid params"); ++ return -1; ++ } ++ if (hostconfig->cdi_requested_devices == NULL) { ++ return 0; ++ } ++ devices_array.items = hostconfig->cdi_requested_devices; ++ devices_array.len = hostconfig->cdi_requested_devices_len; ++ devices_array.cap = hostconfig->cdi_requested_devices_len; ++ if (cdi_operate_refresh() != 0) { ++ WARN("CDI registry has errors, please check past logs"); ++ } ++ if (cdi_operate_inject_devices(oci_spec, &devices_array) != 0) { ++ ERROR("Failed to inject CDI devices"); ++ ret = -1; ++ } ++ return ret; ++} ++#endif /* ENABLE_CDI */ +\ No newline at end of file +diff --git a/src/daemon/modules/spec/specs_mount.h b/src/daemon/modules/spec/specs_mount.h +index b742ca35..1406c557 100644 +--- a/src/daemon/modules/spec/specs_mount.h ++++ b/src/daemon/modules/spec/specs_mount.h +@@ -49,6 +49,10 @@ int setup_ipc_dirs(host_config *host_spec, container_config_v2_common_config *v2 + + int update_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostconfig); + ++#ifdef ENABLE_CDI ++int inject_CDI_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostconfig); ++#endif /* ENABLE_CDI */ ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/modules/spec/verify.c b/src/daemon/modules/spec/verify.c +index af790d6e..617b7f23 100644 +--- a/src/daemon/modules/spec/verify.c ++++ b/src/daemon/modules/spec/verify.c +@@ -1518,7 +1518,7 @@ static int verify_custom_mount(defs_mount **mounts, size_t len) + + for (i = 0; i < len; ++i) { + iter = *(mounts + i); +- if (iter == NULL || strcmp(iter->type, MOUNT_TYPE_BIND)) { ++ if (iter == NULL || iter->type == NULL || strcmp(iter->type, MOUNT_TYPE_BIND)) { + continue; + } + +-- +2.34.1 + diff --git a/0069-bugfix-fix-cni_operate_ut-ut.patch b/0069-bugfix-fix-cni_operate_ut-ut.patch new file mode 100644 index 0000000..538fcff --- /dev/null +++ b/0069-bugfix-fix-cni_operate_ut-ut.patch @@ -0,0 +1,24 @@ +From a1f75fd089309d0f8620195ce7e517294be2c410 Mon Sep 17 00:00:00 2001 +From: liuxu +Date: Fri, 19 Apr 2024 18:37:05 +0800 +Subject: [PATCH 69/69] bugfix:fix cni_operate_ut ut + +--- + test/network/cni_operate/CMakeLists.txt | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/test/network/cni_operate/CMakeLists.txt b/test/network/cni_operate/CMakeLists.txt +index 5b4d7c7d..752e5199 100644 +--- a/test/network/cni_operate/CMakeLists.txt ++++ b/test/network/cni_operate/CMakeLists.txt +@@ -14,6 +14,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_string.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_array.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_regex.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_version.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/namespace.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/sha256/sha256.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/path.c +-- +2.34.1 + diff --git a/0070-isolate-sandboxer-code-by-using-macro.patch b/0070-isolate-sandboxer-code-by-using-macro.patch new file mode 100644 index 0000000..4ab8cb2 --- /dev/null +++ b/0070-isolate-sandboxer-code-by-using-macro.patch @@ -0,0 +1,143 @@ +From c1d445e178cd610f8a6d9156012c6c7922eed9c5 Mon Sep 17 00:00:00 2001 +From: xuxuepeng +Date: Sat, 20 Apr 2024 11:24:18 +0800 +Subject: [PATCH 1/2] isolate sandboxer code by using macro + +Signed-off-by: xuxuepeng +--- + cmake/options.cmake | 2 +- + src/daemon/common/cri/v1/v1_cri_helpers.cc | 7 +++++++ + src/daemon/config/isulad_config.c | 2 ++ + src/daemon/sandbox/controller/CMakeLists.txt | 2 +- + src/daemon/sandbox/controller/controller_manager.cc | 6 ++++++ + src/daemon/sandbox/controller/controller_manager.h | 2 ++ + 6 files changed, 19 insertions(+), 2 deletions(-) + +diff --git a/cmake/options.cmake b/cmake/options.cmake +index c1eac472..a15b8194 100644 +--- a/cmake/options.cmake ++++ b/cmake/options.cmake +@@ -51,7 +51,7 @@ if (ENABLE_CDI STREQUAL "ON") + endif() + endif() + +-option(ENABLE_SANDBOXER "Enable sandbox API" ON) ++option(ENABLE_SANDBOXER "Enable sandbox API" OFF) + if (ENABLE_SANDBOXER STREQUAL "ON") + add_definitions(-DENABLE_SANDBOXER) + set(ENABLE_SANDBOXER 1) +diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.cc b/src/daemon/common/cri/v1/v1_cri_helpers.cc +index 520d23d4..1f797ad7 100644 +--- a/src/daemon/common/cri/v1/v1_cri_helpers.cc ++++ b/src/daemon/common/cri/v1/v1_cri_helpers.cc +@@ -391,6 +391,7 @@ void GetContainerSandboxID(const std::string &containerID, std::string &realCont + realContainerID = info->id; + } + ++#ifdef ENABLE_SANDBOXER + std::string CRISandboxerConvert(const std::string &runtime) + { + std::string sandboxer; +@@ -429,6 +430,12 @@ out: + (void)isulad_server_conf_unlock(); + return sandboxer; + } ++#else ++std::string CRISandboxerConvert(const std::string &runtime) ++{ ++ return DEFAULT_SANDBOXER_NAME; ++} ++#endif + + void ApplySandboxSecurityContextToHostConfig(const runtime::v1::LinuxSandboxSecurityContext &context, host_config *hc, + Errors &error) +diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c +index 695a0d95..617db7a2 100644 +--- a/src/daemon/config/isulad_config.c ++++ b/src/daemon/config/isulad_config.c +@@ -1757,8 +1757,10 @@ int merge_json_confs_into_global(struct service_arguments *args) + args->json_confs->runtimes = tmp_json_confs->runtimes; + tmp_json_confs->runtimes = NULL; + #ifdef ENABLE_CRI_API_V1 ++#ifdef ENABLE_SANDBOXER + args->json_confs->cri_sandboxers = tmp_json_confs->cri_sandboxers; + tmp_json_confs->cri_sandboxers = NULL; ++#endif + args->json_confs->enable_cri_v1 = tmp_json_confs->enable_cri_v1; + args->json_confs->enable_pod_events = tmp_json_confs->enable_pod_events; + #endif +diff --git a/src/daemon/sandbox/controller/CMakeLists.txt b/src/daemon/sandbox/controller/CMakeLists.txt +index f846657a..8764c05b 100644 +--- a/src/daemon/sandbox/controller/CMakeLists.txt ++++ b/src/daemon/sandbox/controller/CMakeLists.txt +@@ -9,7 +9,7 @@ set(local_sandbox_controller_top_incs + ${CMAKE_CURRENT_SOURCE_DIR} + ) + +-if (ENABLE_SANDBOXER) ++if (ENABLE_CRI_API_V1 AND ENABLE_SANDBOXER) + add_subdirectory(sandboxer) + list (APPEND local_sandbox_controller_top_srcs + ${CONTROLLER_SANDBOXER_SRCS} +diff --git a/src/daemon/sandbox/controller/controller_manager.cc b/src/daemon/sandbox/controller/controller_manager.cc +index 21c6f5fe..91c98d26 100644 +--- a/src/daemon/sandbox/controller/controller_manager.cc ++++ b/src/daemon/sandbox/controller/controller_manager.cc +@@ -20,7 +20,9 @@ + #include + + #include "shim_controller.h" ++#ifdef ENABLE_SANDBOXER + #include "sandboxer_controller.h" ++#endif + #include "isulad_config.h" + #include "daemon_arguments.h" + +@@ -44,10 +46,12 @@ bool ControllerManager::Init(Errors &error) + return false; + } + ++#ifdef ENABLE_SANDBOXER + // Initialize sandboxer controller + if (!RegisterAllSandboxerControllers(error)) { + return false; + } ++#endif + return true; + } + +@@ -75,6 +79,7 @@ auto ControllerManager::RegisterShimController(Errors &error) -> bool + return true; + } + ++#ifdef ENABLE_SANDBOXER + auto ControllerManager::RegisterAllSandboxerControllers(Errors &error) -> bool + { + std::map config; +@@ -160,6 +165,7 @@ auto ControllerManager::RegisterSandboxerController(const std::string &sandboxer + INFO("Sandboxer controller initialized successfully, sandboxer: %s", sandboxer.c_str()); + return true; + } ++#endif + + auto ControllerManager::GetController(const std::string &name) -> std::shared_ptr + { +diff --git a/src/daemon/sandbox/controller/controller_manager.h b/src/daemon/sandbox/controller/controller_manager.h +index 28b52c2f..3fd547cf 100644 +--- a/src/daemon/sandbox/controller/controller_manager.h ++++ b/src/daemon/sandbox/controller/controller_manager.h +@@ -31,9 +31,11 @@ public: + auto GetController(const std::string &name) -> std::shared_ptr; + private: + auto RegisterShimController(Errors &error) -> bool; ++#ifdef ENABLE_SANDBOXER + auto RegisterAllSandboxerControllers(Errors &error) -> bool; + auto LoadSandboxerControllersConfig(std::map &config) -> bool; + auto RegisterSandboxerController(const std::string &sandboxer, const std::string &address, Errors &error) -> bool; ++#endif + + protected: + std::map> m_controllers; +-- +2.34.1 + diff --git a/0071-Remove-sandboxer-ut-if-sandboxer-is-not-enabled.patch b/0071-Remove-sandboxer-ut-if-sandboxer-is-not-enabled.patch new file mode 100644 index 0000000..dbfbe5d --- /dev/null +++ b/0071-Remove-sandboxer-ut-if-sandboxer-is-not-enabled.patch @@ -0,0 +1,28 @@ +From 7c0c79e8ad4680f97651dd52721344961c803c15 Mon Sep 17 00:00:00 2001 +From: xuxuepeng +Date: Sat, 20 Apr 2024 11:48:56 +0800 +Subject: [PATCH 71/73] Remove sandboxer ut if sandboxer is not enabled + +Signed-off-by: xuxuepeng +--- + test/sandbox/CMakeLists.txt | 8 +++++--- + 1 file changed, 5 insertions(+), 3 deletions(-) + +diff --git a/test/sandbox/CMakeLists.txt b/test/sandbox/CMakeLists.txt +index 68397477..38d7ccb9 100644 +--- a/test/sandbox/CMakeLists.txt ++++ b/test/sandbox/CMakeLists.txt +@@ -1,5 +1,7 @@ + project(iSulad_UT) + +-add_subdirectory(controller) +-add_subdirectory(sandbox_manager) +-add_subdirectory(sandbox) ++if (ENABLE_SANDBOXER) ++ add_subdirectory(controller) ++ add_subdirectory(sandbox_manager) ++ add_subdirectory(sandbox) ++endif() +-- +2.34.1 + diff --git a/0072-cdi-design-doc.patch b/0072-cdi-design-doc.patch new file mode 100644 index 0000000..78c3116 --- /dev/null +++ b/0072-cdi-design-doc.patch @@ -0,0 +1,391 @@ +From 72246a9e83ada3af7560817534013a93f40d5bae Mon Sep 17 00:00:00 2001 +From: liuxu +Date: Sat, 20 Apr 2024 11:46:11 +0800 +Subject: [PATCH 72/73] cdi:design doc + +Signed-off-by: liuxu +--- + docs/design/README.md | 3 + + docs/design/README_zh.md | 3 + + docs/design/detailed/CDI/cdi_design_zh.md | 341 ++++++++++++++++++++++ + 3 files changed, 347 insertions(+) + create mode 100644 docs/design/detailed/CDI/cdi_design_zh.md + +diff --git a/docs/design/README.md b/docs/design/README.md +index cf29c0a1..d2a3702d 100644 +--- a/docs/design/README.md ++++ b/docs/design/README.md +@@ -18,6 +18,9 @@ This section contains some design documents for users who want to learn more abo + + - You can see how the CRI startup process is refactored in [cri_cni_refactor](./detailed/CRI/cri_cni_refactor.md). + ++## CDI ++- You can see how the CDI is refactored in [cdi_design](./detailed/CDI/cdi_design_zh.md.md)。 ++ + ## Events + + - You can see how the events modules are designed in [events_design](./detailed/Events/events_design.md). +diff --git a/docs/design/README_zh.md b/docs/design/README_zh.md +index 3382bfbe..c6172b6f 100644 +--- a/docs/design/README_zh.md ++++ b/docs/design/README_zh.md +@@ -24,6 +24,9 @@ + + - 查看 CRI的启动程序的重构文档： [cri_cni_refactor](./detailed/CRI/cri_cni_refactor_zh.md) 。 + ++## CDI ++- 查看 CDI 的设计文档： [cdi_design](./detailed/CDI/cdi_design_zh.md.md)。 ++ + ## Events + + - 查看 events 模块的设计文档： [events_design](./detailed/Events/events_design_zh.md) 。 +diff --git a/docs/design/detailed/CDI/cdi_design_zh.md b/docs/design/detailed/CDI/cdi_design_zh.md +new file mode 100644 +index 00000000..15c88f93 +--- /dev/null ++++ b/docs/design/detailed/CDI/cdi_design_zh.md +@@ -0,0 +1,341 @@ ++| Author | liuxu | ++| ------ | --------------------- | ++| Date | 2024-02-27 | ++| Email | liuxu156@huawei.com | ++ ++# 背景介绍 ++## What is CDI? ++CDI（容器设备接口）是容器运行时的一种规范，用于支持第三方设备。 ++ ++它引入了device作为资源的抽象概念。device由完全限定的名称唯一指定，该名称由vendor ID、一个device class和在每个vendor ID-device class对中唯一的name构成。 ++``` ++vendor.com/class=unique_name ++``` ++vendor ID和device class （上例中的vendor.com/class）的组合称为device kind. ++ ++CDI只关心使容器能够感知设备。CDI明确地忽略了诸如资源管理之类的领域（并期望由编排器处理）。由于这个原因，CDI规范实现起来很简单，并且为运行时和编排器提供了极大的灵活性。 ++ ++注：CDI模型基于容器网络接口（CNI）模型和规范。 ++ ++## Why is CDI needed? ++在Linux上，使容器具有设备感知能力过去只需在该容器中暴露一个设备节点。但是，随着设备和软件变得越来越复杂，供应商希望执行更多的操作，例如： ++- 向容器公开设备可能需要公开多个设备节点、从运行时命名空间挂载文件或隐藏procfs条目。 ++- 执行容器和设备之间的兼容性检查（例如：检查容器是否可以在指定设备上运行）。 ++- 执行特定于运行时的操作（例如：虚拟机与基于Linux容器的运行时）。 ++- 执行特定于设备的操作（例如：清理GPU的内存或重新配置FPGA）。 ++ ++在缺乏第三方设备标准的情况下，供应商通常不得不为不同的运行时编写和维护多个插件，甚至直接在运行时中贡献特定于供应商的代码。此外，运行时不统一地暴露插件系统（甚至根本不暴露插件系统），导致在更高级别的抽象（例如Kubernetes设备插件）中重复功能。 ++ ++## How does CDI work? ++要使CDI正常工作，需要完成以下操作： ++- JSON格式的CDI文件应位于CDI规范目录中，它的作用是更新OCI spec。默认目录为/etc/cdi和/var/run/cdi ++- 应使用CRI的annotations与CDI_devices（本次特性支持）将唯一的设备名称传递给运行时 ++- 容器运行时应该能够通过设备名称找到CDI文件在内存中对应的缓存信息，并使用缓存的内容更新容器配置。 ++ ++## How to configure CDI? ++### iSulad ++daemon.json中开启cri-v1和cdi配置： ++ ++当 cdi-spec-dirs 不指定时，默认为"/etc/cdi", "/var/run/cdi" ++```json ++"enable-cri-v1": true, ++"cdi-spec-dirs": ["/etc/cdi", "/var/run/cdi"], # 指定CDI规范所在目录 ++"enable-cdi": true # 打开CDI特性 ++``` ++ ++在CRI创建容器的参数中使用CDI，以下两种方式均可： ++1. annotations中指定设备 ++```json ++{ ++ ... ... ++ "annotations": [ ++ ... ... ++ // key值格式要求含有cdi.k8s.io作为前缀，后面跟随pluginName ++ {"cdi.k8s.io/test": "vendor.com/device=myDevice"}, ++ ... ... ++ ] ++ ... ... ++} ++``` ++2. CDI_Devices中指定设备 ++```json ++{ ++ ... ... ++ "CDI_Devices": [ ++ ... ... ++ {"Name": "vendor.com/device=myDevice"}, ++ ... ... ++ ] ++ ... ... ++} ++``` ++ ++# 方案目标 ++## 概述 ++容器设备接口（Container Device Interface，简称CDI）描述了容器运行时创建能够与第三方设备交互的容器的机制。 ++ ++对于第三方设备，与这些设备进行交互通常需要容器运行时公开多个设备节点。例如，第三方设备可能需要加载内核模块、装载主机库、暴露/屏蔽特定procfs路径。 ++ ++容器设备接口描述了一种允许第三方供应商执行这些操作的机制，从而不需要更改容器运行时。 ++ ++使用的机制是一个JSON文件（类似于容器网络接口(CNI）)，它允许供应商描述容器运行时应该对容器的OCI规范执行的操作。 ++ ++CDI 支持以下两个流程： ++ ++A.设备安装 ++1. 用户在机器上安装第三方设备驱动程序（和第三方设备/被测试的设备）。 ++2. 设备驱动程序安装软件会在一个已知路径（/etc/cdi/vendor.json）上写入一个 JSON 文件。 ++ ++B.容器运行时 ++1. 用户在创建容器时，CRI中指定设备名称 ++2. 容器运行时会读取 JSON 文件。 ++3. 容器运行时会验证 JSON 文件中是否描述了设备。 ++4. 容器运行时会根据 JSON 文件中的指令转换 OCI 规范，并插入OCI Spec中 ++# 总体设计 ++**iSulad支持CDI功能目前仅支持CRI方式调用，CLI方式暂不支持** ++## 整体结构 ++ ++```mermaid ++flowchart TD ++classDef unFinish fill:#c19,stroke:#216,stroke-width:2px,color:#fff,stroke-dasharray: 5 5; ++subgraph isulad ++ subgraph CDIOperate ++ RegistryOps ++ AnnotationsOps ++ ...... ++ end ++ ++ subgraph CDI ++ D[ContainerEdits] ++ E[Device] ++ F[Registry] ++ G[Cache] ++ H[Spec] ++ end ++ CDIOperate --> CDI ++ ++ OA[isula module] --> |call| CDIOperate ++ OB[CRI module] --> |call| CDIOperate ++ CDIOperate:::unFinish ++ CDI:::unFinish ++end ++ PA[CLI] --> OA ++ PB[CRI] --> OB ++``` ++ ++- CDIOperate封装了CDI模块，对外提供更合理的CDI功能的相关接口。 ++- CDI负责实现CDI Specs的读取、校验、解析、devices注入OCI Spec等具体功能。 ++## 时序设计 ++在isulad启动后，以创建一个容器为例。 ++ ++图中isulad启动后，拉起一个新的线程isulad-cdi_watcher，负责监控cdi-spec-dirs，当cdi-spec-dirs中的cdi Spec文件发生修改、删除等动作时，重新扫描cdi-spec-dirs中的cdi Spec文件。 ++ ++```mermaid ++sequenceDiagram ++ participant CRI ++ participant isulad ++ participant cdi_watcher ++ isulad ->> cdi_watcher: init cdi ++ par ++ loop ++ cdi_watcher ->> cdi_watcher:wait for inotify event ++ cdi_watcher ->> cdi_watcher:refresh cdi cache ++ end ++ and ++ opt CRI ++ CRI ->> isulad: create container ++ isulad ->> isulad:parse cdi annotations/CDI_devices ++ isulad ->> isulad:generates an OCI specification with cdi info ++ end ++ end ++``` ++ ++# 接口描述 ++## 3.1 结构体和常量说明 ++ ++```c ++// 实现CDI Spec规范中的json字段，参考 https://github1s.com/containerd/containerd/blob/main/vendor/tags.cncf.io/container-device-interface/specs-go/config.go#L9 ++typedef struct {} cdi_spec; ++typedef struct {} cdi_spec_device; ++typedef struct {} cdi_spec_container_edits; ++typedef struct {} cdi_spec_device_node; ++typedef struct {} cdi_spec_mount; ++typedef struct {} cdi_spec_hook; ++ ++struct cdi_cache_device { ++ const cdi_device *raw_device; ++ const struct cdi_cache_spec *cache_spec; ++}; ++ ++struct cdi_cache_spec { ++ cdi_spec *raw_spec; ++ char *vendor; ++ char *class; ++ char *path; ++ int priority; ++ map_t *devices; // MAP_STR_PTR devices[cdi_device.name] = cdi_cache_device* ++}; ++ ++struct cdi_cache_ops { ++ // injecting CDI devices into an OCI Spec. ++ // Resolver ++ int (*inject_devices)(struct cdi_cache *c, oci_runtime_spec *spec, string_array *devices); ++ ++ // refreshing the cache of CDI Specs and devices. ++ // Refresher ++ int (*configure)(struct cdi_cache *c, string_array *spec_dirs); ++ int (*refresh)(struct cdi_cache *c); ++}; ++ ++struct cdi_watch { ++ int watcher_fd; // inotify fd ++ map_t *tracked; // MAP_STR_BOOL tracked[spec_dirs[i]] = bool ++ map_t *wd_dirs; // MAP_INT_STR wd_dirs[wd] = spec_dirs[i] ++}; ++ ++// Cache stores CDI Specs loaded from Spec directories. ++struct cdi_cache { ++ pthread_mutex_t mutex; ++ string_array *spec_dirs; // cdi-spec-dirs will scan for CDI Spec files ++ map_t *specs; // MAP_STR_PTR specs[vendor] = common_array of cdi_cache_spec* ++ // This map holding the reference to cdi device, the devices will not released when the map is freed. ++ map_t *devices; // MAP_STR_PTR devices[cdi_device.name] = cdi_cache_device* ++ bool refresh_error_flag; ++ bool auto_refresh; ++ struct cdi_watch *watch; ++}; ++ ++struct cdi_registry { ++ struct cdi_cache *cdi_cache; ++ struct cdi_cache_ops *ops; ++}; ++``` ++涉及修改现有的结构体： ++```c ++// 用于从CRI向executor传递devices数据 ++typedef struct { ++ ... ... ++ char **cdi_requested_devices; ++ size_t cdi_requested_devices_len; ++} host_config; ++ ++// isulad的daemon.json增加cdi相关的基本配置 ++typedef struct { ++ ... ... ++ char **cdi_spec_dirs; ++ size_t cdi_spec_dirs_len; ++ bool enable_cdi; ++} isulad_daemon_configs; ++``` ++ ++## 3.2 接口说明 ++CDIOperate的设计目标：使得CDI规范有关的内容不过多的对外暴露，降低CDI模块和外部的耦合。 ++### RegistryOps ++```c ++int cdi_operate_registry_init(char **specs_dirs, size_t specs_dirs_len); ++ ++int cdi_operate_refresh(void); ++ ++int cdi_operate_inject_devices(oci_runtime_spec *spec, string_array *devices); ++ ++int cdi_operate_parse_annotations(json_map_string_string *annotations, string_array **keys, ++ string_array **devices, char **error); ++``` ++ ++# 详细设计 ++## daemon.json ++daemon.json中增加cdi配置： ++```json ++"cdi-spec-dirs": ["/etc/cdi", "/var/run/cdi"], ++"enable-cdi": true ++``` ++## CDIOperate被调用点 ++### CreateContainer ++创建新的Container时需要将devices插入OCI Spec。 ++本次开发暂不涉及isula支持CDI。 ++ ++为什么已经有isulad-cdi_watcher线程了，在create的时候还需要refresh? ++1. isulad-cdi_watcher在触发inotify event后，如果为Rename、Remove、Write事件，会直接执行refresh，将重新扫描cdi-spec-dirs中的CDI Specs到内存。 ++2. 而在create container时，cdi_refresh先检查tracked，tracked标记了是否所有目录都已被跟踪，如果不是才会触发refresh。tracked在cache生成时初始化为未跟踪。 ++ ++```mermaid ++flowchart TD ++O((begin)) ++A(ContainerManagerService::CreateContainer
CRI创建容器) ++B(ContainerManagerService::CDIParseDevices
解析CDI有关的字段) ++C(container_cb::container_create_cb
创建容器的前置准备) ++D(spec::merge_all_specs
准备生成OCI Spec) ++E(spec::spec_inject_cdi_devices
解析需要的设备名) ++F(CDIOperate::cdi_refresh
刷新cdi的缓存) ++G(CDIOperate::cdi_inject_devices
将容器需要的设备的相关信息插入OCI Spec) ++Z((create end)) ++O --> A --> B --> C --> D --> E --> F --> G --> Z ++``` ++ ++ContainerManagerService::CDIParseDevices 需要支持两种CDI devices解析方式： ++1. CDIOperate::cdi_parse_annotations 解析ContainerConfig.annotations() ++2. 解析ContainerConfig.CDI_devices() ++解析后放入container_create_request ++### main ++isulad 启动时，读取所有CDI Specs，初始化cache ++```mermaid ++flowchart TD ++O((begin)) ++A(main) ++B(isulad_server_init_common
isulad服务端初始化) ++C(cdi::cdi_registry_init
读取现有的CDI Specs
并拉起新的线程监控CDI Specs的变化) ++Z(end) ++O --> A --> B --> C --> Z ++``` ++ ++## CDI 模块设计 ++```mermaid ++flowchart TD ++ subgraph CDIoperate ++ operate ++ end ++ ++ subgraph CDI ++ annotations ++ registry ++ cache ++ subgraph behavior ++ spec-dirs ++ spec ++ device ++ container-edits ++ version ++ parser ++ end ++ end ++ ++ operate --> registry ++ operate --> annotations ++ registry --> cache ++ cache --> spec-dirs ++ cache --> device ++ cache --> spec ++ cache --> container-edits ++ device <--> spec ++ device --> parser ++ device --> container-edits ++ device --> parser ++ spec --> container-edits ++ spec --> version ++ version --> parser ++``` ++ ++### new cache ++```mermaid ++flowchart TD ++O((begin)) ++A(ContainerManagerService::CreateContainer
CRI创建容器) ++B(ContainerManagerService::CDIParseDevices
解析CDI有关的字段) ++C(container_cb::container_create_cb
创建容器的前置准备) ++D(spec::merge_all_specs
准备生成OCI Spec) ++E(spec::spec_inject_cdi_devices
解析需要的设备名) ++F(CDIOperate::cdi_refresh
刷新cdi的缓存) ++G(CDIOperate::cdi_inject_devices
将容器需要的设备的相关信息插入OCI Spec) ++Z((create end)) ++O --> A --> B --> C --> D --> E --> F --> G --> Z ++``` +-- +2.34.1 + diff --git a/0073-bugfix-cdi-version-check.patch b/0073-bugfix-cdi-version-check.patch new file mode 100644 index 0000000..00c1b9a --- /dev/null +++ b/0073-bugfix-cdi-version-check.patch @@ -0,0 +1,52 @@ +From 4d8a89e0a3700253db044f6641d91ab10ad6ce10 Mon Sep 17 00:00:00 2001 +From: liuxu +Date: Sat, 20 Apr 2024 19:36:55 +0800 +Subject: [PATCH 73/73] bugfix: cdi version check + +Signed-off-by: liuxu +--- + src/daemon/modules/device/cdi/behavior/cdi_spec.c | 2 +- + src/daemon/modules/device/cdi/behavior/cdi_version.c | 6 +++--- + 2 files changed, 4 insertions(+), 4 deletions(-) + +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec.c b/src/daemon/modules/device/cdi/behavior/cdi_spec.c +index 235b1863..8783debc 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_spec.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec.c +@@ -199,7 +199,7 @@ static int cdi_spec_init(struct cdi_cache_spec *s) + return -1; + } + if (version_result) { +- ERROR("The spec version must be at least v%s", min_version); ++ ERROR("The %s spec version must be at least v%s", s->path, min_version); + return -1; + } + +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_version.c b/src/daemon/modules/device/cdi/behavior/cdi_version.c +index 882a965e..550f3107 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_version.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_version.c +@@ -138,9 +138,9 @@ static struct required_version_map g_valid_spec_versions[VALID_SPEC_VERSIONS_LEN + {CDI_V010, NULL}, + {CDI_V020, NULL}, + {CDI_V030, NULL}, +- {CDI_V040, requires_v060}, ++ {CDI_V040, requires_v040}, + {CDI_V050, requires_v050}, +- {CDI_V060, requires_v040} ++ {CDI_V060, requires_v060} + }; + + const char *cdi_minimum_required_version(cdi_spec *spec) +@@ -166,7 +166,7 @@ const char *cdi_minimum_required_version(cdi_spec *spec) + min_version = g_valid_spec_versions[i].version; + } + } +- if (strcmp(min_version, CDI_CURRENT_VERSION)) { ++ if (strcmp(min_version, CDI_CURRENT_VERSION) == 0) { + break; + } + } +-- +2.34.1 + diff --git a/0074-bugfix-of-background-execution-exec-error-command.patch b/0074-bugfix-of-background-execution-exec-error-command.patch new file mode 100644 index 0000000..6aca398 --- /dev/null +++ b/0074-bugfix-of-background-execution-exec-error-command.patch @@ -0,0 +1,33 @@ +From 2c86e55d98b0d62c534ff5810c1eb1d327d6425a Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Tue, 23 Apr 2024 17:44:00 +1400 +Subject: [PATCH 74/75] bugfix of background execution exec error command + +Signed-off-by: zhongtao +--- + src/daemon/modules/runtime/isula/isula_rt_ops.c | 9 +++++++++ + 1 file changed, 9 insertions(+) + +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 47a14b1d..854752ea 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -1099,6 +1099,15 @@ static int get_container_process_pid(const char *workdir) + util_usleep_nointerupt(100000); + continue; + } ++ // If isulad does not read the container process pid, but isulad-shim reads the pid, ++ // and the container process exits, isulad-shim exits accordingly. ++ // At this time, exec should return true, because the container process has been created successfully ++ // and exec is successful, just because The process executes too fast causing isulad to not be read correctly ++ file_read_int(fname, &pid); ++ if (pid != 0) { ++ DEBUG("Process exit and isulad-shim exit"); ++ return pid; ++ } + ERROR("failed read pid from dead shim %s", workdir); + return -1; + } +-- +2.25.1 + diff --git a/0075-bugfix-for-setting-cpu-rt-to-a-negative-value-when-e.patch b/0075-bugfix-for-setting-cpu-rt-to-a-negative-value-when-e.patch new file mode 100644 index 0000000..7ea77a4 --- /dev/null +++ b/0075-bugfix-for-setting-cpu-rt-to-a-negative-value-when-e.patch @@ -0,0 +1,35 @@ +From bd18051dade97d4f75346cb67beea551a38ca13e Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Tue, 23 Apr 2024 20:52:04 +0800 +Subject: [PATCH 75/75] bugfix for setting cpu-rt to a negative value when env + not supports cpu-rt + +Signed-off-by: zhongtao +--- + src/daemon/modules/spec/verify.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/src/daemon/modules/spec/verify.c b/src/daemon/modules/spec/verify.c +index 617b7f23..57e16ef9 100644 +--- a/src/daemon/modules/spec/verify.c ++++ b/src/daemon/modules/spec/verify.c +@@ -381,14 +381,14 @@ static int verify_cpu_realtime(const sysinfo_t *sysinfo, int64_t realtime_period + { + int ret = 0; + +- if (realtime_period > 0 && !(sysinfo->cgcpuinfo.cpu_rt_period)) { ++ if (realtime_period != 0 && !(sysinfo->cgcpuinfo.cpu_rt_period)) { + ERROR("Invalid --cpu-rt-period: Your kernel does not support cgroup rt period"); + isulad_set_error_message("Invalid --cpu-rt-period: Your kernel does not support cgroup rt period"); + ret = -1; + goto out; + } + +- if (realtime_runtime > 0 && !(sysinfo->cgcpuinfo.cpu_rt_runtime)) { ++ if (realtime_runtime != 0 && !(sysinfo->cgcpuinfo.cpu_rt_runtime)) { + ERROR("Invalid --cpu-rt-runtime: Your kernel does not support cgroup rt runtime"); + isulad_set_error_message("Invalid --cpu-rt-period: Your kernel does not support cgroup rt runtime"); + ret = -1; +-- +2.25.1 + diff --git a/0076-cdi-add-UT.patch b/0076-cdi-add-UT.patch new file mode 100644 index 0000000..678b578 --- /dev/null +++ b/0076-cdi-add-UT.patch @@ -0,0 +1,741 @@ +From 0cd088174c94c56ee86506dab9a6a33f6e8fdaa4 Mon Sep 17 00:00:00 2001 +From: liuxu +Date: Thu, 25 Apr 2024 10:52:20 +0800 +Subject: [PATCH 76/78] cdi:add UT + +Signed-off-by: liuxu +--- + CI/make-and-install.sh | 4 +- + test/cutils/CMakeLists.txt | 1 + + test/cutils/utils_array/utils_array_ut.cc | 128 ++++++++++ + test/cutils/utils_utils/utils_utils_ut.cc | 12 + + test/cutils/utils_version/CMakeLists.txt | 17 ++ + test/cutils/utils_version/utils_version_ut.cc | 71 ++++++ + .../image/oci/oci_config_merge/CMakeLists.txt | 1 + + test/mocks/cdi_operate_api_mock.cc | 58 +++++ + test/mocks/cdi_operate_api_mock.h | 35 +++ + test/sandbox/controller/shim/CMakeLists.txt | 1 + + test/specs/specs/CMakeLists.txt | 1 + + test/specs/specs/specs_ut.cc | 241 ++++++++++++++++++ + test/specs/specs_extend/CMakeLists.txt | 1 + + 13 files changed, 569 insertions(+), 2 deletions(-) + create mode 100644 test/cutils/utils_version/CMakeLists.txt + create mode 100644 test/cutils/utils_version/utils_version_ut.cc + create mode 100644 test/mocks/cdi_operate_api_mock.cc + create mode 100644 test/mocks/cdi_operate_api_mock.h + +diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh +index 9bb984cd..9d4c5533 100755 +--- a/CI/make-and-install.sh ++++ b/CI/make-and-install.sh +@@ -72,7 +72,7 @@ cd $ISULAD_COPY_PATH + sed -i 's/fd == STDIN_FILENO || fd == STDOUT_FILENO || fd == STDERR_FILENO/fd == 0 || fd == 1 || fd == 2 || fd >= 1000/g' ./src/utils/cutils/utils.c + rm -rf build + mkdir build && cd build +-cmake -DCMAKE_BUILD_TYPE=Debug -DENABLE_UT=ON -DENABLE_CRI_API_V1=ON -DENABLE_SHIM_V2=ON -DENABLE_METRICS=ON .. ++cmake -DCMAKE_BUILD_TYPE=Debug -DENABLE_UT=ON -DENABLE_CRI_API_V1=ON -DENABLE_CDI=ON -DENABLE_SHIM_V2=ON -DENABLE_METRICS=ON .. + make -j $(nproc) + make install + ctest -E "driver_devmapper_ut" -T memcheck --output-on-failure +@@ -103,7 +103,7 @@ rm -rf build + mkdir build + cd build + if [[ ${enable_gcov} -ne 0 ]]; then +- cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_CRI_API_V1=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON -DENABLE_GRPC_REMOTE_CONNECT=ON .. ++ cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_CRI_API_V1=ON -DENABLE_CDI=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON -DENABLE_GRPC_REMOTE_CONNECT=ON .. + else + cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON -DENABLE_CRI_API_V1=ON -DENABLE_GRPC_REMOTE_CONNECT=ON .. + fi +diff --git a/test/cutils/CMakeLists.txt b/test/cutils/CMakeLists.txt +index 9e681cc9..bd9def02 100644 +--- a/test/cutils/CMakeLists.txt ++++ b/test/cutils/CMakeLists.txt +@@ -34,4 +34,5 @@ add_subdirectory(utils_utils) + add_subdirectory(utils_verify) + add_subdirectory(utils_network) + add_subdirectory(utils_transform) ++add_subdirectory(utils_version) + add_subdirectory(map) +diff --git a/test/cutils/utils_array/utils_array_ut.cc b/test/cutils/utils_array/utils_array_ut.cc +index 7bd13c25..6c6e76fa 100644 +--- a/test/cutils/utils_array/utils_array_ut.cc ++++ b/test/cutils/utils_array/utils_array_ut.cc +@@ -50,6 +50,33 @@ TEST(utils_array, test_util_free_array) + util_free_array(array); + } + ++TEST(utils_array, test_util_copy_array_by_len) ++{ ++ char **array = nullptr; ++ char **array_copy = nullptr; ++ size_t len = 3; ++ ++ array = (char **)util_common_calloc_s(4 * sizeof(char *)); ++ ASSERT_NE(array, nullptr); ++ array[0] = util_strdup_s("test1"); ++ array[1] = util_strdup_s("test2"); ++ array[2] = util_strdup_s("test3"); ++ ++ array_copy = util_copy_array_by_len(array, len); ++ ASSERT_NE(array_copy, nullptr); ++ for (size_t i = 0; i < len; i++) { ++ ASSERT_EQ(strcmp(array_copy[i], array[i]), 0); ++ free(array[i]); ++ free(array_copy[i]); ++ } ++ ++ ASSERT_EQ(util_copy_array_by_len(array, 0), nullptr); ++ ASSERT_EQ(util_copy_array_by_len(nullptr, len), nullptr); ++ ++ free(array); ++ free(array_copy); ++} ++ + TEST(utils_array, test_util_grow_array) + { + char **array = nullptr; +@@ -229,6 +256,34 @@ TEST(utils_array, test_util_append_string_array) + sarray = nullptr; + } + ++TEST(utils_array, test_util_copy_string_array) ++{ ++ __isula_auto_string_array_t string_array *sarray_copy = nullptr; ++ __isula_auto_string_array_t string_array *sarray = (string_array *)util_common_calloc_s(sizeof(string_array)); ++ ASSERT_NE(sarray, nullptr); ++ int ret; ++ ++ ret = util_append_string_array(sarray, "1234567890"); ++ ASSERT_EQ(ret, 0); ++ ret = util_append_string_array(sarray, "abc"); ++ ASSERT_EQ(ret, 0); ++ ret = util_append_string_array(sarray, "bcd"); ++ ASSERT_EQ(ret, 0); ++ ASSERT_EQ(sarray->len, 3); ++ ++ sarray_copy = util_copy_string_array(sarray); ++ ASSERT_NE(sarray_copy, nullptr); ++ ASSERT_EQ(sarray_copy->len, sarray->len); ++ for (size_t i = 0; i < sarray_copy->len; i++) { ++ ASSERT_EQ(strcmp(sarray_copy->items[i], sarray->items[i]), 0); ++ } ++ ++ ASSERT_EQ(util_copy_string_array(nullptr), nullptr); ++ sarray->cap = 0; ++ ASSERT_EQ(util_copy_string_array(sarray), nullptr); ++ sarray->cap = sarray->len; ++} ++ + TEST(utils_array, test_util_string_array_contain) + { + string_array *sarray = (string_array *)util_common_calloc_s(sizeof(string_array)); +@@ -299,3 +354,76 @@ TEST(utils_array, test_util_common_array_append_pointer) + delete element1; + delete element2; + } ++ ++static void common_array_free_mock(void *ptr) ++{ ++ (void)ptr; ++ return; ++} ++ ++TEST(utils_array, test_util_append_common_array) ++{ ++ __isula_auto_common_array_t common_array *carray = nullptr; ++ int ret; ++ int value1 = 1; ++ int value2 = 2; ++ int value3 = 3; ++ ++ carray = util_common_array_new(1, common_array_free_mock, util_clone_ptr); ++ ASSERT_NE(carray, nullptr); ++ ++ ret = util_append_common_array(carray, &value1); ++ ASSERT_EQ(ret, 0); ++ ASSERT_EQ(carray->items[0], &value1); ++ ASSERT_EQ(carray->len, 1); ++ ++ ret = util_append_common_array(carray, &value2); ++ ASSERT_EQ(ret, 0); ++ ret = util_append_common_array(carray, &value3); ++ ASSERT_EQ(ret, 0); ++ ASSERT_EQ(carray->items[1], &value2); ++ ASSERT_EQ(carray->items[2], &value3); ++ ASSERT_EQ(carray->len, 3); ++ ++ carray->clone_item_cb = nullptr; ++ ASSERT_EQ(util_append_common_array(carray, &value1), -1); ++ carray->clone_item_cb = util_clone_ptr; ++ ASSERT_EQ(util_append_common_array(carray, nullptr), 0); ++} ++ ++TEST(utils_array, test_util_merge_common_array) ++{ ++ __isula_auto_common_array_t common_array *carray1 = nullptr; ++ __isula_auto_common_array_t common_array *carray2 = nullptr; ++ int ret; ++ int value1 = 1; ++ int value2 = 2; ++ ++ carray1 = util_common_array_new(1, common_array_free_mock, util_clone_ptr); ++ ASSERT_NE(carray1, nullptr); ++ carray2 = util_common_array_new(1, common_array_free_mock, util_clone_ptr); ++ ASSERT_NE(carray2, nullptr); ++ ++ ret = util_append_common_array(carray1, &value1); ++ ASSERT_EQ(ret, 0); ++ ASSERT_EQ(carray1->items[0], &value1); ++ ASSERT_EQ(carray1->len, 1); ++ ret = util_append_common_array(carray2, &value2); ++ ASSERT_EQ(ret, 0); ++ ASSERT_EQ(carray2->items[0], &value2); ++ ASSERT_EQ(carray2->len, 1); ++ ++ ret = util_merge_common_array(carray1, carray2); ++ ASSERT_EQ(ret, 0); ++ ASSERT_EQ(carray1->items[1], &value2); ++ ASSERT_EQ(carray1->len, 2); ++ ++ ASSERT_EQ(util_merge_common_array(nullptr, carray2), -1); ++ ASSERT_EQ(util_merge_common_array(carray1, nullptr), -1); ++ carray1->clone_item_cb = nullptr; ++ ASSERT_EQ(util_merge_common_array(carray1, carray2), -1); ++ carray1->clone_item_cb = util_clone_ptr; ++ carray2->clone_item_cb = nullptr; ++ ASSERT_EQ(util_merge_common_array(carray1, carray2), -1); ++ carray2->clone_item_cb = util_clone_ptr; ++} +\ No newline at end of file +diff --git a/test/cutils/utils_utils/utils_utils_ut.cc b/test/cutils/utils_utils/utils_utils_ut.cc +index 0720d1b1..a61e5a21 100644 +--- a/test/cutils/utils_utils/utils_utils_ut.cc ++++ b/test/cutils/utils_utils/utils_utils_ut.cc +@@ -54,6 +54,18 @@ static int status_to_exit_code(int status) + return exit_code; + } + ++TEST(utils_utils, test_util_swap_ptr) ++{ ++ int val1 = 1; ++ int val2 = 2; ++ int *ptr1 = &val1; ++ int *ptr2 = &val2; ++ ++ util_swap_ptr((void **)&ptr1, (void **)&ptr2); ++ ASSERT_EQ(*ptr1, val2); ++ ASSERT_EQ(*ptr2, val1); ++} ++ + TEST(utils_utils, test_util_mem_realloc) + { + char *old = nullptr; +diff --git a/test/cutils/utils_version/CMakeLists.txt b/test/cutils/utils_version/CMakeLists.txt +new file mode 100644 +index 00000000..1ada8e93 +--- /dev/null ++++ b/test/cutils/utils_version/CMakeLists.txt +@@ -0,0 +1,17 @@ ++project(iSulad_UT) ++ ++SET(EXE utils_version_ut) ++ ++add_executable(${EXE} ++ utils_version_ut.cc) ++ ++target_include_directories(${EXE} PUBLIC ++ ${GTEST_INCLUDE_DIR} ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../include ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils ++ ) ++ ++target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) ++add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_version/utils_version_ut.cc b/test/cutils/utils_version/utils_version_ut.cc +new file mode 100644 +index 00000000..d1fc0932 +--- /dev/null ++++ b/test/cutils/utils_version/utils_version_ut.cc +@@ -0,0 +1,71 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-04-25 ++ * Description: utils version unit test ++ *******************************************************************************/ ++ ++#include ++#include ++#include ++#include ++#include "mock.h" ++#include "utils_version.h" ++#include "utils.h" ++ ++TEST(utils_version, test_util_version_compare) ++{ ++ const char *version1 = "1.1.1"; ++ const char *version2 = "1.1.2"; ++ int diff_value = 0; ++ ++ ASSERT_EQ(util_version_compare(version1, version2, &diff_value), 0); ++ ASSERT_TRUE(diff_value < 0); ++ ASSERT_EQ(util_version_compare(version1, version1, &diff_value), 0); ++ ASSERT_TRUE(diff_value == 0); ++ ASSERT_EQ(util_version_compare(version2, version1, &diff_value), 0); ++ ASSERT_TRUE(diff_value > 0); ++ ++ ASSERT_EQ(util_version_compare(version1, nullptr, &diff_value), -1); ++ ASSERT_EQ(util_version_compare(nullptr, version2, &diff_value), -1); ++ ASSERT_EQ(util_version_compare(version1, version2, nullptr), -1); ++ ASSERT_EQ(util_version_compare("1.1.1.1", version2, nullptr), -1); ++ ASSERT_EQ(util_version_compare(version1, "a.b.1.1", nullptr), -1); ++} ++ ++TEST(utils_version, test_util_version_greater_than) ++{ ++ const char *version1 = "0.6.0"; ++ const char *version2 = "1.0.0"; ++ bool result = true; ++ ++ ASSERT_EQ(util_version_greater_than(version1, version2, &result), 0); ++ ASSERT_FALSE(result); ++ ASSERT_EQ(util_version_greater_than(version1, version1, &result), 0); ++ ASSERT_FALSE(result); ++ ASSERT_EQ(util_version_greater_than(version2, version1, &result), 0); ++ ASSERT_TRUE(result); ++} ++ ++TEST(utils_version, test_util_version_greater_than_or_equal_to) ++{ ++ const char *version1 = "0.6.0"; ++ const char *version2 = "1.0.0"; ++ bool result = true; ++ ++ ASSERT_EQ(util_version_greater_than_or_equal_to(version1, version2, &result), 0); ++ ASSERT_FALSE(result); ++ ASSERT_EQ(util_version_greater_than_or_equal_to(version1, version1, &result), 0); ++ ASSERT_TRUE(result); ++ ASSERT_EQ(util_version_greater_than_or_equal_to(version2, version1, &result), 0); ++ ASSERT_TRUE(result); ++} ++ +diff --git a/test/image/oci/oci_config_merge/CMakeLists.txt b/test/image/oci/oci_config_merge/CMakeLists.txt +index ffd3999d..38ade4ec 100644 +--- a/test/image/oci/oci_config_merge/CMakeLists.txt ++++ b/test/image/oci/oci_config_merge/CMakeLists.txt +@@ -47,6 +47,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/selinux_label_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/storage_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/image_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/cdi_operate_api_mock.cc + oci_config_merge_ut.cc) + + target_include_directories(${EXE} PUBLIC +diff --git a/test/mocks/cdi_operate_api_mock.cc b/test/mocks/cdi_operate_api_mock.cc +new file mode 100644 +index 00000000..d8f9f9d4 +--- /dev/null ++++ b/test/mocks/cdi_operate_api_mock.cc +@@ -0,0 +1,58 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2023-10-20 ++ * Description: provide image api mock ++ ******************************************************************************/ ++ ++#include "cdi_operate_api_mock.h" ++ ++namespace { ++std::shared_ptr g_cdi_operate_api_mock = nullptr; ++} ++ ++void MockCdiOperateApi_SetMock(std::shared_ptr mock) ++{ ++ g_cdi_operate_api_mock = mock; ++} ++ ++int cdi_operate_registry_init(char **specs_dirs, size_t specs_dirs_len) ++{ ++ if (g_cdi_operate_api_mock != nullptr) { ++ return g_cdi_operate_api_mock->CdiOperateRegistryInit(specs_dirs, specs_dirs_len); ++ } ++ return 0; ++} ++ ++int cdi_operate_refresh(void) ++{ ++ if (g_cdi_operate_api_mock != nullptr) { ++ return g_cdi_operate_api_mock->CdiOperateRefresh(); ++ } ++ return 0; ++} ++ ++int cdi_operate_inject_devices(oci_runtime_spec *spec, string_array *devices) ++{ ++ if (g_cdi_operate_api_mock != nullptr) { ++ return g_cdi_operate_api_mock->CdiOperateInjectDevices(spec, devices); ++ } ++ return 0; ++} ++ ++int cdi_operate_parse_annotations(json_map_string_string *annotations, string_array **keys, ++ string_array **devices, char **error) ++{ ++ if (g_cdi_operate_api_mock != nullptr) { ++ return g_cdi_operate_api_mock->CdiOperateParseAnnotations(annotations, keys, devices, error); ++ } ++ return 0; ++} +\ No newline at end of file +diff --git a/test/mocks/cdi_operate_api_mock.h b/test/mocks/cdi_operate_api_mock.h +new file mode 100644 +index 00000000..c118ee7a +--- /dev/null ++++ b/test/mocks/cdi_operate_api_mock.h +@@ -0,0 +1,35 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: liuxu ++ * Create: 2024-04-24 ++ * Description: provide cdi api mock ++ ******************************************************************************/ ++ ++#ifndef ISULAD_TEST_MOCKS_CDI_OPERATE_API_MOCK_H ++#define ISULAD_TEST_MOCKS_CDI_OPERATE_API_MOCK_H ++ ++#include ++#include ++ ++#include "cdi_operate_api.h" ++ ++class MockCdiOperateApi { ++public: ++ MOCK_METHOD2(CdiOperateRegistryInit, int(char **specs_dirs, size_t specs_dirs_len)); ++ MOCK_METHOD0(CdiOperateRefresh, int(void)); ++ MOCK_METHOD2(CdiOperateInjectDevices, int(oci_runtime_spec *spec, string_array *devices)); ++ MOCK_METHOD4(CdiOperateParseAnnotations, int(json_map_string_string *annotations, string_array **keys, ++ string_array **devices, char **error)); ++}; ++ ++void MockCdiOperateApi_SetMock(std::shared_ptr mock); ++ ++#endif +diff --git a/test/sandbox/controller/shim/CMakeLists.txt b/test/sandbox/controller/shim/CMakeLists.txt +index 26a66e51..d18d1861 100644 +--- a/test/sandbox/controller/shim/CMakeLists.txt ++++ b/test/sandbox/controller/shim/CMakeLists.txt +@@ -21,6 +21,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/callback_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/image_api_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/service_container_api_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/cdi_operate_api_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/sandbox/controller/controller_common.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/sandbox/controller/shim/shim_controller_ut.cc + ) +diff --git a/test/specs/specs/CMakeLists.txt b/test/specs/specs/CMakeLists.txt +index 12c11f51..892d44d7 100644 +--- a/test/specs/specs/CMakeLists.txt ++++ b/test/specs/specs/CMakeLists.txt +@@ -44,6 +44,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/image_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/cdi_operate_api_mock.cc + specs_ut.cc) + + target_include_directories(${EXE} PUBLIC +diff --git a/test/specs/specs/specs_ut.cc b/test/specs/specs/specs_ut.cc +index 6c42216d..47836e5b 100644 +--- a/test/specs/specs/specs_ut.cc ++++ b/test/specs/specs/specs_ut.cc +@@ -564,6 +564,247 @@ TEST_F(SpecsUnitTest, test_update_devcies_for_oci_spec) + free(err); + } + ++#ifdef ENABLE_CDI ++TEST_F(SpecsUnitTest, test_defs_process_add_multiple_env) ++{ ++ size_t env_len = 2; ++ char **envs = (char **)util_common_calloc_s(sizeof(char *) * env_len); ++ ASSERT_NE(envs, nullptr); ++ defs_process *dp = (defs_process *)util_common_calloc_s(sizeof(defs_process)); ++ ASSERT_NE(dp, nullptr); ++ dp->env_len = 1; ++ dp->env = (char **)util_common_calloc_s(sizeof(char *) * dp->env_len); ++ ASSERT_NE(dp->env, nullptr); ++ ++ envs[0] = util_strdup_s("key0=value0"); ++ envs[1] = util_strdup_s("key1=value1"); ++ dp->env[0] = util_strdup_s("key0=value0_old"); ++ ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), 0); ++ ASSERT_EQ(dp->env_len, 2); ++ ASSERT_EQ(strcmp(dp->env[0], envs[0]), 0); ++ ASSERT_EQ(strcmp(dp->env[1], envs[1]), 0); ++ ++ ASSERT_EQ(defs_process_add_multiple_env(dp, nullptr, env_len), 0); ++ ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, 0), 0); ++ ASSERT_EQ(defs_process_add_multiple_env(nullptr, (const char **)envs, env_len), -1); ++ ++ free(envs[0]); ++ envs[0] = util_strdup_s("=value0"); ++ ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1); ++ free(envs[0]); ++ envs[0] = util_strdup_s("key0="); ++ ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1); ++ free(envs[0]); ++ envs[0] = util_strdup_s("key0xxxx"); ++ ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1); ++ ++ free(dp->env[0]); ++ dp->env[0] = util_strdup_s("=value0"); ++ ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1); ++ free(dp->env[0]); ++ dp->env[0] = util_strdup_s("key0="); ++ ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1); ++ free(dp->env[0]); ++ dp->env[0] = util_strdup_s("key0xxxx"); ++ ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1); ++ ++ free_defs_process(dp); ++ free(envs[0]); ++ free(envs[1]); ++ free(envs); ++} ++ ++TEST_F(SpecsUnitTest, test_spec_add_multiple_process_env) ++{ ++ size_t env_len = 2; ++ char **envs = (char **)util_common_calloc_s(sizeof(char *) * env_len); ++ ASSERT_NE(envs, nullptr); ++ oci_runtime_spec *oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec)); ++ ASSERT_NE(oci_spec, nullptr); ++ oci_spec->process = (defs_process *)util_common_calloc_s(sizeof(defs_process)); ++ ASSERT_NE(oci_spec->process, nullptr); ++ oci_spec->process->env_len = 1; ++ oci_spec->process->env = (char **)util_common_calloc_s(sizeof(char *) * oci_spec->process->env_len); ++ ASSERT_NE(oci_spec->process->env, nullptr); ++ ++ envs[0] = util_strdup_s("key0=value0"); ++ envs[1] = util_strdup_s("key1=value1"); ++ oci_spec->process->env[0] = util_strdup_s("key0=value0_old"); ++ ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), 0); ++ ASSERT_EQ(oci_spec->process->env_len, 2); ++ ASSERT_EQ(strcmp(oci_spec->process->env[0], envs[0]), 0); ++ ASSERT_EQ(strcmp(oci_spec->process->env[1], envs[1]), 0); ++ ++ ASSERT_EQ(spec_add_multiple_process_env(oci_spec, nullptr, env_len), 0); ++ ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, 0), 0); ++ ASSERT_EQ(spec_add_multiple_process_env(nullptr, (const char **)envs, env_len), -1); ++ ++ free(envs[0]); ++ envs[0] = util_strdup_s("=value0"); ++ ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1); ++ free(envs[0]); ++ envs[0] = util_strdup_s("key0="); ++ ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1); ++ free(envs[0]); ++ envs[0] = util_strdup_s("key0xxxx"); ++ ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1); ++ ++ free(oci_spec->process->env[0]); ++ oci_spec->process->env[0] = util_strdup_s("=value0"); ++ ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1); ++ free(oci_spec->process->env[0]); ++ oci_spec->process->env[0] = util_strdup_s("key0="); ++ ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1); ++ free(oci_spec->process->env[0]); ++ oci_spec->process->env[0] = util_strdup_s("key0xxxx"); ++ ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1); ++ ++ free_oci_runtime_spec(oci_spec); ++ free(envs[0]); ++ free(envs[1]); ++ free(envs); ++} ++ ++TEST_F(SpecsUnitTest, test_spec_add_device) ++{ ++ defs_device *device = (defs_device *)util_common_calloc_s(sizeof(defs_device)); ++ ASSERT_NE(device, nullptr); ++ oci_runtime_spec *oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec)); ++ ASSERT_NE(oci_spec, nullptr); ++ oci_spec->linux = (oci_runtime_config_linux *)util_common_calloc_s(sizeof(oci_runtime_config_linux)); ++ ASSERT_NE(oci_spec->linux, nullptr); ++ oci_spec->linux->devices_len = 1; ++ oci_spec->linux->devices = (defs_device **)util_common_calloc_s(sizeof(defs_device *) * oci_spec->linux->devices_len); ++ ASSERT_NE(oci_spec->linux->devices, nullptr); ++ ++ device->path = util_strdup_s("/device/path"); ++ oci_spec->linux->devices[0] = (defs_device *)util_common_calloc_s(sizeof(defs_device)); ++ ASSERT_NE(oci_spec->linux->devices[0], nullptr); ++ oci_spec->linux->devices[0]->path = util_strdup_s("/device/path"); ++ ASSERT_EQ(spec_add_device(oci_spec, device), 0); ++ ASSERT_EQ(oci_spec->linux->devices[0], device); ++ ++ oci_spec->linux->devices[0] = nullptr; ++ oci_spec->linux->devices_len = 0; ++ ASSERT_EQ(spec_add_device(oci_spec, device), 0); ++ ASSERT_EQ(oci_spec->linux->devices_len, 1); ++ ASSERT_EQ(oci_spec->linux->devices[0], device); ++ ++ ASSERT_EQ(spec_add_device(oci_spec, nullptr), -1); ++ ASSERT_EQ(spec_add_device(nullptr, device), -1); ++ ++ free_oci_runtime_spec(oci_spec); ++} ++ ++TEST_F(SpecsUnitTest, test_spec_add_linux_resources_device) ++{ ++ oci_runtime_spec *oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec)); ++ ASSERT_NE(oci_spec, nullptr); ++ oci_spec->linux = (oci_runtime_config_linux *)util_common_calloc_s(sizeof(oci_runtime_config_linux)); ++ ASSERT_NE(oci_spec->linux, nullptr); ++ oci_spec->linux->resources = (defs_resources *)util_common_calloc_s(sizeof(defs_resources)); ++ ASSERT_NE(oci_spec->linux->resources, nullptr); ++ oci_spec->linux->resources->devices_len = 1; ++ oci_spec->linux->resources->devices = (defs_device_cgroup **)util_common_calloc_s(sizeof(defs_device_cgroup *) * oci_spec->linux->resources->devices_len); ++ ASSERT_NE(oci_spec->linux->resources->devices, nullptr); ++ ++ oci_spec->linux->resources->devices[0] = (defs_device_cgroup *)util_common_calloc_s(sizeof(defs_device_cgroup)); ++ ASSERT_NE(oci_spec->linux->resources->devices[0], nullptr); ++ ASSERT_EQ(spec_add_linux_resources_device(oci_spec, true, "bind", 10, 9, "rwm"), 0); ++ ASSERT_EQ(oci_spec->linux->resources->devices_len, 2); ++ ASSERT_EQ(oci_spec->linux->resources->devices[1]->allow, true); ++ ASSERT_EQ(strcmp(oci_spec->linux->resources->devices[1]->type, "bind"), 0); ++ ASSERT_EQ(oci_spec->linux->resources->devices[1]->major, 10); ++ ASSERT_EQ(oci_spec->linux->resources->devices[1]->minor, 9); ++ ASSERT_EQ(strcmp(oci_spec->linux->resources->devices[1]->access, "rwm"), 0); ++ ++ ASSERT_EQ(spec_add_linux_resources_device(nullptr, true, "bind", 10, 9, "rwm"), -1); ++ ++ free_oci_runtime_spec(oci_spec); ++} ++ ++TEST_F(SpecsUnitTest, test_spec_remove_mount) ++{ ++ oci_runtime_spec *oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec)); ++ ASSERT_NE(oci_spec, nullptr); ++ oci_spec->mounts_len = 2; ++ oci_spec->mounts = (defs_mount **)util_common_calloc_s(sizeof(defs_mount *) * oci_spec->mounts_len); ++ ASSERT_NE(oci_spec->mounts, nullptr); ++ ++ oci_spec->mounts[0] = (defs_mount *)util_common_calloc_s(sizeof(defs_mount)); ++ ASSERT_NE(oci_spec->mounts[0], nullptr); ++ oci_spec->mounts[1] = (defs_mount *)util_common_calloc_s(sizeof(defs_mount)); ++ ASSERT_NE(oci_spec->mounts[1], nullptr); ++ oci_spec->mounts[0]->destination = util_strdup_s("/mount/path/0"); ++ oci_spec->mounts[1]->destination = util_strdup_s("/mount/path/1"); ++ spec_remove_mount(oci_spec, oci_spec->mounts[0]->destination); ++ ASSERT_EQ(oci_spec->mounts_len, 1); ++ ASSERT_EQ(strcmp(oci_spec->mounts[0]->destination, "/mount/path/1"), 0); ++ ++ free_oci_runtime_spec(oci_spec); ++} ++ ++TEST_F(SpecsUnitTest, test_spec_add_mount) ++{ ++ defs_mount *mnt = (defs_mount *)util_common_calloc_s(sizeof(defs_mount)); ++ ASSERT_NE(mnt, nullptr); ++ oci_runtime_spec *oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec)); ++ ASSERT_NE(oci_spec, nullptr); ++ oci_spec->mounts_len = 1; ++ oci_spec->mounts = (defs_mount **)util_common_calloc_s(sizeof(defs_mount *) * oci_spec->mounts_len ); ++ ASSERT_NE(oci_spec->mounts, nullptr); ++ ++ oci_spec->mounts[0] = (defs_mount *)util_common_calloc_s(sizeof(defs_mount)); ++ ASSERT_NE(oci_spec->mounts[0], nullptr); ++ oci_spec->mounts[0]->destination = util_strdup_s("/mount/path/0"); ++ ASSERT_EQ(spec_add_mount(oci_spec, mnt), 0); ++ ASSERT_EQ(oci_spec->mounts_len, 2); ++ ASSERT_EQ(oci_spec->mounts[1], mnt); ++ ++ ASSERT_EQ(spec_add_mount(nullptr, mnt), -1); ++ ASSERT_EQ(spec_add_mount(oci_spec, nullptr), -1); ++ ++ free_oci_runtime_spec(oci_spec); ++} ++ ++#define TEST_SPEC_ADD_HOOKS_ITEM_DEF(hooktype) \ ++ void test_spec_add_##hooktype##_hook(void) \ ++ { \ ++ defs_hook *hook = (defs_hook *)util_common_calloc_s(sizeof(defs_hook)); \ ++ ASSERT_NE(hook, nullptr); \ ++ oci_runtime_spec *oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec)); \ ++ ASSERT_NE(oci_spec, nullptr); \ ++ oci_spec->hooks = (oci_runtime_spec_hooks *)util_common_calloc_s(sizeof(oci_runtime_spec_hooks)); \ ++ ASSERT_NE(oci_spec->hooks, nullptr); \ ++ oci_spec->hooks->hooktype##_len = 1; \ ++ oci_spec->hooks->hooktype = (defs_hook **)util_common_calloc_s(sizeof(defs_hook *) * oci_spec->hooks->hooktype##_len); \ ++ ASSERT_NE(oci_spec->hooks->hooktype, nullptr); \ ++ \ ++ oci_spec->hooks->hooktype[0] = (defs_hook *)util_common_calloc_s(sizeof(defs_hook)); \ ++ ASSERT_NE(oci_spec->hooks->hooktype[0], nullptr); \ ++ ASSERT_EQ(spec_add_##hooktype##_hook(oci_spec, hook), 0); \ ++ ASSERT_EQ(oci_spec->hooks->hooktype##_len, 2); \ ++ ASSERT_EQ(oci_spec->hooks->hooktype[1], hook); \ ++ \ ++ ASSERT_EQ(spec_add_##hooktype##_hook(nullptr, hook), -1); \ ++ ASSERT_EQ(spec_add_##hooktype##_hook(oci_spec, nullptr), -1); \ ++ \ ++ free_oci_runtime_spec(oci_spec); \ ++ } ++ ++TEST_SPEC_ADD_HOOKS_ITEM_DEF(prestart) ++TEST_SPEC_ADD_HOOKS_ITEM_DEF(poststart) ++TEST_SPEC_ADD_HOOKS_ITEM_DEF(poststop) ++ ++TEST_F(SpecsUnitTest, test_spec_add_hook) ++{ ++ test_spec_add_prestart_hook(); ++ test_spec_add_poststart_hook(); ++ test_spec_add_poststop_hook(); ++} ++ ++#endif /* ENABLE_CDI */ ++ + /********************************* UT for merge caps *******************************************/ + struct capabilities_lens { + size_t bounding_len; +diff --git a/test/specs/specs_extend/CMakeLists.txt b/test/specs/specs_extend/CMakeLists.txt +index 2fd37e1c..bd4d2dd6 100644 +--- a/test/specs/specs_extend/CMakeLists.txt ++++ b/test/specs/specs_extend/CMakeLists.txt +@@ -44,6 +44,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/image_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/cdi_operate_api_mock.cc + specs_extend_ut.cc) + + target_include_directories(${EXE} PUBLIC +-- +2.25.1 + diff --git a/0077-remove-extra-s-in-CreateContainerLogSymlink.patch b/0077-remove-extra-s-in-CreateContainerLogSymlink.patch new file mode 100644 index 0000000..571f47c --- /dev/null +++ b/0077-remove-extra-s-in-CreateContainerLogSymlink.patch @@ -0,0 +1,26 @@ +From cade2ae3b53848bbedb3f89ff45333f3871e878a Mon Sep 17 00:00:00 2001 +From: jikai +Date: Thu, 25 Apr 2024 12:46:10 +0000 +Subject: [PATCH 77/78] remove extra %s in CreateContainerLogSymlink + +Signed-off-by: jikai +--- + src/daemon/common/cri/cri_helpers.cc | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/daemon/common/cri/cri_helpers.cc b/src/daemon/common/cri/cri_helpers.cc +index 5189ac00..d7ec9f36 100644 +--- a/src/daemon/common/cri/cri_helpers.cc ++++ b/src/daemon/common/cri/cri_helpers.cc +@@ -552,7 +552,7 @@ void CreateContainerLogSymlink(const std::string &containerID, Errors &error) + if (symlink(realPath.c_str(), path.c_str()) != 0) { + SYSERROR("failed to create symbolic link %s to the container log file %s for container %s", path.c_str(), realPath.c_str(), + containerID.c_str()); +- error.Errorf("failed to create symbolic link %s to the container log file %s for container %s: %s", path.c_str(), ++ error.Errorf("failed to create symbolic link %s to the container log file %s for container %s", path.c_str(), + realPath.c_str(), containerID.c_str()); + } + } +-- +2.25.1 + diff --git a/0078-allow-env-variable-has-an-empty-value.patch b/0078-allow-env-variable-has-an-empty-value.patch new file mode 100644 index 0000000..213a996 --- /dev/null +++ b/0078-allow-env-variable-has-an-empty-value.patch @@ -0,0 +1,69 @@ +From f0212d54afc695b2039f09456b10c47f8edaf2de Mon Sep 17 00:00:00 2001 +From: jikai +Date: Sat, 27 Apr 2024 14:32:19 +0800 +Subject: [PATCH 78/78] allow env variable has an empty value + +Signed-off-by: jikai +--- + src/daemon/modules/spec/specs_extend.c | 10 ++++++---- + src/utils/cutils/utils_string.c | 3 +++ + 2 files changed, 9 insertions(+), 4 deletions(-) + +diff --git a/src/daemon/modules/spec/specs_extend.c b/src/daemon/modules/spec/specs_extend.c +index 199cba54..8cad2cbe 100644 +--- a/src/daemon/modules/spec/specs_extend.c ++++ b/src/daemon/modules/spec/specs_extend.c +@@ -195,6 +195,7 @@ static int generate_env_map_from_file(FILE *fp, json_map_string_string *env_map) + char *pline = NULL; + size_t length = 0; + char *saveptr = NULL; ++ char empty_str[1] = {'\0'}; + + while (getline(&pline, &length, fp) != -1) { + util_trim_newline(pline); +@@ -204,7 +205,9 @@ static int generate_env_map_from_file(FILE *fp, json_map_string_string *env_map) + } + key = strtok_r(pline, "=", &saveptr); + value = strtok_r(NULL, "=", &saveptr); +- if (key != NULL && value != NULL) { ++ // value of an env varible is allowed to be empty ++ value = value ? value : empty_str; ++ if (key != NULL) { + key = util_trim_space(key); + value = util_trim_space(value); + if ((size_t)(MAX_BUFFER_SIZE - 1) - strlen(key) < strlen(value)) { +@@ -291,15 +294,14 @@ static int check_env_need_append(const oci_runtime_spec *oci_spec, const char *e + { + size_t i = 0; + char *key = NULL; +- char *value = NULL; + char *saveptr = NULL; + + for (i = 0; i < oci_spec->process->env_len; i++) { + char *tmp_env = NULL; + tmp_env = util_strdup_s(oci_spec->process->env[i]); + key = strtok_r(tmp_env, "=", &saveptr); +- value = strtok_r(NULL, "=", &saveptr); +- if (key == NULL || value == NULL) { ++ // value of an env varible is allowed to be empty ++ if (key == NULL) { + ERROR("Bad env format"); + free(tmp_env); + tmp_env = NULL; +diff --git a/src/utils/cutils/utils_string.c b/src/utils/cutils/utils_string.c +index 64afb570..11a65f19 100644 +--- a/src/utils/cutils/utils_string.c ++++ b/src/utils/cutils/utils_string.c +@@ -534,6 +534,9 @@ static char *util_left_trim_space(char *str) + { + char *begin = str; + char *tmp = str; ++ if (strlen(str) == 0) { ++ return str; ++ } + while (isspace(*begin)) { + begin++; + } +-- +2.25.1 + diff --git a/0079-Fix-Failed-to-execute-image-pull-on-name-tag-digest-.patch b/0079-Fix-Failed-to-execute-image-pull-on-name-tag-digest-.patch new file mode 100644 index 0000000..6c9fc35 --- /dev/null +++ b/0079-Fix-Failed-to-execute-image-pull-on-name-tag-digest-.patch @@ -0,0 +1,72 @@ +From 1d90c5a855a267bb156d53b6a43bc451a993efd5 Mon Sep 17 00:00:00 2001 +From: huj13k4n9 +Date: Mon, 29 Apr 2024 12:52:17 +0800 +Subject: [PATCH 79/85] Fix 'Failed to execute image pull' on 'name:tag@digest' + type image name + +--- + src/daemon/modules/image/oci/utils_images.c | 6 ++++++ + src/utils/cutils/utils_verify.c | 3 ++- + src/utils/cutils/utils_verify.h | 3 ++- + 3 files changed, 10 insertions(+), 2 deletions(-) + +diff --git a/src/daemon/modules/image/oci/utils_images.c b/src/daemon/modules/image/oci/utils_images.c +index d94388bd..b4e2a1a8 100644 +--- a/src/daemon/modules/image/oci/utils_images.c ++++ b/src/daemon/modules/image/oci/utils_images.c +@@ -207,6 +207,7 @@ int oci_split_image_name(const char *image_name, char **host, char **name, char + char *tag_digest_pos = NULL; + char *name_pos = NULL; + char *tmp_image_name = NULL; ++ char *name_end_pos = NULL; + + if (!util_valid_image_name(image_name)) { + ERROR("Invalid full image name %s", image_name); +@@ -234,6 +235,11 @@ int oci_split_image_name(const char *image_name, char **host, char **name, char + *name_pos = '\0'; + name_pos++; + if (name != NULL) { ++ // Need to check if image name contains tag ++ name_end_pos = strchr(name_pos, ':'); ++ if (name_end_pos != NULL) { ++ *name_end_pos = '\0'; ++ } + *name = util_strdup_s(name_pos); + } + if (host != NULL) { +diff --git a/src/utils/cutils/utils_verify.c b/src/utils/cutils/utils_verify.c +index cd636fff..474e28f0 100644 +--- a/src/utils/cutils/utils_verify.c ++++ b/src/utils/cutils/utils_verify.c +@@ -319,6 +319,7 @@ bool util_valid_image_name(const char *name) + } + } + ++ // In name check phase, image name with both tag and digest is also allowed + if (util_reg_match(__NamePattern, copy)) { + goto cleanup; + } +@@ -767,4 +768,4 @@ bool util_valid_search_name(const char *name) + + return true; + } +-#endif +\ No newline at end of file ++#endif +diff --git a/src/utils/cutils/utils_verify.h b/src/utils/cutils/utils_verify.h +index bafd2a82..fc59f6c0 100644 +--- a/src/utils/cutils/utils_verify.h ++++ b/src/utils/cutils/utils_verify.h +@@ -32,7 +32,8 @@ extern "C" { + #define __NamePattern \ + "^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])" \ + "((\\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9]))+)?(:[0-9]+)?/)?[a-z0-9]" \ +- "+((([._]|__|[-]*)[a-z0-9]+)+)?((/[a-z0-9]+((([._]|__|[-]*)[a-z0-9]+)+)?)+)?$" ++ "+((([._]|__|[-]*)[a-z0-9]+)+)?((/[a-z0-9]+((([._]|__|[-]*)[a-z0-9]+)+)?)+)?" \ ++ "(:([A-Za-z_0-9][A-Za-z_0-9.-]{0,127}))?$" + + #define __DIGESTPattern "@[a-z0-9]+:[a-z0-9]{32,}" + +-- +2.34.1 + diff --git a/0080-bugfix-for-hostname-env-set-only-once.patch b/0080-bugfix-for-hostname-env-set-only-once.patch new file mode 100644 index 0000000..191a790 --- /dev/null +++ b/0080-bugfix-for-hostname-env-set-only-once.patch @@ -0,0 +1,148 @@ +From 8ff32819d84f59085c4c541b00f9671db55d0fd1 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Mon, 29 Apr 2024 09:14:53 +0800 +Subject: [PATCH 80/85] bugfix for hostname env: set only once + +Signed-off-by: jikai +--- + src/daemon/modules/spec/specs.c | 11 +++++- + src/daemon/modules/spec/specs_extend.c | 52 +++++++++++++++++--------- + src/daemon/modules/spec/specs_extend.h | 2 + + 3 files changed, 46 insertions(+), 19 deletions(-) + +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index 77ca70f9..65a860d4 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -1863,14 +1863,21 @@ static int merge_process_conf(oci_runtime_spec *oci_spec, const host_config *hos + goto out; + } + +- /* environment variables */ ++ /* 1. merge env from container_spec: --env or --env-file */ + ret = merge_env(oci_spec, (const char **)container_spec->env, container_spec->env_len); + if (ret != 0) { + ERROR("Failed to merge environment variables"); + goto out; + } + +- /* env target file */ ++ /* 2. merge default env hostname, only if hostname not set before */ ++ ret = merge_hostname_env(oci_spec); ++ if (ret != 0) { ++ ERROR("Failed to merge hostname env"); ++ goto out; ++ } ++ ++ /* 3. persist env from --env-target-file, only if the env not set before, system container only */ + ret = merge_env_target_file(oci_spec, host_spec->env_target_file); + if (ret != 0) { + ERROR("Failed to merge env target file"); +diff --git a/src/daemon/modules/spec/specs_extend.c b/src/daemon/modules/spec/specs_extend.c +index 8cad2cbe..4c154281 100644 +--- a/src/daemon/modules/spec/specs_extend.c ++++ b/src/daemon/modules/spec/specs_extend.c +@@ -420,34 +420,23 @@ out: + int merge_env(oci_runtime_spec *oci_spec, const char **env, size_t env_len) + { + int ret = 0; +- int nret = 0; + size_t new_size = 0; + size_t old_size = 0; + size_t i; + char **temp = NULL; +- // 10 is lenght of "HOSTNAME=" and '\0' +- char host_name_env[MAX_HOST_NAME_LEN + 10] = { 0 }; +- +- nret = snprintf(host_name_env, sizeof(host_name_env), "HOSTNAME=%s", oci_spec->hostname); +- if (nret < 0 || (size_t)nret >= sizeof(host_name_env)) { +- ret = -1; +- ERROR("Sprint failed"); +- goto out; +- } + + ret = make_sure_oci_spec_process(oci_spec); + if (ret < 0) { + goto out; + } + +- if (env_len > LIST_ENV_SIZE_MAX - oci_spec->process->env_len - 1) { ++ if (env_len > LIST_ENV_SIZE_MAX - oci_spec->process->env_len) { + ERROR("The length of envionment variables is too long, the limit is %lld", LIST_ENV_SIZE_MAX); + isulad_set_error_message("The length of envionment variables is too long, the limit is %d", LIST_ENV_SIZE_MAX); + ret = -1; + goto out; + } +- // add 1 for hostname env +- new_size = (oci_spec->process->env_len + env_len + 1) * sizeof(char *); ++ new_size = (oci_spec->process->env_len + env_len) * sizeof(char *); + old_size = oci_spec->process->env_len * sizeof(char *); + ret = util_mem_realloc((void **)&temp, new_size, oci_spec->process->env, old_size); + if (ret != 0) { +@@ -458,10 +447,6 @@ int merge_env(oci_runtime_spec *oci_spec, const char **env, size_t env_len) + + oci_spec->process->env = temp; + +- // append hostname env into default oci spec env list +- oci_spec->process->env[oci_spec->process->env_len] = util_strdup_s(host_name_env); +- oci_spec->process->env_len++; +- + for (i = 0; i < env_len && env != NULL; i++) { + oci_spec->process->env[oci_spec->process->env_len] = util_strdup_s(env[i]); + oci_spec->process->env_len++; +@@ -470,6 +455,39 @@ out: + return ret; + } + ++int merge_hostname_env(oci_runtime_spec *oci_spec) ++{ ++ int nret = 0; ++ bool is_append = true; ++ // 10 is lenght of "HOSTNAME=" and '\0' ++ char host_name_env[MAX_HOST_NAME_LEN + 10] = { 0 }; ++ const char *envs[1] = {host_name_env}; ++ ++ if (make_sure_oci_spec_process(oci_spec) < 0) { ++ return -1; ++ } ++ ++ if (check_env_need_append(oci_spec, "HOSTNAME", &is_append) < 0) { ++ return -1; ++ } ++ ++ if (!is_append) { ++ return 0; ++ } ++ ++ nret = snprintf(host_name_env, sizeof(host_name_env), "HOSTNAME=%s", oci_spec->hostname); ++ if (nret < 0 || (size_t)nret >= sizeof(host_name_env)) { ++ ERROR("Sprint failed"); ++ return -1; ++ } ++ ++ if (merge_env(oci_spec, (const char **)envs, 1) < 0) { ++ return -1; ++ } ++ ++ return 0; ++} ++ + char *oci_container_get_env(const oci_runtime_spec *oci_spec, const char *key) + { + const defs_process *op = NULL; +diff --git a/src/daemon/modules/spec/specs_extend.h b/src/daemon/modules/spec/specs_extend.h +index d70f5bec..15ec6b2f 100644 +--- a/src/daemon/modules/spec/specs_extend.h ++++ b/src/daemon/modules/spec/specs_extend.h +@@ -50,6 +50,8 @@ int make_userns_remap(oci_runtime_spec *container, const char *user_remap); + + int merge_env(oci_runtime_spec *oci_spec, const char **env, size_t env_len); + ++int merge_hostname_env(oci_runtime_spec *oci_spec); ++ + int merge_env_target_file(oci_runtime_spec *oci_spec, const char *env_target_file); + + char *oci_container_get_env(const oci_runtime_spec *oci_spec, const char *key); +-- +2.34.1 + diff --git a/0081-set-the-sandbox-status-to-not-ready-under-abnormal-c.patch b/0081-set-the-sandbox-status-to-not-ready-under-abnormal-c.patch new file mode 100644 index 0000000..91a57f2 --- /dev/null +++ b/0081-set-the-sandbox-status-to-not-ready-under-abnormal-c.patch @@ -0,0 +1,86 @@ +From 934d289aa535bbb87bfe484c4de34275b968fb87 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Wed, 8 May 2024 11:40:40 +0800 +Subject: [PATCH 81/85] set the sandbox status to not ready under abnormal + circumstances + +Signed-off-by: zhongtao +--- + src/daemon/sandbox/sandbox.cc | 34 +++++++++++++++++++++++++--------- + src/daemon/sandbox/sandbox.h | 1 + + 2 files changed, 26 insertions(+), 9 deletions(-) + +diff --git a/src/daemon/sandbox/sandbox.cc b/src/daemon/sandbox/sandbox.cc +index bae5b8db..279bf628 100644 +--- a/src/daemon/sandbox/sandbox.cc ++++ b/src/daemon/sandbox/sandbox.cc +@@ -371,6 +371,8 @@ void Sandbox::DoUpdateStatus(std::unique_ptr status, Er + m_state.exitedAt = status->exitedAt; + if (status->state == std::string(SANDBOX_READY_STATE_STR)) { + m_state.status = SANDBOX_STATUS_RUNNING; ++ } else { ++ m_state.status = SANDBOX_STATUS_STOPPED; + } + } + +@@ -459,6 +461,24 @@ auto Sandbox::Save(Errors &error) -> bool + return true; + } + ++bool Sandbox::DoStatusUpdateAndWaitInLoad(const std::string &sandboxID, Errors &error) ++{ ++ if (!UpdateStatus(error)) { ++ ERROR("Failed to update status of Sandbox, id='%s'", sandboxID.c_str()); ++ return false; ++ } ++ ++ // Regardless of whether the sandbox is ready, ++ // Wait() is required to call to monitor whether the kuasar sandbox is ready or exits. ++ // TODO: distinguish the meaning of Wait() return value in different states of sandbox ++ if (!m_controller->Wait(shared_from_this(), sandboxID, error)) { ++ ERROR("Failed to restore wait callback"); ++ return false; ++ } ++ ++ return true; ++} ++ + auto Sandbox::Load(Errors &error) -> bool + { + if (!LoadState(error)) { +@@ -478,15 +498,11 @@ auto Sandbox::Load(Errors &error) -> bool + + LoadNetworkSetting(); + +- if (!UpdateStatus(error)) { +- ERROR("Failed to update status of Sandbox, id='%s'", m_id.c_str()); +- return false; +- } +- +- // TODO: distinguish the meaning of Wait() return value in different states of sandbox +- if (!m_controller->Wait(shared_from_this(), m_id, error)) { +- ERROR("Failed to restore wait callback"); +- return false; ++ // When the sandbox status acquisition fails or wait fails, the sandbox status is set to not ready, ++ // and the user decides whether to delete the sandbox. ++ if (!DoStatusUpdateAndWaitInLoad(m_id, error)) { ++ WriteGuard lock(m_stateMutex); ++ m_state.status = SANDBOX_STATUS_STOPPED; + } + + return true; +diff --git a/src/daemon/sandbox/sandbox.h b/src/daemon/sandbox/sandbox.h +index 20a8e338..42fbee2a 100644 +--- a/src/daemon/sandbox/sandbox.h ++++ b/src/daemon/sandbox/sandbox.h +@@ -156,6 +156,7 @@ private: + auto SetupSandboxFiles(Errors &error) -> bool; + void DoUpdateStatus(std::unique_ptr status, Errors &error); + void DoUpdateExitedStatus(const ControllerExitInfo &exitInfo); ++ bool DoStatusUpdateAndWaitInLoad(const std::string &sandboxID, Errors &error); + + auto GetMetadataJsonPath() -> std::string; + auto GetStatePath() -> std::string; +-- +2.34.1 + diff --git a/0082-fix-shim-controller-set-incorrect-sandbox-status-sta.patch b/0082-fix-shim-controller-set-incorrect-sandbox-status-sta.patch new file mode 100644 index 0000000..13df472 --- /dev/null +++ b/0082-fix-shim-controller-set-incorrect-sandbox-status-sta.patch @@ -0,0 +1,60 @@ +From 1d51e3e9f14199854cc2d586651c5809345aee18 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Wed, 8 May 2024 14:48:47 +0800 +Subject: [PATCH 82/85] fix shim controller set incorrect sandbox status state + +Signed-off-by: jikai +--- + src/daemon/sandbox/controller/controller.h | 3 +++ + src/daemon/sandbox/controller/shim/shim_controller.cc | 6 ++++-- + src/daemon/sandbox/sandbox.cc | 3 --- + 3 files changed, 7 insertions(+), 5 deletions(-) + +diff --git a/src/daemon/sandbox/controller/controller.h b/src/daemon/sandbox/controller/controller.h +index f479a0ac..9ad45855 100644 +--- a/src/daemon/sandbox/controller/controller.h ++++ b/src/daemon/sandbox/controller/controller.h +@@ -27,6 +27,9 @@ + + namespace sandbox { + ++#define SANDBOX_READY_STATE_STR "SANDBOX_READY" ++#define SANDBOX_NOTREADY_STATE_STR "SANDBOX_NOTREADY" ++ + struct ControllerMountInfo { + std::string source; + std::string destination; +diff --git a/src/daemon/sandbox/controller/shim/shim_controller.cc b/src/daemon/sandbox/controller/shim/shim_controller.cc +index 4da637c7..ce09c076 100644 +--- a/src/daemon/sandbox/controller/shim/shim_controller.cc ++++ b/src/daemon/sandbox/controller/shim/shim_controller.cc +@@ -446,8 +446,10 @@ void ShimController::InspectResponseToSandboxStatus(container_inspect *inspect, + sandboxStatus.id = inspect->id; + if (inspect->state != nullptr) { + sandboxStatus.pid = inspect->state->pid; +- if (inspect->state->status != nullptr) { +- sandboxStatus.state = std::string(inspect->state->status); ++ if (inspect->state->running) { ++ sandboxStatus.state = std::string(SANDBOX_READY_STATE_STR); ++ } else { ++ sandboxStatus.state = std::string(SANDBOX_NOTREADY_STATE_STR); + } + } + +diff --git a/src/daemon/sandbox/sandbox.cc b/src/daemon/sandbox/sandbox.cc +index 279bf628..d44abb99 100644 +--- a/src/daemon/sandbox/sandbox.cc ++++ b/src/daemon/sandbox/sandbox.cc +@@ -39,9 +39,6 @@ + #include "utils_timestamp.h" + #include "mailbox.h" + +-#define SANDBOX_READY_STATE_STR "SANDBOX_READY" +-#define SANDBOX_NOTREADY_STATE_STR "SANDBOX_NOTREADY" +- + namespace sandbox { + + const std::string SHM_MOUNT_POINT = "/dev/shm"; +-- +2.34.1 + diff --git a/0083-fix-bug-for-invalid-env-write.patch b/0083-fix-bug-for-invalid-env-write.patch new file mode 100644 index 0000000..527fe0f --- /dev/null +++ b/0083-fix-bug-for-invalid-env-write.patch @@ -0,0 +1,158 @@ +From fb48f036fece9d64c4cfc19c52091afad5f42fd9 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Sat, 11 May 2024 03:46:02 +0000 +Subject: [PATCH 83/85] fix bug for invalid env write + +Signed-off-by: jikai +--- + src/daemon/modules/spec/specs_extend.c | 57 +++++++++----------------- + src/utils/cutils/utils_verify.c | 25 +++++++++++ + src/utils/cutils/utils_verify.h | 2 + + 3 files changed, 46 insertions(+), 38 deletions(-) + +diff --git a/src/daemon/modules/spec/specs_extend.c b/src/daemon/modules/spec/specs_extend.c +index 4c154281..f4208405 100644 +--- a/src/daemon/modules/spec/specs_extend.c ++++ b/src/daemon/modules/spec/specs_extend.c +@@ -190,41 +190,33 @@ int make_userns_remap(oci_runtime_spec *container, const char *user_remap) + static int generate_env_map_from_file(FILE *fp, json_map_string_string *env_map) + { + int ret = 0; +- char *key = NULL; +- char *value = NULL; +- char *pline = NULL; ++ __isula_auto_free char *pline = NULL; + size_t length = 0; +- char *saveptr = NULL; +- char empty_str[1] = {'\0'}; + + while (getline(&pline, &length, fp) != -1) { ++ __isula_auto_free char *key = NULL; ++ __isula_auto_free char *value = NULL; + util_trim_newline(pline); + pline = util_trim_space(pline); + if (pline == NULL || pline[0] == '#') { + continue; + } +- key = strtok_r(pline, "=", &saveptr); +- value = strtok_r(NULL, "=", &saveptr); +- // value of an env varible is allowed to be empty +- value = value ? value : empty_str; +- if (key != NULL) { +- key = util_trim_space(key); +- value = util_trim_space(value); +- if ((size_t)(MAX_BUFFER_SIZE - 1) - strlen(key) < strlen(value)) { +- ERROR("env length exceed %d bytes", MAX_BUFFER_SIZE); +- ret = -1; +- goto out; +- } +- ret = append_json_map_string_string(env_map, key, value); +- if (ret < 0) { +- ERROR("append env to map failed"); +- goto out; +- } ++ if (util_valid_split_env(pline, &key, &value) < 0) { ++ // ignore invalid env ++ continue; ++ } ++ if ((size_t)(MAX_BUFFER_SIZE - 1) - strlen(key) < strlen(value)) { ++ ERROR("env length exceed %d bytes", MAX_BUFFER_SIZE); ++ return -1; ++ } ++ ret = append_json_map_string_string(env_map, key, value); ++ if (ret < 0) { ++ ERROR("append env to map failed"); ++ return -1; + } + } +-out: +- free(pline); +- return ret; ++ ++ return 0; + } + + static json_map_string_string *parse_env_target_file(const char *env_path) +@@ -293,28 +285,17 @@ static int do_append_env(char ***env, size_t *env_len, const char *key, const ch + static int check_env_need_append(const oci_runtime_spec *oci_spec, const char *env_key, bool *is_append) + { + size_t i = 0; +- char *key = NULL; +- char *saveptr = NULL; + + for (i = 0; i < oci_spec->process->env_len; i++) { +- char *tmp_env = NULL; +- tmp_env = util_strdup_s(oci_spec->process->env[i]); +- key = strtok_r(tmp_env, "=", &saveptr); +- // value of an env varible is allowed to be empty +- if (key == NULL) { ++ __isula_auto_free char *key = NULL; ++ if (util_valid_split_env(oci_spec->process->env[i], &key, NULL) < 0) { + ERROR("Bad env format"); +- free(tmp_env); +- tmp_env = NULL; + return -1; + } + if (strcmp(key, env_key) == 0) { + *is_append = false; +- free(tmp_env); +- tmp_env = NULL; + return 0; + } +- free(tmp_env); +- tmp_env = NULL; + } + return 0; + } +diff --git a/src/utils/cutils/utils_verify.c b/src/utils/cutils/utils_verify.c +index 474e28f0..6f1da12c 100644 +--- a/src/utils/cutils/utils_verify.c ++++ b/src/utils/cutils/utils_verify.c +@@ -651,6 +651,31 @@ bool util_valid_device_cgroup_rule(const char *value) + return util_reg_match(patten, value) == 0; + } + ++int util_valid_split_env(const char *env, char **key, char **value) ++{ ++ __isula_auto_array_t char **arr = NULL; ++ ++ arr = util_string_split_n(env, '=', 2); ++ if (arr == NULL) { ++ ERROR("Failed to split env string"); ++ return -1; ++ } ++ ++ if (strlen(arr[0]) == 0) { ++ ERROR("Invalid environment variable: %s", env); ++ return -1; ++ } ++ ++ if (key != NULL) { ++ *key = util_strdup_s(arr[0]); ++ } ++ if (value != NULL) { ++ *value = util_strdup_s(util_array_len((const char **)arr) > 1 ? arr[1] : ""); ++ } ++ ++ return 0; ++} ++ + int util_valid_env(const char *env, char **dst) + { + int ret = 0; +diff --git a/src/utils/cutils/utils_verify.h b/src/utils/cutils/utils_verify.h +index fc59f6c0..58b22b85 100644 +--- a/src/utils/cutils/utils_verify.h ++++ b/src/utils/cutils/utils_verify.h +@@ -119,6 +119,8 @@ bool util_valid_positive_interger(const char *value); + + bool util_valid_device_cgroup_rule(const char *value); + ++int util_valid_split_env(const char *env, char **key, char **value); ++ + int util_valid_env(const char *env, char **dst); + + bool util_valid_sysctl(const char *sysctl_key); +-- +2.34.1 + diff --git a/0084-trim-key-value-for-env.patch b/0084-trim-key-value-for-env.patch new file mode 100644 index 0000000..628f77d --- /dev/null +++ b/0084-trim-key-value-for-env.patch @@ -0,0 +1,26 @@ +From de9ed770a254c8f67ac228f56fe461e1c834609c Mon Sep 17 00:00:00 2001 +From: jikai +Date: Sat, 11 May 2024 07:51:35 +0000 +Subject: [PATCH 84/85] trim key/value for env + +Signed-off-by: jikai +--- + src/daemon/modules/spec/specs_extend.c | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/src/daemon/modules/spec/specs_extend.c b/src/daemon/modules/spec/specs_extend.c +index f4208405..926aaf3c 100644 +--- a/src/daemon/modules/spec/specs_extend.c ++++ b/src/daemon/modules/spec/specs_extend.c +@@ -205,6 +205,8 @@ static int generate_env_map_from_file(FILE *fp, json_map_string_string *env_map) + // ignore invalid env + continue; + } ++ key = util_trim_space(key); ++ value = util_trim_space(value); + if ((size_t)(MAX_BUFFER_SIZE - 1) - strlen(key) < strlen(value)) { + ERROR("env length exceed %d bytes", MAX_BUFFER_SIZE); + return -1; +-- +2.34.1 + diff --git a/0085-cdi-allow-env-variable-has-an-empty-value.patch b/0085-cdi-allow-env-variable-has-an-empty-value.patch new file mode 100644 index 0000000..a694d75 --- /dev/null +++ b/0085-cdi-allow-env-variable-has-an-empty-value.patch @@ -0,0 +1,135 @@ +From 9208d73274da0bd18c0d77cdf59ead3dc8e06021 Mon Sep 17 00:00:00 2001 +From: liuxu +Date: Fri, 10 May 2024 18:12:49 +0800 +Subject: [PATCH 85/85] cdi:allow env variable has an empty value + +Signed-off-by: liuxu +--- + src/daemon/modules/spec/specs.c | 28 ++++++---------------------- + test/specs/specs/specs_ut.cc | 16 ++++++++-------- + 2 files changed, 14 insertions(+), 30 deletions(-) + +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index 65a860d4..e779c22e 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -2607,17 +2607,11 @@ int spec_module_init(void) + static int add_env(defs_process *dp, const char *env, const char *key) + { + size_t i; +- char *oci_key = NULL; +- char *oci_value = NULL; +- char *saveptr = NULL; +- __isula_auto_free char *tmp_env = NULL; + + for (i = 0; i < dp->env_len; i++) { +- tmp_env = util_strdup_s(dp->env[i]); +- oci_key = strtok_r(tmp_env, "=", &saveptr); +- oci_value = strtok_r(NULL, "=", &saveptr); +- if (oci_key == NULL || oci_value == NULL) { +- ERROR("Bad env format"); ++ __isula_auto_free char *oci_key = NULL; ++ if (util_valid_split_env(dp->env[i], &oci_key, NULL) < 0) { ++ ERROR("Bad env format, %s", dp->env[i]); + return -1; + } + if (strcmp(key, oci_key) == 0) { +@@ -2625,8 +2619,6 @@ static int add_env(defs_process *dp, const char *env, const char *key) + dp->env[i] = util_strdup_s(env); + return 0; + } +- free(tmp_env); +- tmp_env = NULL; + } + if (util_mem_realloc((void **)&dp->env, (dp->env_len + 1) * sizeof(char *), + (void *)dp->env, dp->env_len * sizeof(char *)) != 0) { +@@ -2641,10 +2633,6 @@ static int add_env(defs_process *dp, const char *env, const char *key) + int defs_process_add_multiple_env(defs_process *dp, const char **envs, size_t env_len) + { + size_t i; +- char *key = NULL; +- char *value = NULL; +- char *saveptr = NULL; +- __isula_auto_free char *tmp_env = NULL; + + if (envs == NULL || env_len == 0) { + DEBUG("empty envs"); +@@ -2656,18 +2644,14 @@ int defs_process_add_multiple_env(defs_process *dp, const char **envs, size_t en + } + + for (i = 0; i < env_len; i++) { +- tmp_env = util_strdup_s(envs[i]); +- key = strtok_r(tmp_env, "=", &saveptr); +- value = strtok_r(NULL, "=", &saveptr); +- if (key == NULL || value == NULL) { +- ERROR("Bad env format: %s", tmp_env); ++ __isula_auto_free char *key = NULL; ++ if (util_valid_split_env(envs[i], &key, NULL) < 0) { ++ ERROR("Bad env format: %s", envs[i]); + return -1; + } + if (add_env(dp, envs[i], key) != 0) { + return -1; + } +- free(tmp_env); +- tmp_env = NULL; + } + + return 0; +diff --git a/test/specs/specs/specs_ut.cc b/test/specs/specs/specs_ut.cc +index 47836e5b..3f108f0f 100644 +--- a/test/specs/specs/specs_ut.cc ++++ b/test/specs/specs/specs_ut.cc +@@ -593,20 +593,20 @@ TEST_F(SpecsUnitTest, test_defs_process_add_multiple_env) + ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1); + free(envs[0]); + envs[0] = util_strdup_s("key0="); +- ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1); ++ ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), 0); + free(envs[0]); + envs[0] = util_strdup_s("key0xxxx"); +- ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1); ++ ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), 0); + + free(dp->env[0]); + dp->env[0] = util_strdup_s("=value0"); + ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1); + free(dp->env[0]); + dp->env[0] = util_strdup_s("key0="); +- ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1); ++ ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), 0); + free(dp->env[0]); + dp->env[0] = util_strdup_s("key0xxxx"); +- ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1); ++ ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), 0); + + free_defs_process(dp); + free(envs[0]); +@@ -644,20 +644,20 @@ TEST_F(SpecsUnitTest, test_spec_add_multiple_process_env) + ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1); + free(envs[0]); + envs[0] = util_strdup_s("key0="); +- ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1); ++ ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), 0); + free(envs[0]); + envs[0] = util_strdup_s("key0xxxx"); +- ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1); ++ ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), 0); + + free(oci_spec->process->env[0]); + oci_spec->process->env[0] = util_strdup_s("=value0"); + ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1); + free(oci_spec->process->env[0]); + oci_spec->process->env[0] = util_strdup_s("key0="); +- ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1); ++ ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), 0); + free(oci_spec->process->env[0]); + oci_spec->process->env[0] = util_strdup_s("key0xxxx"); +- ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1); ++ ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), 0); + + free_oci_runtime_spec(oci_spec); + free(envs[0]); +-- +2.34.1 + diff --git a/0086-cdi-test-case-and-gateway.patch b/0086-cdi-test-case-and-gateway.patch new file mode 100644 index 0000000..42cef23 --- /dev/null +++ b/0086-cdi-test-case-and-gateway.patch @@ -0,0 +1,359 @@ +From 3fb6de60ecf10278a676a41f8d0c334d1f90d303 Mon Sep 17 00:00:00 2001 +From: liuxu +Date: Wed, 17 Apr 2024 18:02:48 +0800 +Subject: [PATCH 086/108] cdi:test case and gateway + +Signed-off-by: liuxu +--- + CI/make-and-install.sh | 2 +- + CI/test_cases/container_cases/cdi_test.sh | 205 ++++++++++++++++++ + .../criconfigs/container-config-cdi.json | 50 +++++ + .../test_data/cdi_devices.json | 53 +++++ + 4 files changed, 309 insertions(+), 1 deletion(-) + create mode 100755 CI/test_cases/container_cases/cdi_test.sh + create mode 100644 CI/test_cases/container_cases/criconfigs/container-config-cdi.json + create mode 100644 CI/test_cases/container_cases/test_data/cdi_devices.json + +diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh +index 9d4c5533..61281965 100755 +--- a/CI/make-and-install.sh ++++ b/CI/make-and-install.sh +@@ -105,7 +105,7 @@ cd build + if [[ ${enable_gcov} -ne 0 ]]; then + cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_CRI_API_V1=ON -DENABLE_CDI=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON -DENABLE_GRPC_REMOTE_CONNECT=ON .. + else +- cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON -DENABLE_CRI_API_V1=ON -DENABLE_GRPC_REMOTE_CONNECT=ON .. ++ cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON -DENABLE_CRI_API_V1=ON -DENABLE_CDI=ON -DENABLE_GRPC_REMOTE_CONNECT=ON .. + fi + make -j $(nproc) + make install +diff --git a/CI/test_cases/container_cases/cdi_test.sh b/CI/test_cases/container_cases/cdi_test.sh +new file mode 100755 +index 00000000..dd7b1177 +--- /dev/null ++++ b/CI/test_cases/container_cases/cdi_test.sh +@@ -0,0 +1,205 @@ ++#!/bin/bash ++# ++# attributes: isulad cdi ++# concurrent: NA ++# spend time: 41 ++ ++####################################################################### ++##- Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++# - iSulad licensed under the Mulan PSL v2. ++# - You can use this software according to the terms and conditions of the Mulan PSL v2. ++# - You may obtain a copy of Mulan PSL v2 at: ++# - http://license.coscl.org.cn/MulanPSL2 ++# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++# - PURPOSE. ++# - See the Mulan PSL v2 for more details. ++##- @Description:CI ++##- @Author: liuxu ++##- @Create: 2024-04-16 ++####################################################################### ++ ++source ../helpers.sh ++curr_path=$(dirname $(readlink -f "$0")) ++data_path=$(realpath $curr_path/criconfigs) ++pause_img_path=$(realpath $curr_path/test_data) ++cdi_static_dir="/etc/cdi" ++ ++function do_pre() ++{ ++ cp /etc/isulad/daemon.json /etc/isulad/daemon.bak ++ sed -i "s#\"pod-sandbox-image\": \"\"#\"pod-sandbox-image\": \"mirrorgooglecontainers/pause-amd64:3.0\"#g" /etc/isulad/daemon.json ++ sed -i "/\"cni-conf-dir\": \".*\"/a\ \ \ \ \"enable-cri-v1\": true," /etc/isulad/daemon.json ++ sed -i "/\"cni-conf-dir\": \".*\"/a\ \ \ \ \"enable-cdi\": true," /etc/isulad/daemon.json ++ ++ check_valgrind_log ++ start_isulad_without_valgrind ++ ++ isula load -i ${pause_img_path}/pause.tar ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to load pause" && return ${FAILURE} ++ ++ isula pull busybox ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull busybox" && return ${FAILURE} ++ ++ crictl images | grep "mirrorgooglecontainers/pause-amd64" ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Failed to find mirrorgooglecontainers/pause-amd64 image" && return ${FAILURE} ++ ++ return 0 ++} ++ ++function do_post() ++{ ++ cp -f /etc/isulad/daemon.bak /etc/isulad/daemon.json ++ check_valgrind_log ++ start_isulad_without_valgrind ++} ++ ++function verify_injected_vendor0() { ++ # check env ++ output=$(crictl exec --sync "$1" sh -c 'echo $VENDOR0') ++ [[ "$output" != "injected" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - env check failed" && return ${FAILURE} ++ ++ # check hooks ++ cat /tmp/cdi_hook_test.log | grep "prestart" ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - hook check failed" && return ${FAILURE} ++ ++ # check mounts ++ output=$(crictl exec --sync "$1" sh -c 'stat -c %a /tmp/cdi_mounts_test') ++ [[ "$output" != "755" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - mount check failed" && return ${FAILURE} ++ ++ return 0 ++} ++ ++function verify_injected_loop8() { ++ # check env ++ output=$(crictl exec --sync "$1" sh -c 'echo $LOOP8') ++ [[ "$output" != "CDI8" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - env check failed" && return ${FAILURE} ++ ++ # check device nodes ++ output=$(crictl exec --sync "$1" sh -c 'stat -c %a /dev/loop8') ++ [[ "$output" != "640" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - device nodes check failed" && return ${FAILURE} ++ output=$(crictl exec --sync "$1" sh -c 'stat -c %t.%T /dev/loop8') ++ [[ "$output" != "7.8" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - device nodes check failed" && return ${FAILURE} ++ output=$(crictl exec --sync "$1" sh -c 'stat -c %t.%T /dev/loop8c') ++ [[ "$output" != "7.b" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - device nodes check failed" && return ${FAILURE} ++ ++ # check mounts ++ output=$(crictl exec --sync "$1" sh -c 'stat -c %a /tmp/cdi_mounts_test_loop8') ++ [[ "$output" != "755" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - mount check failed" && return ${FAILURE} ++ ++ return 0 ++} ++ ++function verify_injected_loop9() { ++ # check env ++ output=$(crictl exec --sync "$1" sh -c 'echo $LOOP9') ++ [[ "$output" != "present" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - env check failed" && return ${FAILURE} ++ ++ # check device nodes ++ output=$(crictl exec --sync "$1" sh -c 'stat -c %a /dev/loop9') ++ [[ "$output" != "644" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - device nodes check failed" && return ${FAILURE} ++ output=$(crictl exec --sync "$1" sh -c 'stat -c %t.%T /dev/loop9') ++ [[ "$output" != "7.9" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - device nodes check failed" && return ${FAILURE} ++ ++ return 0 ++} ++ ++function check_full_cdi() ++{ ++ verify_injected_vendor0 $1 ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - verify_injected_vendor0 failed" && return ${FAILURE} ++ ++ verify_injected_loop8 $1 ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - verify_injected_loop8 failed" && return ${FAILURE} ++ ++ verify_injected_loop9 $1 ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - verify_injected_loop9 failed" && return ${FAILURE} ++ ++ return 0 ++} ++ ++function do_test_help() ++{ ++ msg_info "cdi test starting..." ++ ++ isula rm -f `isula ps -a -q` ++ ++ sid=`crictl runp ${data_path}/$1` ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Failed to run sandbox" && return ${FAILURE} ++ ++ cid=`crictl create $sid ${data_path}/$2 ${data_path}/$1` ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - create container failed" && return ${FAILURE} ++ ++ crictl start $cid ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start container failed" && return ${FAILURE} ++ ++ crictl stats ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stats container failed" && return ${FAILURE} ++ ++ check_full_cdi $cid ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check cdi failed" && return ${FAILURE} ++ ++ crictl stop $cid ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop container failed" && return ${FAILURE} ++ ++ crictl rm $cid ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - rm container failed" && return ${FAILURE} ++ ++ crictl stopp $sid ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop sandbox failed" && return ${FAILURE} ++ ++ crictl rmp $sid ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - rm sandbox failed" && return ${FAILURE} ++ ++ return 0 ++} ++ ++function do_test_full_cdi() ++{ ++ if [ ! -d "$cdi_static_dir" ]; then ++ mkdir -p "$cdi_static_dir" ++ fi ++ cp -f ${pause_img_path}/cdi_devices.json ${cdi_static_dir}/cdi_devices.json ++ ++ mkdir -p /tmp/cdi_mounts_test ++ cat > /tmp/cdi_mounts_test_loop8 << EOF ++origin data ++EOF ++ chmod 755 /tmp/cdi_mounts_test_loop8 ++ mkdir -p /tmp/cdi_mounts_test_loop9 ++ ++ mknod /dev/loop8 b 7 8 ++ mknod /dev/loop9 b 7 9 ++ mknod /dev/loop8c c 7 11 ++ ++ cat > /tmp/cdi_printargs.sh << EOF ++#!/bin/bash ++echo "\$(date +'%Y-%m-%d %H:%M:%S') Input parameter: \$1 \$2" >> /tmp/cdi_hook_test.log ++EOF ++ chmod 755 /tmp/cdi_printargs.sh ++ ++ do_test_help "sandbox-config.json" "container-config-cdi.json" || ((ans++)) ++ ++ rm -f /tmp/cdi_printargs.sh ++ rm -f /tmp/cdi_hook_test.log ++ rm -f /dev/loop8 ++ rm -f /dev/loop9 ++ rm -f /dev/loop8c ++ ++ rm -f ${cdi_static_dir}/cdi_devices.json ++ rm -f /tmp/cdi_printargs ++ rmdir /tmp/cdi_mounts_test ++ rm -f /tmp/cdi_mounts_test_loop8 ++ rmdir /tmp/cdi_mounts_test_loop9 ++ rm -f /tmp/cdi_printargs.sh ++ ++ return 0 ++} ++ ++declare -i ans=0 ++ ++do_pre || ((ans++)) ++do_test_full_cdi || ((ans++)) ++do_post ++ ++show_result ${ans} "${curr_path}/${0}" +diff --git a/CI/test_cases/container_cases/criconfigs/container-config-cdi.json b/CI/test_cases/container_cases/criconfigs/container-config-cdi.json +new file mode 100644 +index 00000000..b9805c8a +--- /dev/null ++++ b/CI/test_cases/container_cases/criconfigs/container-config-cdi.json +@@ -0,0 +1,50 @@ ++{ ++ "metadata": { ++ "name": "haozi" ++ }, ++ "image":{ ++ "image": "busybox:latest" ++ }, ++ "command": [ ++ "/bin/sh", ++ "-c", ++ "i=0; while true; do echo \"$i: $(date)\"; i=$((i+1)); sleep 10; done" ++ ], ++ "labels": { ++ "filter_label_key": "filter_label_val" ++ }, ++ "annotations": { ++ "extension.network.kubernetes.io/cni/instancename": "pod_instance_name", ++ "cdi.k8s.io/test": "vendor0.com/device=loop8,vendor0.com/device=loop9" ++ }, ++ "CDI_Devices":[ ++ { "Name": "vendor0.com/device=loop8" }, ++ { "Name": "vendor0.com/device=loop9" } ++ ], ++ "mounts" : [ ++ { ++ "container_path": "/tmp/contpath", ++ "host_path": "/tmp/hostpath", ++ "readonly": true, ++ "selinux_relabel": true, ++ "propagation": 0 ++ }, ++ { ++ "container_path": "/tmp/contpath2", ++ "host_path": "/tmp/hostpath2", ++ "readonly": false, ++ "selinux_relabel": false, ++ "propagation": 0 ++ } ++ ], ++ "linux": { ++ "security_context": { ++ "namespace_options": { ++ "host_network": true, ++ "host_pid": false, ++ "host_ipc": false ++ } ++ } ++ }, ++ "log_path": "cri_test.log" ++} +diff --git a/CI/test_cases/container_cases/test_data/cdi_devices.json b/CI/test_cases/container_cases/test_data/cdi_devices.json +new file mode 100644 +index 00000000..f7fe65d4 +--- /dev/null ++++ b/CI/test_cases/container_cases/test_data/cdi_devices.json +@@ -0,0 +1,53 @@ ++{ ++ "cdiVersion": "0.6.0", ++ "kind": "vendor0.com/device", ++ "annotations":{ ++ "cdi_annotation":"cdi_annotation_value" ++ }, ++ "devices": [ ++ { ++ "name": "loop8", ++ "annotations":{ ++ "loop8_annotation":"loop8_annotation_value" ++ }, ++ "containerEdits": { ++ "env": [ ++ "LOOP8=CDI8", ++ "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/dev" ++ ], ++ "deviceNodes": [ ++ {"path": "/dev/loop8", "type": "b", "fileMode": 416}, ++ {"path": "/dev/loop8c", "type": "c"} ++ ], ++ "mounts": [ ++ {"hostPath": "/tmp/cdi_mounts_test_loop8", "containerPath": "/tmp/cdi_mounts_test_loop8", "options": ["ro","nosuid","nodev","bind"]}, ++ {"hostPath": "tmpfs", "containerPath": "/tmp/data", "type": "tmpfs", "options": ["nosuid","strictatime","mode=755","size=65536k"]} ++ ] ++ } ++ },{ ++ "name": "loop9", ++ "containerEdits": { ++ "env":["LOOP9=present"], ++ "deviceNodes": [{ ++ "path": "/dev/loop9", "type": "b", "major": 7, "minor": 9, "fileMode": 420 ++ } ++ ] ++ } ++ } ++ ], ++ "containerEdits": { ++ "env": [ ++ "VENDOR0=injected", ++ "BAR=BARVALUE1" ++ ], ++ "hooks": [ ++ {"hookName": "prestart", "path": "/tmp/cdi_printargs.sh", "args":["prestart0", "prestart1"], "env":["prestartenv=value"]}, ++ {"hookName": "prestart", "path": "/tmp/cdi_printargs.sh", "args":["prestart0", "prestart1"], "env":["prestartenv=value"]}, ++ {"hookName": "poststart", "path": "/tmp/cdi_printargs.sh", "args":["poststart0", "poststart1"], "env":["poststartenv=value"]}, ++ {"hookName": "poststop", "path": "/tmp/cdi_printargs.sh", "args":["poststop0", "poststop1"], "env":["poststopenv=value"]} ++ ], ++ "mounts": [ ++ {"hostPath": "/tmp/cdi_mounts_test", "containerPath": "/tmp/cdi_mounts_test", "options": ["ro","nosuid","nodev","bind"]} ++ ] ++ } ++} +\ No newline at end of file +-- +2.25.1 + diff --git a/0087-code-improve.patch b/0087-code-improve.patch new file mode 100644 index 0000000..66f9f05 --- /dev/null +++ b/0087-code-improve.patch @@ -0,0 +1,1690 @@ +From e17bb2f5995d261790c453d81a453d25024450d1 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Mon, 13 May 2024 17:06:07 +1400 +Subject: [PATCH 087/108] code improve + +Signed-off-by: zhongtao +--- + src/client/connect/protocol_type.c | 2 +- + src/cmd/isula/base/create.c | 3 +- + src/cmd/isula/inspect_format.c | 4 +- + src/cmd/isulad-shim/common.c | 2 +- + src/cmd/isulad-shim/main.c | 5 +- + src/cmd/isulad-shim/process.c | 18 +++---- + src/daemon/common/cgroup/cgroup.c | 9 ++-- + src/daemon/common/cgroup/cgroup.h | 9 ++-- + src/daemon/common/cgroup/cgroup_common.h | 9 ++-- + src/daemon/common/cgroup/cgroup_v1.c | 10 ++-- + src/daemon/common/cgroup/cgroup_v2.c | 9 ++-- + src/daemon/common/cri/cri_helpers.cc | 4 +- + src/daemon/common/cri/v1/v1_cri_helpers.cc | 11 ++-- + src/daemon/common/sysinfo.c | 4 +- + src/daemon/config/isulad_config.c | 2 +- + .../cri/v1/cri_v1_runtime_runtime_service.cc | 5 +- + .../v1/v1_cri_container_manager_service.cc | 3 +- + .../v1/v1_cri_pod_sandbox_manager_service.cc | 8 +-- + .../v1/v1_cri_pod_sandbox_manager_service.h | 6 ++- + .../cri/v1/v1_cri_runtime_service_impl.cc | 3 +- + .../cri/v1/v1_cri_runtime_service_impl.h | 3 +- + .../executor/container_cb/execution_create.c | 2 +- + src/daemon/mailbox/mailbox.c | 3 +- + src/daemon/mailbox/mailbox.h | 4 +- + src/daemon/mailbox/mailbox_message.c | 9 ++-- + src/daemon/mailbox/mailbox_message.h | 4 +- + src/daemon/modules/api/container_api.h | 3 +- + src/daemon/modules/api/network_api.h | 2 +- + .../modules/container/supervisor/supervisor.c | 3 +- + .../device/cdi/behavior/cdi_container_edits.c | 46 ++++++++-------- + .../modules/device/cdi/behavior/cdi_device.c | 16 +++--- + .../modules/device/cdi/behavior/cdi_spec.c | 10 ++-- + .../modules/device/cdi/behavior/cdi_spec.h | 2 +- + .../device/cdi/behavior/cdi_spec_dirs.c | 6 +-- + .../device/cdi/behavior/cdi_spec_dirs.h | 2 +- + .../modules/device/cdi/behavior/cdi_version.c | 2 +- + .../device/cdi/behavior/parser/cdi_parser.c | 6 +-- + src/daemon/modules/device/cdi/cdi_cache.c | 22 ++++---- + src/daemon/modules/device/cdi/cdi_cache.h | 6 +-- + src/daemon/modules/device/cdi_operate.c | 8 +-- + src/daemon/modules/image/oci/oci_image.c | 3 +- + src/daemon/modules/image/oci/progress.h | 2 +- + .../modules/image/oci/registry/registry.c | 3 +- + .../cni_operator/libcni/invoke/libcni_exec.h | 2 +- + src/daemon/modules/network/cri/adaptor_cri.c | 2 +- + src/daemon/modules/network/cri/adaptor_cri.h | 4 +- + .../modules/runtime/engines/lcr/lcr_rt_ops.c | 6 ++- + .../modules/runtime/isula/isula_rt_ops.c | 9 ++-- + .../modules/service/service_container.c | 5 +- + src/daemon/modules/spec/specs.c | 52 ++++++++++--------- + src/daemon/modules/spec/specs_mount.c | 4 +- + src/utils/cutils/blocking_queue.c | 3 +- + src/utils/cutils/network_namespace.h | 4 +- + src/utils/cutils/utils_array.c | 12 ++--- + src/utils/cutils/utils_port.h | 2 +- + src/utils/cutils/utils_verify.c | 4 +- + src/utils/progress/show.h | 8 +-- + src/utils/tar/isulad_tar.c | 2 +- + test/cgroup/cpu/cgroup_cpu_ut.cc | 2 +- + 59 files changed, 223 insertions(+), 191 deletions(-) + +diff --git a/src/client/connect/protocol_type.c b/src/client/connect/protocol_type.c +index 4d530aea..57dcf753 100644 +--- a/src/client/connect/protocol_type.c ++++ b/src/client/connect/protocol_type.c +@@ -72,7 +72,7 @@ int isula_filters_last_parse_args(size_t last_n, struct isula_filters **flt) + goto cleanup; + } + +- ret = snprintf(value, VALUE_SIZE, "%ld", last_n); ++ ret = snprintf(value, VALUE_SIZE, "%lu", last_n); + if (ret < 0 || ret >= VALUE_SIZE) { + ret = -1; + ERROR("Sprintf lastest n containers args failed"); +diff --git a/src/cmd/isula/base/create.c b/src/cmd/isula/base/create.c +index b8e3667e..543b8fd6 100644 +--- a/src/cmd/isula/base/create.c ++++ b/src/cmd/isula/base/create.c +@@ -932,7 +932,8 @@ inline static int request_pack_host_device_cgroup_rules(const struct client_argu + + inline static int request_pack_host_blockio(const struct client_arguments *args, isula_host_config_t *hostconfig) + { +- return (request_pack_host_weight_devices(args, hostconfig) != 0 || request_pack_host_device_read_bps(args, hostconfig) != 0 || ++ return (request_pack_host_weight_devices(args, hostconfig) != 0 || ++ request_pack_host_device_read_bps(args, hostconfig) != 0 || + request_pack_host_device_write_bps(args, hostconfig) != 0 || + request_pack_host_device_read_iops(args, hostconfig) != 0 || + request_pack_host_device_write_iops(args, hostconfig) != 0); +diff --git a/src/cmd/isula/inspect_format.c b/src/cmd/isula/inspect_format.c +index 36e8a31c..1fd67924 100644 +--- a/src/cmd/isula/inspect_format.c ++++ b/src/cmd/isula/inspect_format.c +@@ -26,12 +26,12 @@ + + #define PRINTF_TAB_LEN 4 + #define TOP_LEVEL_OBJ 0x10 +-#define IS_TOP_LEVEL_OBJ(value) ((value)&TOP_LEVEL_OBJ) ++#define IS_TOP_LEVEL_OBJ(value) ((value) & TOP_LEVEL_OBJ) + + #define LAST_ELEMENT_BIT 0x0F + #define NOT_LAST_ELEMENT 0x00 + #define LAST_ELEMENT 0x01 +-#define IS_LAST_ELEMENT(value) (LAST_ELEMENT == ((value)&LAST_ELEMENT_BIT)) ++#define IS_LAST_ELEMENT(value) (LAST_ELEMENT == ((value) & LAST_ELEMENT_BIT)) + + #define YAJL_TYPEOF(json) ((json)->type) + +diff --git a/src/cmd/isulad-shim/common.c b/src/cmd/isulad-shim/common.c +index a288c170..799da3c2 100644 +--- a/src/cmd/isulad-shim/common.c ++++ b/src/cmd/isulad-shim/common.c +@@ -40,7 +40,7 @@ + static __thread char *g_shim_errmsg = NULL; + + // currently, log_to_stderr is only modified in the main process +-// and there is no need to set a thread-local variable. ++// and there is no need to set a thread-local variable. + // if it can be modified by multiple threads in the future, + // this variable needs to be set as a thread-local variable. + static bool log_to_stderr = false; +diff --git a/src/cmd/isulad-shim/main.c b/src/cmd/isulad-shim/main.c +index 54876d44..9061e568 100644 +--- a/src/cmd/isulad-shim/main.c ++++ b/src/cmd/isulad-shim/main.c +@@ -103,7 +103,7 @@ int main(int argc, char **argv) + engine_log_path = getenv(SHIIM_LOG_PATH_ENV); + if (engine_log_path == NULL) { + dprintf(STDERR_FILENO, "empty SHIIM_LOG_PATH_ENV"); +- _exit(EXIT_FAILURE); ++ _exit(EXIT_FAILURE); + } + + log_level = getenv(SHIIM_LOG_LEVEL_ENV); +@@ -149,7 +149,8 @@ int main(int argc, char **argv) + + // If isulad-shim is a child process of the isulad process, + // print the log to stderr so that isulad can obtain the exit information of isulad-shim. +- set_log_to_stderr((p->state->exec) && (p->state->isulad_stdin != NULL || p->state->isulad_stdout != NULL || p->state->isulad_stderr != NULL)); ++ set_log_to_stderr((p->state->exec) && (p->state->isulad_stdin != NULL || p->state->isulad_stdout != NULL || ++ p->state->isulad_stderr != NULL)); + + /* + * Open exit pipe +diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c +index 6b5f8f7f..18fae03f 100644 +--- a/src/cmd/isulad-shim/process.c ++++ b/src/cmd/isulad-shim/process.c +@@ -199,7 +199,7 @@ static int add_attach_terminal_fifos(const char *in, const char *out, const char + struct isula_linked_list *node = NULL; + + bool invalid = (in != NULL && !fifo_exists(in)) || (out != NULL && !fifo_exists(out)) || (err != NULL && +- !fifo_exists(err)); ++ !fifo_exists(err)); + if (invalid) { + ERROR("File %s or %s or %s does not refer to a FIFO", in, out, err); + return -1; +@@ -371,7 +371,7 @@ static int stdout_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t + (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF); + + r_count = isula_file_read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF); +- if (r_count <= 0 ) { ++ if (r_count <= 0) { + isula_epoll_remove_handler(descr, fd); + // fd cannot be closed here, which will cause the container process to exit abnormally + // due to terminal fd receiving the sighup signal. +@@ -417,7 +417,7 @@ static int stderr_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t + (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF); + + r_count = isula_file_read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF); +- if (r_count <= 0 ) { ++ if (r_count <= 0) { + isula_epoll_remove_handler(descr, fd); + // fd cannot be closed here, which will cause the container process to exit abnormally + // due to terminal fd receiving the sighup signal. +@@ -563,7 +563,7 @@ static int attach_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t + // limit the number of attach connections to MAX_ATTACH_NUM + if (isula_linked_list_len(p->attach_fifos) >= MAX_ATTACH_NUM) { + ERROR("The number of attach connections exceeds the limit:%d, and this connection is rejected.", +- MAX_ATTACH_NUM); ++ MAX_ATTACH_NUM); + goto out; + } + +@@ -934,8 +934,8 @@ static void *io_epoll_loop(void *data) + + (void)sem_post(&p->sem_mainloop); + +- // th frist epoll_loop will exit in the following scenarios: +- // 1. Receive sync fd event ++ // th frist epoll_loop will exit in the following scenarios: ++ // 1. Receive sync fd event + // 2. stdin fd receive EPOLLHUP event + // 3. stdin fd read failed + ret = isula_epoll_loop(&descr, -1); +@@ -945,8 +945,8 @@ static void *io_epoll_loop(void *data) + error_exit(EXIT_FAILURE); + } + +- // use a timeout epoll loop to ensure complete data reception +- // th second epoll_loop will exit in the following scenarios: ++ // use a timeout epoll loop to ensure complete data reception ++ // th second epoll_loop will exit in the following scenarios: + // 1. both stdout fd and stderr fd failed to read + // 2. no event received within 100 milliseconds + ret = isula_epoll_loop(&descr, 100); +@@ -1408,7 +1408,7 @@ static void exec_runtime_process(process_t *p, int exec_fd) + // the standard streams of the child process are set to /dev/null to prevent incorrect information acquisition. + if (isula_null_stdfds() != 0) { + (void)dprintf(exec_fd, "failed to set std console to /dev/null"); +- exit(EXIT_FAILURE); ++ exit(EXIT_FAILURE); + } + } + +diff --git a/src/daemon/common/cgroup/cgroup.c b/src/daemon/common/cgroup/cgroup.c +index 007dbb70..71bf9801 100644 +--- a/src/daemon/common/cgroup/cgroup.c ++++ b/src/daemon/common/cgroup/cgroup.c +@@ -82,9 +82,9 @@ int common_get_cgroup_version(void) + } + + int common_get_cgroup_info(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, +- cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, +- cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, +- cgroup_files_info_t *filesinfo, bool quiet) ++ cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, ++ cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, ++ cgroup_files_info_t *filesinfo, bool quiet) + { + if (g_cgroup_ops.get_cgroup_info == NULL) { + ERROR("Unimplemented get_cgroup_info ops"); +@@ -197,7 +197,8 @@ char *common_convert_cgroup_path(const char *cgroup_path) + return util_strdup_s(result); + } + +-cgroup_oom_handler_info_t *common_get_cgroup_oom_handler(int fd, const char *name, const char *cgroup_path, const char *exit_fifo) ++cgroup_oom_handler_info_t *common_get_cgroup_oom_handler(int fd, const char *name, const char *cgroup_path, ++ const char *exit_fifo) + { + if (g_cgroup_ops.get_cgroup_oom_handler == NULL) { + ERROR("Unimplmented get_cgroup_oom_handler op"); +diff --git a/src/daemon/common/cgroup/cgroup.h b/src/daemon/common/cgroup/cgroup.h +index 8c76d99d..0bbb70a0 100644 +--- a/src/daemon/common/cgroup/cgroup.h ++++ b/src/daemon/common/cgroup/cgroup.h +@@ -31,9 +31,9 @@ int cgroup_ops_init(void); + + int common_get_cgroup_version(void); + int common_get_cgroup_info(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, +- cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, +- cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, +- cgroup_files_info_t *filesinfo, bool quiet); ++ cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, ++ cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, ++ cgroup_files_info_t *filesinfo, bool quiet); + int common_get_cgroup_metrics(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics); + int common_get_cgroup_mnt_and_root_path(const char *subsystem, char **mountpoint, char **root); + +@@ -43,7 +43,8 @@ char *common_get_own_cgroup_path(const char *subsystem); + + char *common_convert_cgroup_path(const char *cgroup_path); + +-cgroup_oom_handler_info_t *common_get_cgroup_oom_handler(int fd, const char *name, const char *cgroup_path, const char *exit_fifo); ++cgroup_oom_handler_info_t *common_get_cgroup_oom_handler(int fd, const char *name, const char *cgroup_path, ++ const char *exit_fifo); + void common_free_cgroup_oom_handler_info(cgroup_oom_handler_info_t *info); + + #ifdef __cplusplus +diff --git a/src/daemon/common/cgroup/cgroup_common.h b/src/daemon/common/cgroup/cgroup_common.h +index e3912bf0..46a7de50 100644 +--- a/src/daemon/common/cgroup/cgroup_common.h ++++ b/src/daemon/common/cgroup/cgroup_common.h +@@ -130,9 +130,9 @@ typedef struct _cgroup_oom_handler_info_t { + typedef struct { + int (*get_cgroup_version)(void); + int (*get_cgroup_info)(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, +- cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, +- cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, +- cgroup_files_info_t *filesinfo, bool quiet); ++ cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, ++ cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, ++ cgroup_files_info_t *filesinfo, bool quiet); + int (*get_cgroup_metrics)(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics); + + int (*get_cgroup_mnt_and_root_path)(const char *subsystem, char **mountpoint, char **root); +@@ -140,7 +140,8 @@ typedef struct { + char *(*get_init_cgroup_path)(const char *subsystem); + char *(*get_own_cgroup_path)(const char *subsystem); + +- cgroup_oom_handler_info_t *(*get_cgroup_oom_handler)(int fd, const char *name, const char *cgroup_path, const char *exit_fifo); ++ cgroup_oom_handler_info_t *(*get_cgroup_oom_handler)(int fd, const char *name, const char *cgroup_path, ++ const char *exit_fifo); + } cgroup_ops; + + #ifdef __cplusplus +diff --git a/src/daemon/common/cgroup/cgroup_v1.c b/src/daemon/common/cgroup/cgroup_v1.c +index 41f3110a..45b1d096 100644 +--- a/src/daemon/common/cgroup/cgroup_v1.c ++++ b/src/daemon/common/cgroup/cgroup_v1.c +@@ -772,7 +772,8 @@ static void get_cgroup_v1_blkio_info(const cgroup_layer_t *layers, const bool qu + blkioinfo->blkio_write_iops_device = check_cgroup_v1_file_exists(mountpoint, BLKIO_WRITE_IOPS, quiet); + } + +-static void get_cgroup_v1_hugetlb_info(const cgroup_layer_t *layers, const bool quiet, cgroup_hugetlb_info_t *hugetlbinfo) ++static void get_cgroup_v1_hugetlb_info(const cgroup_layer_t *layers, const bool quiet, ++ cgroup_hugetlb_info_t *hugetlbinfo) + { + int nret; + char *mountpoint = NULL; +@@ -1038,8 +1039,8 @@ static char *common_get_cgroup_path(const char *path, const char *subsystem) + + for (i = 0; i < util_array_len((const char **)nlist); i++) { + const char *prefix = "name="; +- bool find_sub = (strcmp(nlist[i], subsystem) == 0 || (strncmp(nlist[i], prefix, strlen(prefix)) == 0 +- && strcmp(nlist[i]+strlen(prefix), subsystem) == 0)); ++ bool find_sub = (strcmp(nlist[i], subsystem) == 0 || (strncmp(nlist[i], prefix, strlen(prefix)) == 0 && ++ strcmp(nlist[i] + strlen(prefix), subsystem) == 0)); + if (find_sub) { + res = util_strdup_s(plist[i]); + break; +@@ -1135,7 +1136,8 @@ static char *get_memory_cgroup_path_v1(const char *cgroup_path) + return util_path_join(fpath, converted_cgroup_path); + } + +-static cgroup_oom_handler_info_t *get_cgroup_oom_handler_v1(int fd, const char *name, const char *cgroup_path, const char *exit_fifo) ++static cgroup_oom_handler_info_t *get_cgroup_oom_handler_v1(int fd, const char *name, const char *cgroup_path, ++ const char *exit_fifo) + { + __isula_auto_free char *memory_cgroup_path = NULL; + __isula_auto_free char *memory_cgroup_oom_control_path = NULL; +diff --git a/src/daemon/common/cgroup/cgroup_v2.c b/src/daemon/common/cgroup/cgroup_v2.c +index a36258f0..76754dc1 100644 +--- a/src/daemon/common/cgroup/cgroup_v2.c ++++ b/src/daemon/common/cgroup/cgroup_v2.c +@@ -387,7 +387,7 @@ static int get_cgroup_info_v2(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpu + if (ret != 0) { + return ret; + } +- ++ + get_cgroup_v2_pids_info(quiet, pidsinfo); + get_cgroup_v2_files_info(quiet, filesinfo); + +@@ -433,7 +433,7 @@ static bool oom_cb_cgroup_v2(int fd, void *cbdata) + return CGROUP_OOM_HANDLE_CLOSE; + } + +- if (((struct inotify_event *)events)->mask & ( IN_DELETE | IN_DELETE_SELF)) { ++ if (((struct inotify_event *)events)->mask & (IN_DELETE | IN_DELETE_SELF)) { + return CGROUP_OOM_HANDLE_CLOSE; + } + +@@ -450,7 +450,7 @@ static bool oom_cb_cgroup_v2(int fd, void *cbdata) + int count; + const char *oom_str = "oom "; + const char *oom_kill_str = "oom_kill "; +- const int oom_len = strlen(oom_str), oom_kill_len = strlen(oom_kill_str); ++ const size_t oom_len = strlen(oom_str), oom_kill_len = strlen(oom_kill_str); + + if (read >= oom_kill_len + 2 && memcmp(line, oom_kill_str, oom_kill_len) == 0) { + len = oom_kill_len; +@@ -492,7 +492,8 @@ static char *get_real_cgroup_path_v2(const char *cgroup_path) + return util_path_join(CGROUP_MOUNTPOINT, converted_cgroup_path); + } + +-cgroup_oom_handler_info_t *get_cgroup_oom_handler_v2(int fd, const char *name, const char *cgroup_path, const char *exit_fifo) ++cgroup_oom_handler_info_t *get_cgroup_oom_handler_v2(int fd, const char *name, const char *cgroup_path, ++ const char *exit_fifo) + { + __isula_auto_free char *real_cgroup_path = NULL; + if (name == NULL || cgroup_path == NULL || exit_fifo == NULL) { +diff --git a/src/daemon/common/cri/cri_helpers.cc b/src/daemon/common/cri/cri_helpers.cc +index d7ec9f36..68d569cc 100644 +--- a/src/daemon/common/cri/cri_helpers.cc ++++ b/src/daemon/common/cri/cri_helpers.cc +@@ -550,8 +550,8 @@ void CreateContainerLogSymlink(const std::string &containerID, Errors &error) + WARN("Deleted previously existing symlink file: %s", path.c_str()); + } + if (symlink(realPath.c_str(), path.c_str()) != 0) { +- SYSERROR("failed to create symbolic link %s to the container log file %s for container %s", path.c_str(), realPath.c_str(), +- containerID.c_str()); ++ SYSERROR("failed to create symbolic link %s to the container log file %s for container %s", path.c_str(), ++ realPath.c_str(), containerID.c_str()); + error.Errorf("failed to create symbolic link %s to the container log file %s for container %s", path.c_str(), + realPath.c_str(), containerID.c_str()); + } +diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.cc b/src/daemon/common/cri/v1/v1_cri_helpers.cc +index 1f797ad7..478dd105 100644 +--- a/src/daemon/common/cri/v1/v1_cri_helpers.cc ++++ b/src/daemon/common/cri/v1/v1_cri_helpers.cc +@@ -636,7 +636,8 @@ void ContainerStatusToGRPC(container_inspect *inspect, + ConvertResourcesToStatus(inspect, contStatus); + } + +-std::unique_ptr GetContainerStatus(service_executor_t *m_cb, const std::string &containerID, Errors &error) ++std::unique_ptr GetContainerStatus(service_executor_t *m_cb, ++ const std::string &containerID, Errors &error) + { + if (m_cb == nullptr) { + error.SetError("Invalid input arguments: empty service executor"); +@@ -694,7 +695,7 @@ static int InsertCDIDevices(std::unordered_set &fromCRI, const std: + } + return 0; + } +- ++ + void GenerateCDIRequestedDevices(const runtime::v1::ContainerConfig &config, host_config *hostconfig, Errors &err) + { + std::unordered_set fromCRI; +@@ -703,13 +704,13 @@ void GenerateCDIRequestedDevices(const runtime::v1::ContainerConfig &config, hos + __isula_auto_string_array_t string_array *devices = nullptr; + json_map_string_string *annotations = nullptr; + __isula_auto_free char *error = nullptr; +- ++ + if (hostconfig == nullptr) { + ERROR("Invalid input arguments"); + err.Errorf("Invalid input arguments"); + return; + } +- ++ + if (config.cdi_devices().empty() && config.annotations().empty()) { + return; + } +@@ -746,7 +747,7 @@ void GenerateCDIRequestedDevices(const runtime::v1::ContainerConfig &config, hos + requested->items = nullptr; + hostconfig->cdi_requested_devices_len = requested->len; + requested->len = 0; +- ++ + free_out: + free_json_map_string_string(annotations); + } +diff --git a/src/daemon/common/sysinfo.c b/src/daemon/common/sysinfo.c +index e369c3e3..ed1c01a3 100644 +--- a/src/daemon/common/sysinfo.c ++++ b/src/daemon/common/sysinfo.c +@@ -384,8 +384,8 @@ sysinfo_t *get_sys_info(bool quiet) + sysinfo->ncpus_conf = get_nprocs_conf(); + + ret = common_get_cgroup_info(&sysinfo->cgmeminfo, &sysinfo->cgcpuinfo, &sysinfo->hugetlbinfo, +- &sysinfo->blkioinfo, &sysinfo->cpusetinfo, &sysinfo->pidsinfo, +- &sysinfo->filesinfo, quiet); ++ &sysinfo->blkioinfo, &sysinfo->cpusetinfo, &sysinfo->pidsinfo, ++ &sysinfo->filesinfo, quiet); + if (ret != 0) { + goto out; + } +diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c +index 617db7a2..80689bce 100644 +--- a/src/daemon/config/isulad_config.c ++++ b/src/daemon/config/isulad_config.c +@@ -1839,7 +1839,7 @@ int merge_json_confs_into_global(struct service_arguments *args) + args->json_confs->cdi_spec_dirs_len = tmp_json_confs->cdi_spec_dirs_len; + tmp_json_confs->cdi_spec_dirs_len = 0; + #endif /* ENABLE_CDI */ +- ++ + out: + free(err); + free_isulad_daemon_configs(tmp_json_confs); +diff --git a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc +index fb5aad3c..f8df8e0a 100644 +--- a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc ++++ b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc +@@ -90,7 +90,8 @@ void RuntimeV1RuntimeServiceImpl::Init(std::string &podSandboxImage, + m_enablePodEvents = enablePodEvents; + } + +- m_rService = std::unique_ptr(new CRIRuntimeServiceImpl(podSandboxImage, cb, networkPlugin, m_enablePodEvents)); ++ m_rService = std::unique_ptr(new CRIRuntimeServiceImpl(podSandboxImage, cb, networkPlugin, ++ m_enablePodEvents)); + } + + void RuntimeV1RuntimeServiceImpl::Wait() +@@ -134,7 +135,7 @@ auto RuntimeV1RuntimeServiceImpl::GenerateCRIContainerEvent(const char *containe + m_rService->PodSandboxStatus(sandboxID, statusReply, error); + if (!error.Empty()) { + WARN("Object: CRI, Type: Failed to status pod:%s due to %s", sandboxID.c_str(), +- error.GetMessage().c_str()); ++ error.GetMessage().c_str()); + } else { + *(response->mutable_pod_sandbox_status()) = *(statusReply->mutable_status()); + for (auto &containerStatus : statusReply->containers_statuses()) { +diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +index 1097c32c..1cee68ec 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +@@ -1015,7 +1015,8 @@ cleanup: + return contStats; + } + +-std::unique_ptr ContainerManagerService::ContainerStatus(const std::string &containerID, Errors &error) ++std::unique_ptr ContainerManagerService::ContainerStatus(const std::string &containerID, ++ Errors &error) + { + return CRIHelpersV1::GetContainerStatus(m_cb, containerID, error); + } +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +index 4291d8a0..fa726e2c 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +@@ -819,7 +819,8 @@ void PodSandboxManagerService::SetSandboxStatusNetwork(std::shared_ptr> &containerStatuses, +- std::vector &errors) { ++ std::vector &errors) ++{ + auto list_response_wrapper = GetContainerListResponse(podSandboxID, errors); + if (list_response_wrapper == nullptr) { + return; +@@ -837,7 +838,8 @@ void PodSandboxManagerService::GetContainerStatuses(const std::string &podSandbo + } + } + +-std::unique_ptr PodSandboxManagerService::GetPodSandboxStatus(const std::string &podSandboxID, Errors &error) ++std::unique_ptr PodSandboxManagerService::GetPodSandboxStatus( ++ const std::string &podSandboxID, Errors &error) + { + std::unique_ptr podStatus(new (std::nothrow) runtime::v1::PodSandboxStatus); + if (podStatus == nullptr) { +@@ -876,7 +878,7 @@ void PodSandboxManagerService::PodSandboxStatus(const std::string &podSandboxID, + return; + } + +- ++ + auto podStatus = GetPodSandboxStatus(podSandboxID, error); + if (error.NotEmpty()) { + ERROR("Failed to get pod sandbox status: %s", error.GetCMessage()); +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h +index 3872c4c9..d5c0cf91 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h +@@ -129,8 +129,10 @@ private: + std::vector &podSandboxIDs, Errors &error); + void ApplySandboxLinuxOptions(const runtime::v1::LinuxPodSandboxConfig &lc, host_config *hc, + container_config *custom_config, Errors &error); +- auto GetPodSandboxStatus(const std::string &podSandboxID, Errors &error) -> std::unique_ptr; +- void GetContainerStatuses(const std::string &podSandboxID, std::vector> &containerStatuses, ++ auto GetPodSandboxStatus(const std::string &podSandboxID, ++ Errors &error) -> std::unique_ptr; ++ void GetContainerStatuses(const std::string &podSandboxID, ++ std::vector> &containerStatuses, + std::vector &errors); + + private: +diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc +index 7b40e29d..56c89c1e 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc +@@ -125,7 +125,8 @@ void CRIRuntimeServiceImpl::RemovePodSandbox(const std::string &podSandboxID, Er + m_podSandboxManager->RemovePodSandbox(podSandboxID, error); + } + +-void CRIRuntimeServiceImpl::PodSandboxStatus(const std::string &podSandboxID, runtime::v1::PodSandboxStatusResponse *reply, Errors &error) ++void CRIRuntimeServiceImpl::PodSandboxStatus(const std::string &podSandboxID, ++ runtime::v1::PodSandboxStatusResponse *reply, Errors &error) + { + m_podSandboxManager->PodSandboxStatus(podSandboxID, reply, error); + } +diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h +index 6ae59bfa..3d93c7bb 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h ++++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h +@@ -73,7 +73,8 @@ public: + + void RemovePodSandbox(const std::string &podSandboxID, Errors &error) override; + +- void PodSandboxStatus(const std::string &podSandboxID, runtime::v1::PodSandboxStatusResponse *reply, Errors &error) override; ++ void PodSandboxStatus(const std::string &podSandboxID, runtime::v1::PodSandboxStatusResponse *reply, ++ Errors &error) override; + + void ListPodSandbox(const runtime::v1::PodSandboxFilter &filter, + std::vector> &pods, Errors &error) override; +diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c +index 785b4e27..8e930c8a 100644 +--- a/src/daemon/executor/container_cb/execution_create.c ++++ b/src/daemon/executor/container_cb/execution_create.c +@@ -551,7 +551,7 @@ static int merge_config_for_syscontainer(const container_create_request *request + goto out; + } + if (request->rootfs != NULL && (append_json_map_string_string(container_spec->annotations, "external.rootfs", "true") +- || append_json_map_string_string(oci_spec->annotations, "external.rootfs", "true"))) { ++ || append_json_map_string_string(oci_spec->annotations, "external.rootfs", "true"))) { + ERROR("Realloc annotations failed"); + ret = -1; + goto out; +diff --git a/src/daemon/mailbox/mailbox.c b/src/daemon/mailbox/mailbox.c +index 732b91b9..39f003f3 100644 +--- a/src/daemon/mailbox/mailbox.c ++++ b/src/daemon/mailbox/mailbox.c +@@ -23,7 +23,8 @@ + + mailbox_topic_handler_t mailbox_topic_handlers[MAILBOX_TOPIC_MAX] = { 0 }; + +-static bool mailbox_topic_valid(mailbox_topic topic) { ++static bool mailbox_topic_valid(mailbox_topic topic) ++{ + return topic > MAILBOX_TOPIC_INVALID && topic < MAILBOX_TOPIC_MAX; + } + +diff --git a/src/daemon/mailbox/mailbox.h b/src/daemon/mailbox/mailbox.h +index 1dc2e934..aef63514 100644 +--- a/src/daemon/mailbox/mailbox.h ++++ b/src/daemon/mailbox/mailbox.h +@@ -64,7 +64,7 @@ typedef struct { + cri_container_message_type type; + } cri_container_message_t; + +-int mailbox_register_topic_handler(mailbox_topic topic, message_generator_t handle, void *context, ++int mailbox_register_topic_handler(mailbox_topic topic, message_generator_t generator, void *context, + message_release_t release, bool async); + + void mailbox_unregister_topic_handler(mailbox_topic topic); +@@ -73,7 +73,7 @@ void mailbox_publish(mailbox_topic topic, void *data); + + message_subscriber *mailbox_subscribe(mailbox_topic topic); + +-void mailbox_unsubscribe(mailbox_topic, message_subscriber *sub); ++void mailbox_unsubscribe(mailbox_topic topic, message_subscriber *sub); + + #ifdef __cplusplus + } +diff --git a/src/daemon/mailbox/mailbox_message.c b/src/daemon/mailbox/mailbox_message.c +index b16a1bdd..316b7428 100644 +--- a/src/daemon/mailbox/mailbox_message.c ++++ b/src/daemon/mailbox/mailbox_message.c +@@ -20,7 +20,8 @@ + #include "utils.h" + + // Once the create succeeds, the ownership is transferred to the mailbox_message. +-mailbox_message *mailbox_message_create(void *data, void (*destroy)(void *)) { ++mailbox_message *mailbox_message_create(void *data, void (*destroy)(void *)) ++{ + __isula_auto_free mailbox_message *msg = NULL; + msg = util_common_calloc_s(sizeof(mailbox_message)); + if (msg == NULL) { +@@ -40,7 +41,8 @@ mailbox_message *mailbox_message_create(void *data, void (*destroy)(void *)) { + return isula_transfer_ptr(msg); + } + +-int mailbox_message_ref(mailbox_message *dest) { ++int mailbox_message_ref(mailbox_message *dest) ++{ + __isula_auto_pm_unlock pthread_mutex_t *lock = NULL; + if (dest == NULL) { + ERROR("Invalid mailbox_message"); +@@ -63,7 +65,8 @@ int mailbox_message_ref(mailbox_message *dest) { + return 0; + } + +-void mailbox_message_unref(mailbox_message *dest) { ++void mailbox_message_unref(mailbox_message *dest) ++{ + __isula_auto_pm_unlock pthread_mutex_t *lock = NULL; + if (dest == NULL) { + return; +diff --git a/src/daemon/mailbox/mailbox_message.h b/src/daemon/mailbox/mailbox_message.h +index 39e40b70..af323069 100644 +--- a/src/daemon/mailbox/mailbox_message.h ++++ b/src/daemon/mailbox/mailbox_message.h +@@ -34,9 +34,9 @@ typedef struct mailbox_message { + + mailbox_message *mailbox_message_create(void *ptr, void (*destroy)(void *)); + +-int mailbox_message_ref(mailbox_message *p); ++int mailbox_message_ref(mailbox_message *dest); + +-void mailbox_message_unref(mailbox_message *p); ++void mailbox_message_unref(mailbox_message *dest); + + // define auto free function callback for mailbox_message + define_auto_cleanup_callback(mailbox_message_unref, mailbox_message); +diff --git a/src/daemon/modules/api/container_api.h b/src/daemon/modules/api/container_api.h +index 55c59980..a6ec8e22 100644 +--- a/src/daemon/modules/api/container_api.h ++++ b/src/daemon/modules/api/container_api.h +@@ -271,7 +271,8 @@ bool container_is_valid_state_string(const char *state); + + void container_update_health_monitor(const char *container_id); + +-extern int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const pid_ppid_info_t *pid_info, const container_t *cont); ++extern int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const pid_ppid_info_t *pid_info, ++ const container_t *cont); + + extern char *container_exit_fifo_create(const char *cont_state_path); + +diff --git a/src/daemon/modules/api/network_api.h b/src/daemon/modules/api/network_api.h +index 4c1e3480..fd8607b3 100644 +--- a/src/daemon/modules/api/network_api.h ++++ b/src/daemon/modules/api/network_api.h +@@ -77,7 +77,7 @@ void free_network_api_result_list(network_api_result_list *ptr); + + void free_attach_net_conf(struct attach_net_conf *ptr); + +-void free_network_api_conf(network_api_conf *ptr); ++void free_network_api_conf(network_api_conf *conf); + + bool network_module_init(const char *network_plugin, const char *cache_dir, const char *conf_dir, const char* bin_path); + +diff --git a/src/daemon/modules/container/supervisor/supervisor.c b/src/daemon/modules/container/supervisor/supervisor.c +index 83d46268..39d9fdb8 100644 +--- a/src/daemon/modules/container/supervisor/supervisor.c ++++ b/src/daemon/modules/container/supervisor/supervisor.c +@@ -307,7 +307,8 @@ static int oom_handle_cb(int fd, uint32_t events, void *cbdata, struct epoll_des + } + + /* supervisor add exit monitor */ +-int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const pid_ppid_info_t *pid_info, const container_t *cont) ++int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const pid_ppid_info_t *pid_info, ++ const container_t *cont) + { + int ret = 0; + struct supervisor_handler_data *data = NULL; +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c +index 816b9c2d..d146f7e1 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c +@@ -27,8 +27,8 @@ + #include "utils_array.h" + #include "specs_api.h" + +-/* +- * The OCI being used by the iSulad not supportes ++/* ++ * The OCI being used by the iSulad not supportes + * createRuntime/createContainer/startContainer currently. + */ + // PRESTART_HOOK is the name of the OCI "prestart" hook. +@@ -105,8 +105,8 @@ static int fill_device_node_info(cdi_device_node *d) + dev_type = NULL; + } else { + if (strcmp(d->type, dev_type) != 0) { +- ERROR("CDI device (%s, %s), host type mismatch (%s, %s)", +- d->path, d->host_path, d->type, dev_type); ++ ERROR("CDI device (%s, %s), host type mismatch (%s, %s)", ++ d->path, d->host_path, d->type, dev_type); + return -1; + } + } +@@ -168,7 +168,7 @@ static cdi_hook *clone_cdi_hook(cdi_hook *h) + hook->env_len = h->env_len; + } + hook->timeout = h->timeout; +- ++ + return hook; + + error_out: +@@ -280,7 +280,7 @@ static defs_device *cdi_device_node_to_oci(cdi_device_node *d) + oci_device->file_mode = d->file_mode; + oci_device->uid = d->uid; + oci_device->gid = d->gid; +- ++ + return oci_device; + } + +@@ -331,7 +331,7 @@ static int apply_cdi_device_nodes(cdi_container_edits *e, oci_runtime_spec *spec + access = "rwm"; + } + if (spec_add_linux_resources_device(spec, true, dev->type, +- dev->major, dev->minor, access)) { ++ dev->major, dev->minor, access)) { + dev = NULL; + goto error_out; + } +@@ -391,9 +391,9 @@ static int apply_cdi_mounts(cdi_container_edits *e, oci_runtime_spec *spec) + return -1; + } + } +- ++ + qsort(spec->mounts, spec->mounts_len, +- sizeof(defs_mount *), (int (*)(const void *, const void *))defs_mount_cmp); ++ sizeof(defs_mount *), (int (*)(const void *, const void *))defs_mount_cmp); + return 0; + } + +@@ -411,8 +411,8 @@ static int apply_cdi_hooks(cdi_container_edits *e, oci_runtime_spec *spec) + } else if (strcmp(e->hooks[i]->hook_name, POSTSTOP_HOOK)) { + ret = spec_add_poststop_hook(spec, oci_hook); + } else { +- /* +- * The OCI being used by the iSulad not supportes ++ /* ++ * The OCI being used by the iSulad not supportes + * createRuntime/createContainer/startContainer currently. + */ + ERROR("Unknown hook name %s", e->hooks[i]->hook_name); +@@ -503,19 +503,19 @@ int cdi_container_edits_validate(cdi_container_edits *e) + static int append_##item(cdi_container_edits *e, cdi_container_edits *o, clone_common_array_item_cb cb) \ + { \ + common_array e_array = { \ +- .items = (void **)e->item, \ +- .len = e->item##_len, \ +- .cap = e->item##_len, \ +- .free_item_cb = NULL, \ +- .clone_item_cb = cb \ +- }; \ ++ .items = (void **)e->item, \ ++ .len = e->item##_len, \ ++ .cap = e->item##_len, \ ++ .free_item_cb = NULL, \ ++ .clone_item_cb = cb \ ++ }; \ + common_array o_array = { \ +- .items = (void **)o->item, \ +- .len = o->item##_len, \ +- .cap = o->item##_len, \ +- .free_item_cb = NULL, \ +- .clone_item_cb = cb \ +- }; \ ++ .items = (void **)o->item, \ ++ .len = o->item##_len, \ ++ .cap = o->item##_len, \ ++ .free_item_cb = NULL, \ ++ .clone_item_cb = cb \ ++ }; \ + if (util_merge_common_array(&e_array, &o_array) != 0) { \ + ERROR("Out of memory"); \ + return -1; \ +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_device.c b/src/daemon/modules/device/cdi/behavior/cdi_device.c +index aec3d7c0..9104416c 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_device.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_device.c +@@ -28,21 +28,21 @@ void free_cdi_cache_device(struct cdi_cache_device *d) + if (d == NULL) { + return; + } +- +- /* ++ ++ /* + * free_cdi_cache_device should not be recursively free raw_device. + * Otherwise, the function conflicts with the raw_spec free raw_device +- * when cdi_cache_spec free raw_spec, triggering double free. ++ * when cdi_cache_spec free raw_spec, triggering double free. + */ + d->raw_device = NULL; +- +- /* ++ ++ /* + * free_cdi_cache_device should not be recursively free cache_spec. + * Otherwise, the function conflicts with the cache free specs, +- * triggering double free. ++ * triggering double free. + */ + d->cache_spec = NULL; +- ++ + free(d); + } + +@@ -86,7 +86,7 @@ char *cdi_device_get_qualified_name(const struct cdi_cache_device *d) + return NULL; + } + return cdi_parser_qualified_name(cdi_spec_get_vendor(d->cache_spec), +- cdi_spec_get_class(d->cache_spec), d->raw_device->name); ++ cdi_spec_get_class(d->cache_spec), d->raw_device->name); + } + + cdi_container_edits *cdi_device_get_edits(const struct cdi_cache_device *d) +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec.c b/src/daemon/modules/device/cdi/behavior/cdi_spec.c +index 8783debc..ec563230 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_spec.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec.c +@@ -33,7 +33,7 @@ void free_cdi_cache_spec(struct cdi_cache_spec *s) + if (s == NULL) { + return; + } +- ++ + free_cdi_spec(s->raw_spec); + s->raw_spec = NULL; + free(s->vendor); +@@ -44,7 +44,7 @@ void free_cdi_cache_spec(struct cdi_cache_spec *s) + s->path = NULL; + map_free(s->devices); + s->devices = NULL; +- ++ + free(s); + } + +@@ -58,7 +58,7 @@ struct cdi_cache_spec *cdi_spec_read_spec(const char *path, int priority) + ERROR("Failed to get clean path %s", path); + return NULL; + } +- ++ + raw_spec = cdi_spec_parse_file(cleanpath, NULL, &err); + if (raw_spec == NULL) { + ERROR("Failed to read CDI Spec %s: %s", cleanpath, err); +@@ -106,7 +106,7 @@ struct cdi_cache_spec *cdi_spec_new_spec(cdi_spec *raw, const char *path, int pr + ERROR("Invalid CDI Spec"); + goto error_out; + } +- ++ + return spec; + + error_out: +@@ -183,7 +183,7 @@ static int cdi_spec_init(struct cdi_cache_spec *s) + cdi_device *d = NULL; + size_t i; + bool version_result = true; +- ++ + if (!cdi_is_valid_version(s->raw_spec->cdi_version)) { + ERROR("Failed to validate cdi spec version: %s", s->raw_spec->cdi_version); + return -1; +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec.h b/src/daemon/modules/device/cdi/behavior/cdi_spec.h +index ca7b2cfa..d20826c6 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_spec.h ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec.h +@@ -37,7 +37,7 @@ struct cdi_cache_spec { + }; + + #define CDI_DEFAULT_SPEC_EXT ".json" +- ++ + void free_cdi_cache_spec(struct cdi_cache_spec *s); + + struct cdi_cache_spec *cdi_spec_read_spec(const char *path, int priority); +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c +index cafb52b8..75cd7abf 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c +@@ -27,13 +27,13 @@ + + #define DEFAULT_SPEC_DIRS_LEN 2 + static char *default_spec_dirs_items[DEFAULT_SPEC_DIRS_LEN] = {CDI_DEFAULT_STATIC_DIR, CDI_DEFAULT_DYNAMIC_DIR}; +- ++ + string_array g_default_spec_dirs = { + .items = default_spec_dirs_items, + .len = DEFAULT_SPEC_DIRS_LEN, + .cap = DEFAULT_SPEC_DIRS_LEN, + }; +- ++ + struct scan_spec_dir_cb_args { + struct cdi_scan_fn_maps *scan_fn_maps; + cdi_scan_spec_func scan_fn; +@@ -64,7 +64,7 @@ static bool scan_spec_dir_cb(const char *dir, const struct dirent *pdirent, void + DEBUG("Skip dir %s", file_path); + return true; + } +- ++ + if (!util_has_suffix(file_path, ".json")) { + DEBUG("Skip file %s", file_path); + return true; +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h +index b17a0cd8..fdb0f2f9 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h ++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h +@@ -27,7 +27,7 @@ extern "C" { + #define CDI_DEFAULT_DYNAMIC_DIR "/var/run/cdi" + + extern string_array g_default_spec_dirs; +- ++ + struct cdi_scan_fn_maps { + map_t *specs; + map_t *devices; +diff --git a/src/daemon/modules/device/cdi/behavior/cdi_version.c b/src/daemon/modules/device/cdi/behavior/cdi_version.c +index 550f3107..4cdc116a 100644 +--- a/src/daemon/modules/device/cdi/behavior/cdi_version.c ++++ b/src/daemon/modules/device/cdi/behavior/cdi_version.c +@@ -177,7 +177,7 @@ const char *cdi_minimum_required_version(cdi_spec *spec) + bool cdi_is_valid_version(const char *spec_version) + { + int i; +- ++ + for (i = 0; i < VALID_SPEC_VERSIONS_LEN; i++) { + if (strcmp(g_valid_spec_versions[i].version, spec_version) == 0) { + return true; +diff --git a/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c +index 8824d29c..603ce28d 100644 +--- a/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c ++++ b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c +@@ -47,7 +47,7 @@ char *cdi_parser_qualified_name(const char *vendor, const char *class, const cha + } + + nret = snprintf(device_name, sizeof(device_name), "%s/%s=%s", +- vendor, class, name); ++ vendor, class, name); + if (nret < 0 || (size_t)nret >= sizeof(device_name)) { + ERROR("Device name is too long"); + return NULL; +@@ -118,7 +118,7 @@ int cdi_parser_parse_device(const char *device, char **vendor, char **class, cha + { + __isula_auto_array_t char **parts = NULL; + +- if (vendor == NULL || class == NULL || name == NULL || ++ if (vendor == NULL || class == NULL || name == NULL || + device == NULL || device[0] == '/') { + ERROR("Invalid argument"); + return -1; +@@ -214,7 +214,7 @@ int cdi_parser_validate_class_name(const char *class) + int cdi_parser_validate_device_name(const char *name) + { + size_t i; +- ++ + if (name == NULL) { + ERROR("Invalid (empty) device name"); + return -1; +diff --git a/src/daemon/modules/device/cdi/cdi_cache.c b/src/daemon/modules/device/cdi/cdi_cache.c +index e637f7cd..e9a9b804 100644 +--- a/src/daemon/modules/device/cdi/cdi_cache.c ++++ b/src/daemon/modules/device/cdi/cdi_cache.c +@@ -179,7 +179,7 @@ static int cdi_refresh(struct cdi_cache *c) + { + bool refreshed; + int ret = 0; +- ++ + if (c == NULL) { + ERROR("Invalid arguments"); + return -1; +@@ -206,10 +206,10 @@ static void map_cdi_cache_specs_kvfree(void *key, void *value) + static void map_cdi_cache_device_kvfree(void *key, void *value) + { + free(key); +- /* ++ /* + * map_cdi_cache_device_kvfree should not be recursively free cdi_cache_device. + * Otherwise, the function conflicts with the cdi_cache_specs free devices, +- * triggering double free. ++ * triggering double free. + */ + (void)value; + } +@@ -249,8 +249,8 @@ static bool resolve_conflict(struct cdi_scan_fn_maps *scan_fn_maps, const char * + return true; + } + +-static void refresh_scan_spec_func(struct cdi_scan_fn_maps *scan_fn_maps, const char *path, +- int priority, struct cdi_cache_spec *spec) ++static void refresh_scan_spec_func(struct cdi_scan_fn_maps *scan_fn_maps, const char *path, ++ int priority, struct cdi_cache_spec *spec) + { + map_t *specs = scan_fn_maps->specs; + map_t *devices = scan_fn_maps->devices; +@@ -375,7 +375,7 @@ static int refresh(struct cdi_cache *c) + util_swap_ptr((void **)&c->devices, (void **)&devices); + + ret = c->refresh_error_flag ? -1 : 0; +- ++ + free_out: + map_itor_free(itor); + map_free(specs); +@@ -443,7 +443,7 @@ static int cdi_inject_devices(struct cdi_cache *c, oci_runtime_spec *oci_spec, s + + (void)refresh_if_required(c, false, &ret); + +- for(i = 0; i < devices->len; i++) { ++ for (i = 0; i < devices->len; i++) { + device = devices->items[i]; + d = map_search(c->devices, (void *)device); + if (d == NULL) { +@@ -524,7 +524,7 @@ static int init_tracked(struct cdi_watch *w, string_array *dirs) + ERROR("Out of memory"); + return -1; + } +- for(i = 0; i < dirs->len; i++) { ++ for (i = 0; i < dirs->len; i++) { + if (!map_replace(w->tracked, (void *)dirs->items[i], (void *)&tmp_value)) { + ERROR("Failed to insert tracked by dir %s", dirs->items[i]); + goto error_out; +@@ -624,7 +624,7 @@ static int process_cdi_events(int watcher_fd, struct cdi_cache *c) + } + + (void)pthread_mutex_lock(&c->mutex); +- ++ + while (events_index < events_length) { + cdi_event = (struct inotify_event *)(&buffer[events_index]); + ssize_t event_size = (ssize_t)(cdi_event->len) + (ssize_t)offsetof(struct inotify_event, name); +@@ -633,8 +633,8 @@ static int process_cdi_events(int watcher_fd, struct cdi_cache *c) + } + events_index += event_size; + +- /* +- * file: ++ /* ++ * file: + * Rename: mask == IN_MOVED_TO | IN_MOVED_FROM + * Remove: mask == IN_MOVED_FROM || mask == IN_DELETE + * Write: mask == IN_MODIFY +diff --git a/src/daemon/modules/device/cdi/cdi_cache.h b/src/daemon/modules/device/cdi/cdi_cache.h +index 638e954e..9f6793c8 100644 +--- a/src/daemon/modules/device/cdi/cdi_cache.h ++++ b/src/daemon/modules/device/cdi/cdi_cache.h +@@ -34,7 +34,7 @@ struct cdi_cache_ops { + // injecting CDI devices into an OCI Spec. + // Resolver + int (*inject_devices)(struct cdi_cache *c, oci_runtime_spec *spec, string_array *devices); +- ++ + // refreshing the cache of CDI Specs and devices. + // Refresher + int (*configure)(struct cdi_cache *c, string_array *spec_dirs); +@@ -55,12 +55,12 @@ struct cdi_cache { + // This map holding the reference to cdi device, the devices will not released when the map is freed. + map_t *devices; // MAP_STR_PTR devices[cdi_device.name] = cdi_cache_device* + bool refresh_error_flag; +- bool auto_refresh; ++ bool auto_refresh; + struct cdi_watch *watch; + }; + + void free_cdi_cache(struct cdi_cache *c); +- ++ + struct cdi_cache *cdi_new_cache(string_array *spec_dirs); + struct cdi_cache_ops *cdi_get_cache_ops(void); + +diff --git a/src/daemon/modules/device/cdi_operate.c b/src/daemon/modules/device/cdi_operate.c +index f99bb7e4..2cc2a17d 100644 +--- a/src/daemon/modules/device/cdi_operate.c ++++ b/src/daemon/modules/device/cdi_operate.c +@@ -29,7 +29,7 @@ int cdi_operate_registry_init(char **specs_dirs, size_t specs_dirs_len) + .len = specs_dirs_len, + .cap = specs_dirs_len, + }; +- ++ + return cdi_registry_init(&spec_dirs_array); + } + +@@ -40,7 +40,7 @@ int cdi_operate_refresh(void) + ERROR("Failed to get registry"); + return -1; + } +- ++ + return registry->ops->refresh(registry->cdi_cache); + } + +@@ -52,13 +52,13 @@ int cdi_operate_inject_devices(oci_runtime_spec *spec, string_array *devices) + ERROR("Invalid params"); + return -1; + } +- ++ + registry = cdi_get_registry(); + if (registry == NULL || registry->ops == NULL || registry->ops->inject_devices == NULL) { + ERROR("Failed to get registry"); + return -1; + } +- ++ + return registry->ops->inject_devices(registry->cdi_cache, spec, devices); + } + +diff --git a/src/daemon/modules/image/oci/oci_image.c b/src/daemon/modules/image/oci/oci_image.c +index ce1c8a6b..e9f16024 100644 +--- a/src/daemon/modules/image/oci/oci_image.c ++++ b/src/daemon/modules/image/oci/oci_image.c +@@ -295,7 +295,8 @@ static bool remove_image_tmpdir_cb(const char *path_name, const struct dirent *s + return true; + } + +- if (!util_has_prefix(sub_dir->d_name, LOAD_TMPDIR_PREFIX) && !util_has_prefix(sub_dir->d_name, REGISTRY_TMPDIR_PREFIX)) { ++ if (!util_has_prefix(sub_dir->d_name, LOAD_TMPDIR_PREFIX) && ++ !util_has_prefix(sub_dir->d_name, REGISTRY_TMPDIR_PREFIX)) { + // only remove directory that image module created + return true; + } +diff --git a/src/daemon/modules/image/oci/progress.h b/src/daemon/modules/image/oci/progress.h +index dcc8e144..673019a4 100644 +--- a/src/daemon/modules/image/oci/progress.h ++++ b/src/daemon/modules/image/oci/progress.h +@@ -35,7 +35,7 @@ typedef struct progress { + + bool progress_status_map_udpate(progress_status_map *progress_status_map, char *key, int64_t current, int64_t total); + +-progress_status_map *progress_status_map_new(); ++progress_status_map *progress_status_map_new(void); + + size_t progress_status_map_size(progress_status_map *progress_status_map); + +diff --git a/src/daemon/modules/image/oci/registry/registry.c b/src/daemon/modules/image/oci/registry/registry.c +index 66fa0076..2e99255a 100644 +--- a/src/daemon/modules/image/oci/registry/registry.c ++++ b/src/daemon/modules/image/oci/registry/registry.c +@@ -293,7 +293,8 @@ static bool is_manifest_schemav1(char *media_type) + return false; + } + +- if (strcmp(media_type, DOCKER_MANIFEST_SCHEMA1_JSON) == 0 || strcmp(media_type, DOCKER_MANIFEST_SCHEMA1_PRETTYJWS) == 0 || ++ if (strcmp(media_type, DOCKER_MANIFEST_SCHEMA1_JSON) == 0 || ++ strcmp(media_type, DOCKER_MANIFEST_SCHEMA1_PRETTYJWS) == 0 || + strcmp(media_type, MEDIA_TYPE_APPLICATION_JSON) == 0) { + return true; + } +diff --git a/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.h b/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.h +index 48d8d8b6..10c14f05 100644 +--- a/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.h ++++ b/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.h +@@ -36,7 +36,7 @@ struct cni_args { + void free_cni_args(struct cni_args *cargs); + + int exec_plugin_with_result(const char *plugin_path, const char *cni_net_conf_json, const struct cni_args *cniargs, +- struct cni_opt_result **ret); ++ struct cni_opt_result **result); + + int exec_plugin_without_result(const char *plugin_path, const char *cni_net_conf_json, const struct cni_args *cniargs); + +diff --git a/src/daemon/modules/network/cri/adaptor_cri.c b/src/daemon/modules/network/cri/adaptor_cri.c +index 2d03dd98..55826d52 100644 +--- a/src/daemon/modules/network/cri/adaptor_cri.c ++++ b/src/daemon/modules/network/cri/adaptor_cri.c +@@ -71,7 +71,7 @@ int adaptor_cni_update_confs() + size_t tmp_net_list_len = 0; + size_t i; + char message[MAX_BUFFER_SIZE] = { 0 }; +- int pos = 0; ++ size_t pos = 0; + + work = map_new(MAP_STR_INT, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); + if (work == NULL) { +diff --git a/src/daemon/modules/network/cri/adaptor_cri.h b/src/daemon/modules/network/cri/adaptor_cri.h +index 3d4fe82d..bc157626 100644 +--- a/src/daemon/modules/network/cri/adaptor_cri.h ++++ b/src/daemon/modules/network/cri/adaptor_cri.h +@@ -23,9 +23,9 @@ extern "C" { + + int adaptor_cni_init_confs(const char *conf_dir, const char **bin_paths, const size_t bin_paths_len); + +-int adaptor_cni_update_confs(); ++int adaptor_cni_update_confs(void); + +-bool adaptor_cni_check_inited(); ++bool adaptor_cni_check_inited(void); + + int adaptor_cni_setup(const network_api_conf *conf, network_api_result_list *result); + +diff --git a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c +index a89d0375..64a8adbc 100644 +--- a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c ++++ b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c +@@ -166,7 +166,8 @@ int rt_lcr_start(const char *name, const char *runtime, const rt_start_params_t + } + isulad_set_error_message("Start container error: %s", + (tmpmsg != NULL && strcmp(tmpmsg, DEF_SUCCESS_STR) != 0) ? tmpmsg : DEF_ERR_RUNTIME_STR); +- ERROR("Start container error: %s", (tmpmsg != NULL && strcmp(tmpmsg, DEF_SUCCESS_STR) != 0) ? tmpmsg : DEF_ERR_RUNTIME_STR); ++ ERROR("Start container error: %s", (tmpmsg != NULL && ++ strcmp(tmpmsg, DEF_SUCCESS_STR) != 0) ? tmpmsg : DEF_ERR_RUNTIME_STR); + ret = -1; + goto out; + } +@@ -264,7 +265,8 @@ int rt_lcr_rm(const char *name, const char *runtime, const rt_rm_params_t *param + if (engine_ops == NULL || engine_ops->engine_delete_op == NULL) { + // if engine_ops is NULL, container root path may have been corrupted, try to remove by daemon + // If user runs container with lcr but remove lcr runtime after, there might be resources remaining +- ERROR("Failed to get engine delete operations, container %s root path may have been corrupted, try to remove by daemon", name); ++ ERROR("Failed to get engine delete operations, container %s root path may have been corrupted, try to remove by daemon", ++ name); + if (remove_container_rootpath(name, params->rootpath) == 0) { + ret = 0; + goto out; +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 854752ea..62cff3cf 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -623,7 +623,8 @@ out: + return ret; + } + +-static void transform_stats_info_from_runtime(shim_client_runtime_stats *stats, struct runtime_container_resources_stats_info *info) ++static void transform_stats_info_from_runtime(shim_client_runtime_stats *stats, ++ struct runtime_container_resources_stats_info *info) + { + size_t i; + if (stats == NULL || stats->data == NULL) { +@@ -887,7 +888,7 @@ static int shim_create(shim_create_args *args) + + if (get_engine_routine_log_info(&engine_log_path, &log_level) != 0) { + ERROR("failed to get engine log path"); +- return -1; ++ return -1; + } + + nret = snprintf(fpid, sizeof(fpid), "%s/shim-pid", args->workdir); +@@ -938,7 +939,7 @@ static int shim_create(shim_create_args *args) + //prevent the child process from having the same standard streams as the parent process + if (isula_null_stdfds() != 0) { + (void)dprintf(exec_err_pipe[1], "failed to set std console to /dev/null"); +- exit(EXIT_FAILURE); ++ exit(EXIT_FAILURE); + } + + if (args->fg) { +@@ -1054,7 +1055,7 @@ out: + close(shim_stdout_pipe[0]); + if (ret != 0) { + show_shim_errlog(shim_stderr_pipe[0]); +- // Since users are more concerned about runtime error information, ++ // Since users are more concerned about runtime error information, + // the runtime log will overwrite the shim log if it exists. + show_runtime_errlog(args->workdir); + if (args->timeout != NULL) { +diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c +index fe8ee72f..0b95cdad 100644 +--- a/src/daemon/modules/service/service_container.c ++++ b/src/daemon/modules/service/service_container.c +@@ -691,7 +691,8 @@ out: + epoll_loop_close(&descr); + } + +-static int do_oci_spec_update(const char *id, oci_runtime_spec *oci_spec, container_config *container_spec, host_config *hostconfig) ++static int do_oci_spec_update(const char *id, oci_runtime_spec *oci_spec, container_config *container_spec, ++ host_config *hostconfig) + { + int ret; + +@@ -2013,7 +2014,7 @@ static defs_process *make_exec_process_spec(const container_config *container_sp + #ifdef ENABLE_CDI + // extend step: merge env from oci_spec which comes from injected devices + ret = defs_process_add_multiple_env(spec, (const char **)oci_spec->process->env, +- oci_spec->process->env_len); ++ oci_spec->process->env_len); + if (ret != 0) { + ERROR("Failed to dup oci env for exec process spec"); + goto err_out; +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index e779c22e..122f9992 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -2483,7 +2483,8 @@ out: + return ret; + } + +-int update_oci_ulimit(oci_runtime_spec *oci_spec, const host_config *hostconfig) { ++int update_oci_ulimit(oci_runtime_spec *oci_spec, const host_config *hostconfig) ++{ + if (oci_spec == NULL || hostconfig == NULL) { + ERROR("Invalid arguments"); + return -1; +@@ -2660,7 +2661,7 @@ int defs_process_add_multiple_env(defs_process *dp, const char **envs, size_t en + int spec_add_multiple_process_env(oci_runtime_spec *oci_spec, const char **envs, size_t env_len) + { + int ret = 0; +- ++ + if (envs == NULL || env_len == 0) { + DEBUG("empty envs"); + return 0; +@@ -2669,26 +2670,26 @@ int spec_add_multiple_process_env(oci_runtime_spec *oci_spec, const char **envs, + ERROR("Invalid params"); + return -1; + } +- ++ + ret = make_sure_oci_spec_process(oci_spec); + if (ret < 0) { + ERROR("Out of memory"); + return -1; + } +- ++ + ret = defs_process_add_multiple_env(oci_spec->process, envs, env_len); + if (ret < 0) { + ERROR("Failed to add envs"); + } +- ++ + return ret; + } +- ++ + int spec_add_device(oci_runtime_spec *oci_spec, defs_device *device) + { + int ret = 0; + size_t i; +- ++ + if (device == NULL) { + return -1; + } +@@ -2696,7 +2697,7 @@ int spec_add_device(oci_runtime_spec *oci_spec, defs_device *device) + if (ret < 0) { + return -1; + } +- ++ + for (i = 0; i < oci_spec->linux->devices_len; i++) { + if (strcmp(oci_spec->linux->devices[i]->path, device->path) == 0) { + free_defs_device(oci_spec->linux->devices[i]); +@@ -2712,21 +2713,21 @@ int spec_add_device(oci_runtime_spec *oci_spec, defs_device *device) + } + oci_spec->linux->devices[oci_spec->linux->devices_len] = device; + oci_spec->linux->devices_len++; +- ++ + return 0; + } +- ++ + int spec_add_linux_resources_device(oci_runtime_spec *oci_spec, bool allow, const char *dev_type, + int64_t major, int64_t minor, const char *access) + { + int ret = 0; + defs_device_cgroup *device = NULL; +- ++ + ret = make_sure_oci_spec_linux_resources(oci_spec); + if (ret < 0) { + return -1; + } +- ++ + device = util_common_calloc_s(sizeof(*device)); + if (device == NULL) { + ERROR("Out of memory"); +@@ -2738,7 +2739,8 @@ int spec_add_linux_resources_device(oci_runtime_spec *oci_spec, bool allow, cons + device->major = major; + device->minor = minor; + +- if (util_mem_realloc((void **)&oci_spec->linux->resources->devices, (oci_spec->linux->resources->devices_len + 1) * sizeof(char *), ++ if (util_mem_realloc((void **)&oci_spec->linux->resources->devices, ++ (oci_spec->linux->resources->devices_len + 1) * sizeof(char *), + (void *)oci_spec->linux->resources->devices, oci_spec->linux->resources->devices_len * sizeof(char *)) != 0) { + ERROR("Out of memory"); + free_defs_device_cgroup(device); +@@ -2746,35 +2748,35 @@ int spec_add_linux_resources_device(oci_runtime_spec *oci_spec, bool allow, cons + } + oci_spec->linux->resources->devices[oci_spec->linux->resources->devices_len] = device; + oci_spec->linux->resources->devices_len++; +- ++ + return 0; + } +- ++ + void spec_remove_mount(oci_runtime_spec *oci_spec, const char *dest) + { + size_t i; +- ++ + if (oci_spec == NULL || oci_spec->mounts == NULL || dest == NULL) { + return; + } +- ++ + for (i = 0; i < oci_spec->mounts_len; i++) { + if (strcmp(oci_spec->mounts[i]->destination, dest) == 0) { + free_defs_mount(oci_spec->mounts[i]); + (void)memcpy((void **)&oci_spec->mounts[i], (void **)&oci_spec->mounts[i + 1], +- (oci_spec->mounts_len - i - 1) * sizeof(void *)); ++ (oci_spec->mounts_len - i - 1) * sizeof(void *)); + oci_spec->mounts_len--; + return; + } + } + } +- ++ + int spec_add_mount(oci_runtime_spec *oci_spec, defs_mount *mnt) + { + if (oci_spec == NULL || mnt == NULL) { + return -1; + } +- ++ + if (util_mem_realloc((void **)&oci_spec->mounts, (oci_spec->mounts_len + 1) * sizeof(char *), + (void *)oci_spec->mounts, oci_spec->mounts_len * sizeof(char *)) != 0) { + ERROR("Out of memory"); +@@ -2782,10 +2784,10 @@ int spec_add_mount(oci_runtime_spec *oci_spec, defs_mount *mnt) + } + oci_spec->mounts[oci_spec->mounts_len] = mnt; + oci_spec->mounts_len++; +- ++ + return 0; + } +- ++ + #define SPEC_ADD_HOOKS_ITEM_DEF(hooktype) \ + int spec_add_##hooktype##_hook(oci_runtime_spec *oci_spec, defs_hook *hooktype##_hook) \ + { \ +@@ -2806,9 +2808,9 @@ int spec_add_mount(oci_runtime_spec *oci_spec, defs_mount *mnt) + oci_spec->hooks->hooktype##_len++; \ + return 0; \ + } +- +-/* +-* The OCI being used by the iSulad not supportes ++ ++/* ++* The OCI being used by the iSulad not supportes + * createRuntime/createContainer/startContainer currently. + */ + SPEC_ADD_HOOKS_ITEM_DEF(prestart) +diff --git a/src/daemon/modules/spec/specs_mount.c b/src/daemon/modules/spec/specs_mount.c +index 12bd261b..2e065e3d 100644 +--- a/src/daemon/modules/spec/specs_mount.c ++++ b/src/daemon/modules/spec/specs_mount.c +@@ -2871,7 +2871,7 @@ static inline int set_host_ipc_shm_path(container_config_v2_common_config *v2_sp + * 1. The user defined /dev/shm in mounts, which takes the first priority + * 2. If sharable is set in ipc mode (or by default ipc_mode is null), the container provides shm path, + * in the case of sandbox API is used, the sandbox module has already provided shm path +- * 3. Use the connected container's shm path if ipc_mode is set to container:, ++ * 3. Use the connected container's shm path if ipc_mode is set to container:, + * if connected containerd is a sandbox, use the sandbox's shm path + * 4. Use /dev/shm if ipc_mode is set to host + */ +@@ -3613,7 +3613,7 @@ int inject_CDI_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hos + int ret = 0; + string_array devices_array = { 0 }; + __isula_auto_free char *error = NULL; +- ++ + if (oci_spec == NULL || hostconfig == NULL) { + ERROR("Invalid params"); + return -1; +diff --git a/src/utils/cutils/blocking_queue.c b/src/utils/cutils/blocking_queue.c +index 9bdb2ca3..02059690 100644 +--- a/src/utils/cutils/blocking_queue.c ++++ b/src/utils/cutils/blocking_queue.c +@@ -97,7 +97,8 @@ int blocking_queue_push(blocking_queue *queue, void *data) + return 0; + } + +-int blocking_queue_pop(blocking_queue *queue, void **data) { ++int blocking_queue_pop(blocking_queue *queue, void **data) ++{ + if (queue == NULL || data == NULL) { + ERROR("Invalid NULL arguments"); + return -1; +diff --git a/src/utils/cutils/network_namespace.h b/src/utils/cutils/network_namespace.h +index 6ac7b28b..14410736 100644 +--- a/src/utils/cutils/network_namespace.h ++++ b/src/utils/cutils/network_namespace.h +@@ -22,9 +22,9 @@ + extern "C" { + #endif + +-int prepare_network_namespace(const char *netns_path, const bool post_prepare_network, const int pid); ++int prepare_network_namespace(const char *netns_path, const bool post_setup_network, const int pid); + +-int remove_network_namespace(const char *netns); ++int remove_network_namespace(const char *netns_path); + + int create_network_namespace_file(const char *netns_path); + +diff --git a/src/utils/cutils/utils_array.c b/src/utils/cutils/utils_array.c +index 72294005..6c7444f0 100644 +--- a/src/utils/cutils/utils_array.c ++++ b/src/utils/cutils/utils_array.c +@@ -90,17 +90,17 @@ char **util_copy_array_by_len(char **array, size_t len) + { + char **new_array = NULL; + size_t i; +- ++ + if (array == NULL || len == 0) { + return NULL; + } +- ++ + new_array = util_smart_calloc_s(sizeof(char *), len); + if (new_array == NULL) { + ERROR("Out of memory"); + return NULL; + } +- ++ + for (i = 0; i < len; i++) { + new_array[i] = util_strdup_s(array[i]); + } +@@ -262,12 +262,12 @@ string_array *util_copy_string_array(string_array *sarr) + { + string_array *ptr = NULL; + size_t i; +- ++ + if (sarr == NULL) { + ERROR("Invalid string array"); + return NULL; + } +- ++ + ptr = util_string_array_new(sarr->cap); + if (ptr == NULL) { + ERROR("Out of memory"); +@@ -277,7 +277,7 @@ string_array *util_copy_string_array(string_array *sarr) + ptr->items[i] = util_strdup_s(sarr->items[i]); + ptr->len += 1; + } +- ++ + return ptr; + } + +diff --git a/src/utils/cutils/utils_port.h b/src/utils/cutils/utils_port.h +index dbbf2a5a..ae1676f2 100644 +--- a/src/utils/cutils/utils_port.h ++++ b/src/utils/cutils/utils_port.h +@@ -66,7 +66,7 @@ void util_free_network_port(struct network_port *ptr); + + bool util_valid_proto(const char *proto); + +-int util_get_random_port(); ++int util_get_random_port(void); + + static inline bool is_valid_port(const int port) + { +diff --git a/src/utils/cutils/utils_verify.c b/src/utils/cutils/utils_verify.c +index 6f1da12c..0d7c17f4 100644 +--- a/src/utils/cutils/utils_verify.c ++++ b/src/utils/cutils/utils_verify.c +@@ -540,8 +540,8 @@ bool util_valid_propagation_mode(const char *mode) + if (mode == NULL) { + return false; + } +- return strcmp(mode, "private") == 0 || strcmp(mode, "rprivate") == 0 || strcmp(mode, "slave") == 0 || strcmp(mode, "rslave") == 0 || +- strcmp(mode, "shared") == 0 || strcmp(mode, "rshared") == 0; ++ return strcmp(mode, "private") == 0 || strcmp(mode, "rprivate") == 0 || strcmp(mode, "slave") == 0 || ++ strcmp(mode, "rslave") == 0 || strcmp(mode, "shared") == 0 || strcmp(mode, "rshared") == 0; + } + + bool util_valid_mount_mode(const char *mode) +diff --git a/src/utils/progress/show.h b/src/utils/progress/show.h +index c1f71d86..1942db8e 100644 +--- a/src/utils/progress/show.h ++++ b/src/utils/progress/show.h +@@ -21,11 +21,11 @@ extern "C" { + #endif + + void move_to_row(int row); +-void move_cursor_up(int lines); ++void move_cursor_up(int rows); + void clear_row(int row); +-void clear_lines_below(); +-int get_current_row(); +-int get_terminal_width(); ++void clear_lines_below(void); ++int get_current_row(void); ++int get_terminal_width(void); + + #ifdef __cplusplus + } +diff --git a/src/utils/tar/isulad_tar.c b/src/utils/tar/isulad_tar.c +index 13343922..29008420 100644 +--- a/src/utils/tar/isulad_tar.c ++++ b/src/utils/tar/isulad_tar.c +@@ -445,7 +445,7 @@ static int tar_resource_rebase(const char *path, const char *rebase, const char + ERROR("Can not split path: %s", path); + goto cleanup; + } +- ++ + if (realpath(srcdir, cleanpath) == NULL) { + ERROR("Failed to get real path for %s", srcdir); + goto cleanup; +diff --git a/test/cgroup/cpu/cgroup_cpu_ut.cc b/test/cgroup/cpu/cgroup_cpu_ut.cc +index 6e6e04f4..16eec69b 100644 +--- a/test/cgroup/cpu/cgroup_cpu_ut.cc ++++ b/test/cgroup/cpu/cgroup_cpu_ut.cc +@@ -80,7 +80,7 @@ TEST(CgroupCpuUnitTest, test_sysinfo_cgroup_controller_cpurt_mnt_path) + { + MOCK_SET(util_common_calloc_s, nullptr); + ASSERT_EQ(get_sys_info(true), nullptr); +- ++ + int ret = cgroup_ops_init(); + ASSERT_EQ(ret, 0); + +-- +2.25.1 + diff --git a/0088-testcase-close-cdi-testcase.patch b/0088-testcase-close-cdi-testcase.patch new file mode 100644 index 0000000..702342f --- /dev/null +++ b/0088-testcase-close-cdi-testcase.patch @@ -0,0 +1,29 @@ +From eba353bef72bf62cd47f1b03a9fbd4c621ad479e Mon Sep 17 00:00:00 2001 +From: liuxu +Date: Thu, 16 May 2024 18:00:01 +0800 +Subject: [PATCH 088/108] testcase:close cdi testcase + +Signed-off-by: liuxu +--- + CI/test_cases/container_cases/cdi_test.sh | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/CI/test_cases/container_cases/cdi_test.sh b/CI/test_cases/container_cases/cdi_test.sh +index dd7b1177..f9fd4567 100755 +--- a/CI/test_cases/container_cases/cdi_test.sh ++++ b/CI/test_cases/container_cases/cdi_test.sh +@@ -198,8 +198,8 @@ EOF + + declare -i ans=0 + +-do_pre || ((ans++)) +-do_test_full_cdi || ((ans++)) +-do_post ++# do_pre || ((ans++)) ++# do_test_full_cdi || ((ans++)) ++# do_post + + show_result ${ans} "${curr_path}/${0}" +-- +2.25.1 + diff --git a/0089-docs-update-cni-doc.patch b/0089-docs-update-cni-doc.patch new file mode 100644 index 0000000..32d1fad --- /dev/null +++ b/0089-docs-update-cni-doc.patch @@ -0,0 +1,68 @@ +From 7fc8578097b9f8254962dc4fb277492b3251e5cb Mon Sep 17 00:00:00 2001 +From: liuxu +Date: Thu, 16 May 2024 17:56:08 +0800 +Subject: [PATCH 089/108] docs:update cni doc + +Signed-off-by: liuxu +--- + README.md | 2 +- + README_zh.md | 2 +- + docs/design/README.md | 2 ++ + docs/design/README_zh.md | 2 ++ + 4 files changed, 6 insertions(+), 2 deletions(-) + +diff --git a/README.md b/README.md +index 694ddbc2..7ddc62ab 100644 +--- a/README.md ++++ b/README.md +@@ -228,7 +228,7 @@ Using [ptcr](https://gitee.com/openeuler/ptcr) as a performance test tool , it s + The standard specification versions that `iSulad` is compatible with are as follows: + + - Compatible with OCI 1.0.0. +-- Compatible with CNI 0.3.0 and above. ++- Compatible with CNI 0.3.0 - 1.0.0, iSulad supports CNI 1.0.0 from 2.1.4 version. + - Compatible with lcr 2.1.x and above. + + ## Kubernetes Support +diff --git a/README_zh.md b/README_zh.md +index 5db28f3a..45ac66ac 100755 +--- a/README_zh.md ++++ b/README_zh.md +@@ -224,7 +224,7 @@ $ sudo isula rm test + `iSulad` 能够兼容的标准规范版本如下： + + - 兼容 1.0.0 版本的OCI +-- 兼容 0.3.0 版本以上的CNI ++- 兼容 0.3.0-1.0.0 版本的CNI，iSulad从2.1.4版本后支持 CNI 1.0.0版本 + - 兼容 2.1.x 版本以上的lcr + + ## Kubernetes Support +diff --git a/docs/design/README.md b/docs/design/README.md +index d2a3702d..c171cb20 100644 +--- a/docs/design/README.md ++++ b/docs/design/README.md +@@ -43,6 +43,8 @@ This section contains some design documents for users who want to learn more abo + + - You can see how the cni operator modules are designed in [cni_operator_design](./detailed/Network/cni_operator_design.md). + ++- You can see how the cni operator modules update to CNI v1.0.0 in [cni_1.0.0_change](./detailed/Network/cni_1.0.0_change.md)。 ++ + - You can see how the CRI adapter modules are designed in [CRI_adapter_design](./detailed/Network/CRI_adapter_design.md). + + - You can see how the native network adapter modules are designed in [native_network_adapter_design](./detailed/Network/native_network_adapter_design.md). +diff --git a/docs/design/README_zh.md b/docs/design/README_zh.md +index c6172b6f..0f4cf13e 100644 +--- a/docs/design/README_zh.md ++++ b/docs/design/README_zh.md +@@ -49,6 +49,8 @@ + + - 查看 cni operator 模块的设计文档： [cni_operator_design](./detailed/Network/cni_operator_design_zh.md) 。 + ++- 查看 cni operator 模块升级到CNI v1.0.0的设计文档： [cni_1.0.0_change](./detailed/Network/cni_1.0.0_change.md) 。 ++ + - 查看 CRI adapter 模块的设计文档： [CRI_adapter_design](./detailed/Network/CRI_adapter_design_zh.md) 。 + + - 查看 native network adapter 模块的设计文档： [native_network_adapter_design](./detailed/Network/native_network_adapter_design_zh.md) 。 +-- +2.25.1 + diff --git a/0090-modify-the-user-error-log-to-be-the-same-as-before.patch b/0090-modify-the-user-error-log-to-be-the-same-as-before.patch new file mode 100644 index 0000000..582f7e4 --- /dev/null +++ b/0090-modify-the-user-error-log-to-be-the-same-as-before.patch @@ -0,0 +1,100 @@ +From 3b0f34c7cd55686cf18f65efbdc0be8a84f13e3e Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Mon, 20 May 2024 17:54:04 +1400 +Subject: [PATCH 090/108] modify the user error log to be the same as before + +Signed-off-by: zhongtao +--- + src/daemon/common/id_name_manager.c | 4 +-- + .../executor/container_cb/execution_create.c | 26 +++++++++++-------- + .../container_cb/execution_information.c | 4 ++- + 3 files changed, 20 insertions(+), 14 deletions(-) + +diff --git a/src/daemon/common/id_name_manager.c b/src/daemon/common/id_name_manager.c +index 263a584d..f64094b9 100644 +--- a/src/daemon/common/id_name_manager.c ++++ b/src/daemon/common/id_name_manager.c +@@ -242,7 +242,7 @@ static bool try_add_name(const char *name) + } + + if (!util_valid_container_name(name)) { +- ERROR("Failed to add invalid name: %s", name); ++ ERROR("Invalid container name (%s), only [a-zA-Z0-9][a-zA-Z0-9_.-]+$ are allowed.", name); + return false; + } + +@@ -262,7 +262,7 @@ static bool try_remove_name(const char *name) + } + + if (!util_valid_container_name(name)) { +- ERROR("Failed to remove invalid name: %s", name); ++ ERROR("Invalid container name (%s), only [a-zA-Z0-9][a-zA-Z0-9_.-]+$ are allowed.", name); + return false; + } + +diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c +index 785b4e27..041089dd 100644 +--- a/src/daemon/executor/container_cb/execution_create.c ++++ b/src/daemon/executor/container_cb/execution_create.c +@@ -761,8 +761,17 @@ static int maintain_container_id(const container_create_request *request, char * + #endif + + if (!nret) { +- ERROR("Failed to add entry to id name manager with new id and name"); +- isulad_set_error_message("Failed to add entry to id name manager with new id and name"); ++ __isula_auto_free char *used_id = NULL; ++ used_id = container_name_index_get(name); ++ if(used_id != NULL) { ++ ERROR("Name %s is in use by container %s", name, used_id); ++ isulad_set_error_message("Conflict. The name \"%s\" is already in use by container %s. " ++ "You have to remove (or rename) that container to be able to reuse that name.", ++ name, used_id); ++ } else { ++ ERROR("Failed to add entry to id name manager with new id and name"); ++ isulad_set_error_message("Failed to add entry to id name manager with new id and name"); ++ } + ret = -1; + goto out; + } +@@ -775,19 +784,14 @@ static int maintain_container_id(const container_create_request *request, char * + goto out; + } + +- char *used_id = NULL; +- used_id = container_name_index_get(name); +- ERROR("Name %s is in use by container %s", name, used_id); +- isulad_set_error_message("Conflict. The name \"%s\" is already in use by container %s. " +- "You have to remove (or rename) that container to be able to reuse that name.", +- name, used_id); +- free(used_id); +- used_id = NULL; +- ret = -1; + if (!skip_id_name_manage && !id_name_manager_remove_entry(id, name)) { + WARN("Failed to remove %s and %s from id name manager", id, name); + } + ++ ERROR("Failed to add %s to container name index", name); ++ isulad_set_error_message("Failed to add %s to container name index", name); ++ ret = -1; ++ + out: + *out_id = id; + *out_name = name; +diff --git a/src/daemon/executor/container_cb/execution_information.c b/src/daemon/executor/container_cb/execution_information.c +index c02cc830..58924257 100644 +--- a/src/daemon/executor/container_cb/execution_information.c ++++ b/src/daemon/executor/container_cb/execution_information.c +@@ -1149,7 +1149,9 @@ static int container_rename(container_t *cont, const char *new_name) + + if (!id_name_manager_rename(new_name, old_name)) { + ERROR("Failed to rename %s to %s in id-name manager", old_name, new_name); +- isulad_set_error_message("Failed to rename %s to %s in id-name manager", old_name, new_name); ++ isulad_set_error_message("Conflict. The name \"%s\" is already in use by container %s. " ++ "You have to remove (or rename) that container to be able to reuse that name.", ++ new_name, new_name); + ret = -1; + goto out; + } +-- +2.25.1 + diff --git a/0091-add-enable-cri-v1-in-k8s-integration.patch b/0091-add-enable-cri-v1-in-k8s-integration.patch new file mode 100644 index 0000000..8c36478 --- /dev/null +++ b/0091-add-enable-cri-v1-in-k8s-integration.patch @@ -0,0 +1,54 @@ +From 1f69ffe589f7225a1db83377e276ddbab963bd16 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Tue, 21 May 2024 01:13:08 +0000 +Subject: [PATCH 091/108] add enable cri v1 in k8s integration + +Signed-off-by: jikai +--- + docs/manual/k8s_integration.md | 9 +++++++++ + docs/manual/k8s_integration_zh.md | 9 +++++++++ + 2 files changed, 18 insertions(+) + +diff --git a/docs/manual/k8s_integration.md b/docs/manual/k8s_integration.md +index 8fcd0a54..14de0ef4 100644 +--- a/docs/manual/k8s_integration.md ++++ b/docs/manual/k8s_integration.md +@@ -20,6 +20,15 @@ + + if `hosts` is not configured, the default endpoint is `unix:///var/run/isulad.sock`. + ++ `iSulad` supports both `CRI V1alpha2` and `CRI V1`, and uses `CRI V1alph2` by default. ++ If `CRI V1` is required, it can be configured in `/etc/isulad/daemon.json` to enable `CRI V1`: ++ ++ ```json ++ "enable-cri-v1": true, ++ ``` ++ ++ If `iSulad` is compiled from source codes, `-D ENABLE_CRI_API_V1=ON` option is required in cmake. ++ + 2. Restart `isulad`: + + ```bash +diff --git a/docs/manual/k8s_integration_zh.md b/docs/manual/k8s_integration_zh.md +index 6dda1e4d..26ba6cc4 100644 +--- a/docs/manual/k8s_integration_zh.md ++++ b/docs/manual/k8s_integration_zh.md +@@ -20,6 +20,15 @@ + + 如果`hosts`没有配置，默认的`endpoint`为``unix:///var/run/isulad.sock`` + ++ `iSulad`同时支持`CRI V1alpha2`和`CRI V1`两种`CRI`接口，默认使用`CRI V1alph2`，若需使用`CRI V1`， ++ 需要在`/etc/isulad/daemon.json`对`iSulad`进行相关配置，配置方式为： ++ ++ ```json ++ "enable-cri-v1": true, ++ ``` ++ ++ 若使用源码编译`iSulad`，还需在编译时增加cmake编译选项`-D ENABLE_CRI_API_V1=ON`。 ++ + 2. 重启`isulad` + + ```bash +-- +2.25.1 + diff --git a/0092-isolate-oom-monitor-codes.patch b/0092-isolate-oom-monitor-codes.patch new file mode 100644 index 0000000..d7d61de --- /dev/null +++ b/0092-isolate-oom-monitor-codes.patch @@ -0,0 +1,317 @@ +From d97656a8b99f4fa95a9c15abfbac777a94b84d55 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Mon, 20 May 2024 08:48:00 +0000 +Subject: [PATCH 092/108] isolate oom monitor codes + +Signed-off-by: jikai +--- + cmake/options.cmake | 7 +++++++ + src/daemon/common/cgroup/cgroup.c | 2 ++ + src/daemon/common/cgroup/cgroup.h | 2 ++ + src/daemon/common/cgroup/cgroup_common.h | 4 ++++ + src/daemon/common/cgroup/cgroup_v1.c | 8 ++++++++ + src/daemon/common/cgroup/cgroup_v2.c | 8 ++++++++ + src/daemon/common/cri/v1/v1_cri_helpers.cc | 2 ++ + src/daemon/modules/container/container_state.c | 2 ++ + .../modules/container/supervisor/supervisor.c | 14 ++++++++++++++ + 9 files changed, 49 insertions(+) + +diff --git a/cmake/options.cmake b/cmake/options.cmake +index a15b8194..5b17f631 100644 +--- a/cmake/options.cmake ++++ b/cmake/options.cmake +@@ -58,6 +58,13 @@ if (ENABLE_SANDBOXER STREQUAL "ON") + message("${Green}-- Enable sandbox API${ColourReset}") + endif() + ++option(ENABLE_OOM_MONITOR "Enable oom monitor" ON) ++IF (ENABLE_OOM_MONITOR STREQUAL "ON") ++ add_definitions(-DENABLE_OOM_MONITOR) ++ set(ENABLE_OOM_MONITOR 1) ++ message("${Green}-- Enable oom monitor${ColourReset}") ++endif() ++ + option(ENABLE_SYSTEMD_NOTIFY "Enable systemd notify" ON) + if (ENABLE_SYSTEMD_NOTIFY STREQUAL "ON") + add_definitions(-DSYSTEMD_NOTIFY) +diff --git a/src/daemon/common/cgroup/cgroup.c b/src/daemon/common/cgroup/cgroup.c +index 71bf9801..77fafdae 100644 +--- a/src/daemon/common/cgroup/cgroup.c ++++ b/src/daemon/common/cgroup/cgroup.c +@@ -197,6 +197,7 @@ char *common_convert_cgroup_path(const char *cgroup_path) + return util_strdup_s(result); + } + ++#ifdef ENABLE_OOM_MONITOR + cgroup_oom_handler_info_t *common_get_cgroup_oom_handler(int fd, const char *name, const char *cgroup_path, + const char *exit_fifo) + { +@@ -225,3 +226,4 @@ void common_free_cgroup_oom_handler_info(cgroup_oom_handler_info_t *info) + free(info->cgroup_memory_event_path); + free(info); + } ++#endif +diff --git a/src/daemon/common/cgroup/cgroup.h b/src/daemon/common/cgroup/cgroup.h +index 0bbb70a0..1ebbfa98 100644 +--- a/src/daemon/common/cgroup/cgroup.h ++++ b/src/daemon/common/cgroup/cgroup.h +@@ -43,9 +43,11 @@ char *common_get_own_cgroup_path(const char *subsystem); + + char *common_convert_cgroup_path(const char *cgroup_path); + ++#ifdef ENABLE_OOM_MONITOR + cgroup_oom_handler_info_t *common_get_cgroup_oom_handler(int fd, const char *name, const char *cgroup_path, + const char *exit_fifo); + void common_free_cgroup_oom_handler_info(cgroup_oom_handler_info_t *info); ++#endif + + #ifdef __cplusplus + } +diff --git a/src/daemon/common/cgroup/cgroup_common.h b/src/daemon/common/cgroup/cgroup_common.h +index 46a7de50..01fc669c 100644 +--- a/src/daemon/common/cgroup/cgroup_common.h ++++ b/src/daemon/common/cgroup/cgroup_common.h +@@ -116,6 +116,7 @@ typedef struct { + cgroup_pids_metrics_t cgpids_metrics; + } cgroup_metrics_t; + ++#ifdef ENABLE_OOM_MONITOR + #define CGROUP_OOM_HANDLE_CONTINUE false + #define CGROUP_OOM_HANDLE_CLOSE true + +@@ -126,6 +127,7 @@ typedef struct _cgroup_oom_handler_info_t { + char *cgroup_memory_event_path; + bool (*oom_event_handler)(int, void *); + } cgroup_oom_handler_info_t; ++#endif + + typedef struct { + int (*get_cgroup_version)(void); +@@ -140,8 +142,10 @@ typedef struct { + char *(*get_init_cgroup_path)(const char *subsystem); + char *(*get_own_cgroup_path)(const char *subsystem); + ++#ifdef ENABLE_OOM_MONITOR + cgroup_oom_handler_info_t *(*get_cgroup_oom_handler)(int fd, const char *name, const char *cgroup_path, + const char *exit_fifo); ++#endif + } cgroup_ops; + + #ifdef __cplusplus +diff --git a/src/daemon/common/cgroup/cgroup_v1.c b/src/daemon/common/cgroup/cgroup_v1.c +index 45b1d096..018336ea 100644 +--- a/src/daemon/common/cgroup/cgroup_v1.c ++++ b/src/daemon/common/cgroup/cgroup_v1.c +@@ -20,12 +20,16 @@ + + #include + #include ++#ifdef ENABLE_OOM_MONITOR + #include ++#endif + + #include "utils.h" + #include "sysinfo.h" + #include "err_msg.h" ++#ifdef ENABLE_OOM_MONITOR + #include "events_sender_api.h" ++#endif + + #define CGROUP_HUGETLB_LIMIT "hugetlb.%s.limit_in_bytes" + #define CGROUP_MOUNT_PATH_PREFIX "/sys/fs/cgroup/" +@@ -1052,6 +1056,7 @@ static char *common_get_cgroup_path(const char *path, const char *subsystem) + return res; + } + ++#ifdef ENABLE_OOM_MONITOR + static bool oom_cb_cgroup_v1(int fd, void *cbdata) + { + cgroup_oom_handler_info_t *info = (cgroup_oom_handler_info_t *)cbdata; +@@ -1205,6 +1210,7 @@ cleanup: + common_free_cgroup_oom_handler_info(info); + return NULL; + } ++#endif + + char *get_init_cgroup_path_v1(const char *subsystem) + { +@@ -1232,6 +1238,8 @@ int cgroup_v1_ops_init(cgroup_ops *ops) + ops->get_cgroup_mnt_and_root_path = get_cgroup_mnt_and_root_path_v1; + ops->get_init_cgroup_path = get_init_cgroup_path_v1; + ops->get_own_cgroup_path = get_own_cgroup_v1; ++#ifdef ENABLE_OOM_MONITOR + ops->get_cgroup_oom_handler = get_cgroup_oom_handler_v1; ++#endif + return 0; + } +\ No newline at end of file +diff --git a/src/daemon/common/cgroup/cgroup_v2.c b/src/daemon/common/cgroup/cgroup_v2.c +index 76754dc1..ce72e6c4 100644 +--- a/src/daemon/common/cgroup/cgroup_v2.c ++++ b/src/daemon/common/cgroup/cgroup_v2.c +@@ -17,14 +17,18 @@ + #include + #include + #include ++#ifdef ENABLE_OOM_MONITOR + #include ++#endif + + #include + + #include "utils.h" + #include "path.h" + #include "sysinfo.h" ++#ifdef ENABLE_OOM_MONITOR + #include "events_sender_api.h" ++#endif + + // Cgroup V2 Item Definition + #define CGROUP2_CPU_WEIGHT "cpu.weight" +@@ -416,6 +420,7 @@ static int get_cgroup_mnt_and_root_v2(const char *subsystem, char **mountpoint, + return 0; + } + ++#ifdef ENABLE_OOM_MONITOR + static bool oom_cb_cgroup_v2(int fd, void *cbdata) + { + const size_t events_size = sizeof(struct inotify_event) + NAME_MAX + 1; +@@ -547,6 +552,7 @@ cleanup: + common_free_cgroup_oom_handler_info(info); + return NULL; + } ++#endif + + int get_cgroup_version_v2() + { +@@ -562,6 +568,8 @@ int cgroup_v2_ops_init(cgroup_ops *ops) + ops->get_cgroup_info = get_cgroup_info_v2; + ops->get_cgroup_metrics = get_cgroup_metrics_v2; + ops->get_cgroup_mnt_and_root_path = get_cgroup_mnt_and_root_v2; ++#ifdef ENABLE_OOM_MONITOR + ops->get_cgroup_oom_handler = get_cgroup_oom_handler_v2; ++#endif + return 0; + } +\ No newline at end of file +diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.cc b/src/daemon/common/cri/v1/v1_cri_helpers.cc +index 478dd105..31b6b137 100644 +--- a/src/daemon/common/cri/v1/v1_cri_helpers.cc ++++ b/src/daemon/common/cri/v1/v1_cri_helpers.cc +@@ -517,9 +517,11 @@ void UpdateBaseStatusFromInspect( + } else { // Case 3 + state = runtime::v1::CONTAINER_CREATED; + } ++#ifdef ENABLE_OOM_MONITOR + if (inspect->state->oom_killed == true) { + reason = "OOMKilled"; + } ++#endif + if (inspect->state->error != nullptr) { + message = inspect->state->error; + } +diff --git a/src/daemon/modules/container/container_state.c b/src/daemon/modules/container/container_state.c +index 452a2b26..f8ad0537 100644 +--- a/src/daemon/modules/container/container_state.c ++++ b/src/daemon/modules/container/container_state.c +@@ -587,7 +587,9 @@ container_inspect_state *container_state_to_inspect_state(container_state_t *s) + state->running = s->state->running; + state->paused = s->state->paused; + state->restarting = s->state->restarting; ++#ifdef ENABLE_OOM_MONITOR + state->oom_killed = s->state->oom_killed; ++#endif + state->pid = s->state->pid; + + state->exit_code = s->state->exit_code; +diff --git a/src/daemon/modules/container/supervisor/supervisor.c b/src/daemon/modules/container/supervisor/supervisor.c +index 39d9fdb8..294783eb 100644 +--- a/src/daemon/modules/container/supervisor/supervisor.c ++++ b/src/daemon/modules/container/supervisor/supervisor.c +@@ -42,8 +42,10 @@ + #ifdef ENABLE_CRI_API_V1 + #include "sandbox_ops.h" + #endif ++#ifdef ENABLE_OOM_MONITOR + #include "cgroup.h" + #include "specs_api.h" ++#endif + + pthread_mutex_t g_supervisor_lock = PTHREAD_MUTEX_INITIALIZER; + struct epoll_descr g_supervisor_descr; +@@ -286,6 +288,7 @@ static int supervisor_exit_cb(int fd, uint32_t events, void *cbdata, struct epol + return EPOLL_LOOP_HANDLE_CONTINUE; + } + ++#ifdef ENABLE_OOM_MONITOR + static int oom_handle_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr) + { + cgroup_oom_handler_info_t *oom_handler_info = (cgroup_oom_handler_info_t *)cbdata; +@@ -305,6 +308,7 @@ static int oom_handle_cb(int fd, uint32_t events, void *cbdata, struct epoll_des + + return EPOLL_LOOP_HANDLE_CONTINUE; + } ++#endif + + /* supervisor add exit monitor */ + int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const pid_ppid_info_t *pid_info, +@@ -312,8 +316,10 @@ int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const p + { + int ret = 0; + struct supervisor_handler_data *data = NULL; ++#ifdef ENABLE_OOM_MONITOR + cgroup_oom_handler_info_t *oom_handler_info = NULL; + __isula_auto_free char *cgroup_path = NULL; ++#endif + + if (fd < 0) { + ERROR("Invalid exit fifo fd"); +@@ -326,12 +332,14 @@ int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const p + return -1; + } + ++#ifdef ENABLE_OOM_MONITOR + cgroup_path = merge_container_cgroups_path(cont->common_config->id, cont->hostconfig); + if (cgroup_path == NULL) { + ERROR("Failed to get cgroup path"); + close(fd); + return -1; + } ++#endif + + data = util_common_calloc_s(sizeof(struct supervisor_handler_data)); + if (data == NULL) { +@@ -353,9 +361,12 @@ int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const p + data->pid_info.start_time = pid_info->start_time; + data->pid_info.ppid = pid_info->ppid; + data->pid_info.pstart_time = pid_info->pstart_time; ++#ifdef ENABLE_OOM_MONITOR + oom_handler_info = common_get_cgroup_oom_handler(fd, cont->common_config->id, cgroup_path, exit_fifo); ++#endif + + supervisor_handler_lock(); ++#ifdef ENABLE_OOM_MONITOR + if (oom_handler_info != NULL) { + ret = epoll_loop_add_handler(&g_supervisor_descr, oom_handler_info->oom_event_fd, oom_handle_cb, oom_handler_info); + if (ret != 0) { +@@ -363,6 +374,7 @@ int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const p + goto err; + } + } ++#endif + + ret = epoll_loop_add_handler(&g_supervisor_descr, fd, supervisor_exit_cb, data); + if (ret != 0) { +@@ -374,7 +386,9 @@ int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const p + + err: + supervisor_handler_data_free(data); ++#ifdef ENABLE_OOM_MONITOR + common_free_cgroup_oom_handler_info(oom_handler_info); ++#endif + out: + supervisor_handler_unlock(); + return ret; +-- +2.25.1 + diff --git a/0093-change-fork-process-exit-mode.patch b/0093-change-fork-process-exit-mode.patch new file mode 100644 index 0000000..2806227 --- /dev/null +++ b/0093-change-fork-process-exit-mode.patch @@ -0,0 +1,64 @@ +From 0ff5a421e31096fbd10cf00c45a3849297023391 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Thu, 23 May 2024 01:09:41 +1400 +Subject: [PATCH 093/108] change fork process exit mode + +--- + src/utils/tar/util_archive.c | 16 ++++++++-------- + 1 file changed, 8 insertions(+), 8 deletions(-) + +diff --git a/src/utils/tar/util_archive.c b/src/utils/tar/util_archive.c +index 52b51162..204dab83 100644 +--- a/src/utils/tar/util_archive.c ++++ b/src/utils/tar/util_archive.c +@@ -897,9 +897,9 @@ int archive_unpack(const struct io_read_wrapper *content, const char *dstdir, co + + child_out: + if (ret != 0) { +- exit(EXIT_FAILURE); ++ _exit(EXIT_FAILURE); + } +- exit(EXIT_SUCCESS); ++ _exit(EXIT_SUCCESS); + } + close(pipe_stderr[1]); + pipe_stderr[1] = -1; +@@ -1342,9 +1342,9 @@ int archive_chroot_tar(const char *path, const char *file, const char *root_dir, + child_out: + + if (ret != 0) { +- exit(EXIT_FAILURE); ++ _exit(EXIT_FAILURE); + } else { +- exit(EXIT_SUCCESS); ++ _exit(EXIT_SUCCESS); + } + } + close(pipe_for_read[1]); +@@ -1577,9 +1577,9 @@ int archive_chroot_untar_stream(const struct io_read_wrapper *context, const cha + + child_out: + if (ret != 0) { +- exit(EXIT_FAILURE); ++ _exit(EXIT_FAILURE); + } +- exit(EXIT_SUCCESS); ++ _exit(EXIT_SUCCESS); + } + + close(pipe_stderr[1]); +@@ -1727,9 +1727,9 @@ child_out: + free(tar_base_name); + + if (ret != 0) { +- exit(EXIT_FAILURE); ++ _exit(EXIT_FAILURE); + } else { +- exit(EXIT_SUCCESS); ++ _exit(EXIT_SUCCESS); + } + } + +-- +2.25.1 + diff --git a/0094-fix-error-log-for-verify_cpu_realtime.patch b/0094-fix-error-log-for-verify_cpu_realtime.patch new file mode 100644 index 0000000..c62bbe8 --- /dev/null +++ b/0094-fix-error-log-for-verify_cpu_realtime.patch @@ -0,0 +1,26 @@ +From 1671a136d1b7d209c453a8ad2b1bf062a3afbe09 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Tue, 28 May 2024 17:23:45 +1400 +Subject: [PATCH 094/108] fix error log for verify_cpu_realtime + +Signed-off-by: zhongtao +--- + src/daemon/modules/spec/verify.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/daemon/modules/spec/verify.c b/src/daemon/modules/spec/verify.c +index 57e16ef9..1ce76c1d 100644 +--- a/src/daemon/modules/spec/verify.c ++++ b/src/daemon/modules/spec/verify.c +@@ -390,7 +390,7 @@ static int verify_cpu_realtime(const sysinfo_t *sysinfo, int64_t realtime_period + + if (realtime_runtime != 0 && !(sysinfo->cgcpuinfo.cpu_rt_runtime)) { + ERROR("Invalid --cpu-rt-runtime: Your kernel does not support cgroup rt runtime"); +- isulad_set_error_message("Invalid --cpu-rt-period: Your kernel does not support cgroup rt runtime"); ++ isulad_set_error_message("Invalid --cpu-rt-runtime: Your kernel does not support cgroup rt runtime"); + ret = -1; + goto out; + } +-- +2.25.1 + diff --git a/0095-bugfix-change-max-network-name-len.patch b/0095-bugfix-change-max-network-name-len.patch new file mode 100644 index 0000000..3a4ffb7 --- /dev/null +++ b/0095-bugfix-change-max-network-name-len.patch @@ -0,0 +1,54 @@ +From e2a7e6bfb0f0e97e5e1543fac7a5e0807fadaba0 Mon Sep 17 00:00:00 2001 +From: liuxu +Date: Thu, 30 May 2024 16:50:56 +0800 +Subject: [PATCH 095/108] bugfix: change max network name len + +Signed-off-by: liuxu +--- + docs/design/detailed/Network/native_network_adapter_design.md | 2 +- + .../design/detailed/Network/native_network_adapter_design_zh.md | 2 +- + src/utils/cutils/utils.h | 2 +- + 3 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/docs/design/detailed/Network/native_network_adapter_design.md b/docs/design/detailed/Network/native_network_adapter_design.md +index b58989eb..1010bd73 100644 +--- a/docs/design/detailed/Network/native_network_adapter_design.md ++++ b/docs/design/detailed/Network/native_network_adapter_design.md +@@ -204,7 +204,7 @@ Client: + 1. Parse the parameters passed in by the user. + 2. Verify the incoming parameters, including: + - Only one network is allowed to be created at a time, that is, at most one name can be specified. +- - If name is specified, check whether the length of name exceeds MAX_NETWORK_NAME_LEN(128). ++ - If name is specified, check whether the length of name exceeds MAX_NETWORK_NAME_LEN(255). + 3. Send the request to the server + + Server: +diff --git a/docs/design/detailed/Network/native_network_adapter_design_zh.md b/docs/design/detailed/Network/native_network_adapter_design_zh.md +index 30860f3d..62c4b6ef 100644 +--- a/docs/design/detailed/Network/native_network_adapter_design_zh.md ++++ b/docs/design/detailed/Network/native_network_adapter_design_zh.md +@@ -204,7 +204,7 @@ int native_network_add_container_list(const char *network_name, const char *cont + 1. 解析用户传入的参数 + 2. 对传入的参数进行校验，包括 + - 每次只允许创建一个网络， 即最多指定一个name +- - 若指定name，检查name长度是否超过MAX_NETWORK_NAME_LEN(128) ++ - 若指定name，检查name长度是否超过MAX_NETWORK_NAME_LEN(255) + 3. 发送请求到服务端 + + 服务端： +diff --git a/src/utils/cutils/utils.h b/src/utils/cutils/utils.h +index 3671272a..4417a165 100644 +--- a/src/utils/cutils/utils.h ++++ b/src/utils/cutils/utils.h +@@ -99,7 +99,7 @@ int malloc_trim(size_t pad); + #define MAX_IMAGE_REF_LEN 384 + #define MAX_CONTAINER_NAME_LEN 1024 + #define MAX_RUNTIME_NAME_LEN 32 +-#define MAX_NETWORK_NAME_LEN 128 ++#define MAX_NETWORK_NAME_LEN 255 + + #define LOGIN_USERNAME_LEN 255 + #define LOGIN_PASSWORD_LEN 255 +-- +2.25.1 + diff --git a/0096-del-useless-info.patch b/0096-del-useless-info.patch new file mode 100644 index 0000000..ea885dd --- /dev/null +++ b/0096-del-useless-info.patch @@ -0,0 +1,26 @@ +From b36cfa4325f43b3fa1468ba360b3d51f6ef1c3ca Mon Sep 17 00:00:00 2001 +From: liuxu +Date: Thu, 30 May 2024 17:14:48 +0800 +Subject: [PATCH 096/108] del useless info + +Signed-off-by: liuxu +--- + release_notes | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/release_notes b/release_notes +index 2d564c7e..fdcd3690 100644 +--- a/release_notes ++++ b/release_notes +@@ -72,7 +72,7 @@ + + dev stats: + - 357 files changed, 7886 insertions(+), 2849 deletions(-) +- - contributors: zhongtao, jikai, haozi007, jake, liuxu, xuxuepeng, zhangxiaoyu, sailorvii, chen524, dreamloy, l00804245, yangjiaqi ++ - contributors: zhongtao, jikai, haozi007, jake, liuxu, xuxuepeng, zhangxiaoyu, sailorvii, chen524, dreamloy, yangjiaqi + + 2023-11-07 xuepengxu release 2.1.4 + - !2238 modify the default value of EANBLE_IMAGE_LIBARAY to off * modify the default value of EANBLE_IMAGE_LIBARAY to off +-- +2.25.1 + diff --git a/0097-code-improve.patch b/0097-code-improve.patch new file mode 100644 index 0000000..cb8d41d --- /dev/null +++ b/0097-code-improve.patch @@ -0,0 +1,83 @@ +From 2aa26649f20ae5992ace7bb8cb62a5ee9c3f7a81 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Thu, 30 May 2024 21:30:43 +1400 +Subject: [PATCH 097/108] code improve + +Signed-off-by: zhongtao +--- + src/cmd/isula/volume/prune.c | 2 +- + src/cmd/options/opt_log.c | 2 +- + src/daemon/executor/container_cb/execution_extend.c | 3 ++- + src/daemon/executor/container_cb/execution_information.c | 2 +- + src/daemon/modules/image/oci/registry_type.c | 2 +- + 5 files changed, 6 insertions(+), 5 deletions(-) + +diff --git a/src/cmd/isula/volume/prune.c b/src/cmd/isula/volume/prune.c +index c8d632ed..3b5bfbf1 100644 +--- a/src/cmd/isula/volume/prune.c ++++ b/src/cmd/isula/volume/prune.c +@@ -79,7 +79,7 @@ out: + + int cmd_volume_prune_main(int argc, const char **argv) + { +- int i = 0; ++ size_t i = 0; + struct isula_libutils_log_config lconf = { 0 }; + int exit_code = 1; + command_t cmd; +diff --git a/src/cmd/options/opt_log.c b/src/cmd/options/opt_log.c +index b1abcfaf..8ffb9966 100644 +--- a/src/cmd/options/opt_log.c ++++ b/src/cmd/options/opt_log.c +@@ -67,7 +67,7 @@ static int log_opt_syslog_facility(const char *key, const char *value, char **pa + "authpriv", "ftp", "local0", "local1", "local2", + "local3", "local4", "local5", "local6", "local7" + }; +- int i; ++ size_t i; + size_t f_len = sizeof(facility_values) / sizeof(const char *); + + for (i = 0; i < f_len; i++) { +diff --git a/src/daemon/executor/container_cb/execution_extend.c b/src/daemon/executor/container_cb/execution_extend.c +index 52401633..ed072848 100644 +--- a/src/daemon/executor/container_cb/execution_extend.c ++++ b/src/daemon/executor/container_cb/execution_extend.c +@@ -990,7 +990,8 @@ out: + + static int update_container_unified(const char *id, const host_config *hostconfig, host_config *chostconfig) + { +- int i, cgroup_version; ++ int cgroup_version; ++ size_t i; + + if (hostconfig->unified == NULL || hostconfig->unified->len == 0) { + return 0; +diff --git a/src/daemon/executor/container_cb/execution_information.c b/src/daemon/executor/container_cb/execution_information.c +index 58924257..4aee3aef 100644 +--- a/src/daemon/executor/container_cb/execution_information.c ++++ b/src/daemon/executor/container_cb/execution_information.c +@@ -244,7 +244,7 @@ static int get_proxy_env(char **proxy, const char *type) + } + *col_pos = '\0'; + nret = snprintf(*proxy, proxy_len, "%s:%s%s", tmp_proxy, mask_str, at_pos); +- if (nret < 0 || nret >= proxy_len) { ++ if (nret < 0 || (size_t)nret >= proxy_len) { + ret = -1; + free(*proxy); + *proxy = NULL; +diff --git a/src/daemon/modules/image/oci/registry_type.c b/src/daemon/modules/image/oci/registry_type.c +index 6c9ff747..7a2c25ed 100644 +--- a/src/daemon/modules/image/oci/registry_type.c ++++ b/src/daemon/modules/image/oci/registry_type.c +@@ -59,7 +59,7 @@ void free_layer_blob(layer_blob *layer) + + void free_pull_desc(pull_descriptor *desc) + { +- int i = 0; ++ size_t i = 0; + + if (desc == NULL) { + return; +-- +2.25.1 + diff --git a/0098-cdi-add-debug-info.patch b/0098-cdi-add-debug-info.patch new file mode 100644 index 0000000..aec57f6 --- /dev/null +++ b/0098-cdi-add-debug-info.patch @@ -0,0 +1,33 @@ +From 5a5f4879246783932ab620b2461a7cd832ddc1f0 Mon Sep 17 00:00:00 2001 +From: liuxu +Date: Fri, 31 May 2024 16:38:26 +0800 +Subject: [PATCH 098/108] cdi:add debug info + +Signed-off-by: liuxu +--- + src/daemon/modules/device/cdi/cdi_cache.c | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/src/daemon/modules/device/cdi/cdi_cache.c b/src/daemon/modules/device/cdi/cdi_cache.c +index e9a9b804..cd7158dd 100644 +--- a/src/daemon/modules/device/cdi/cdi_cache.c ++++ b/src/daemon/modules/device/cdi/cdi_cache.c +@@ -306,6 +306,7 @@ static void refresh_scan_spec_func(struct cdi_scan_fn_maps *scan_fn_maps, const + ERROR("Failed to insert device to devices by name %s", qualified); + goto error_out; + } ++ DEBUG("Add device %s into memory", qualified); + free(qualified); + qualified = NULL; + } +@@ -445,6 +446,7 @@ static int cdi_inject_devices(struct cdi_cache *c, oci_runtime_spec *oci_spec, s + + for (i = 0; i < devices->len; i++) { + device = devices->items[i]; ++ DEBUG("Search cdi devices %s.", device); + d = map_search(c->devices, (void *)device); + if (d == NULL) { + if (util_append_string_array(unresolved, device) != 0) { +-- +2.25.1 + diff --git a/0099-bugfix-cni-network-name-UT.patch b/0099-bugfix-cni-network-name-UT.patch new file mode 100644 index 0000000..4ed04c3 --- /dev/null +++ b/0099-bugfix-cni-network-name-UT.patch @@ -0,0 +1,28 @@ +From 4a98535064319a9df3143d9c4b397f44fbbb56c5 Mon Sep 17 00:00:00 2001 +From: liuxu +Date: Mon, 3 Jun 2024 16:11:06 +0800 +Subject: [PATCH 099/108] bugfix:cni network name UT + +Signed-off-by: liuxu +--- + test/cutils/utils_network/utils_network_ut.cc | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/test/cutils/utils_network/utils_network_ut.cc b/test/cutils/utils_network/utils_network_ut.cc +index 33eb6eb5..68f6f011 100644 +--- a/test/cutils/utils_network/utils_network_ut.cc ++++ b/test/cutils/utils_network/utils_network_ut.cc +@@ -344,8 +344,8 @@ TEST(utils_network, test_net_contain_ip) + TEST(utils_network, test_validate_network_name) + { + ASSERT_EQ(util_validate_network_name(nullptr), false); +- ASSERT_EQ(util_validate_network_name(std::string(128, 'a').c_str()), true); +- ASSERT_EQ(util_validate_network_name(std::string(129, 'a').c_str()), false); ++ ASSERT_EQ(util_validate_network_name(std::string(255, 'a').c_str()), true); ++ ASSERT_EQ(util_validate_network_name(std::string(256, 'a').c_str()), false); + ASSERT_EQ(util_validate_network_name(std::string(".abce").c_str()), false); + } + +-- +2.25.1 + diff --git a/0100-bugfix-malloc-right-type-size.patch b/0100-bugfix-malloc-right-type-size.patch new file mode 100644 index 0000000..983d55e --- /dev/null +++ b/0100-bugfix-malloc-right-type-size.patch @@ -0,0 +1,40 @@ +From 7bf26415fcba090e281324ba92f7d7e6487b94fc Mon Sep 17 00:00:00 2001 +From: liuxu +Date: Tue, 4 Jun 2024 16:30:19 +0800 +Subject: [PATCH 100/108] bugfix:malloc right type size + +Signed-off-by: liuxu +--- + src/client/connect/rest/rest_volumes_client.c | 2 +- + src/daemon/modules/service/service_network.c | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/src/client/connect/rest/rest_volumes_client.c b/src/client/connect/rest/rest_volumes_client.c +index 88254e6c..2327c28e 100644 +--- a/src/client/connect/rest/rest_volumes_client.c ++++ b/src/client/connect/rest/rest_volumes_client.c +@@ -251,7 +251,7 @@ static int prune_request_to_rest(const struct isula_prune_volume_request *reques + parser_error err = NULL; + int ret = 0; + +- nrequest = util_common_calloc_s(sizeof(volume_list_volume_request)); ++ nrequest = util_common_calloc_s(sizeof(volume_prune_volume_request)); + if (nrequest == NULL) { + ERROR("Out of memory"); + return -1; +diff --git a/src/daemon/modules/service/service_network.c b/src/daemon/modules/service/service_network.c +index 6754cf1a..fe6dbf1d 100644 +--- a/src/daemon/modules/service/service_network.c ++++ b/src/daemon/modules/service/service_network.c +@@ -74,7 +74,7 @@ static struct attach_net_conf_list *build_attach_networks(const defs_map_string_ + return NULL; + } + +- list = (struct attach_net_conf_list *)util_common_calloc_s(sizeof(struct attach_net_conf)); ++ list = (struct attach_net_conf_list *)util_common_calloc_s(sizeof(struct attach_net_conf_list)); + if (list == NULL) { + ERROR("Out of memory"); + return NULL; +-- +2.25.1 + diff --git a/0101-use-isula_clean_path-rather-than-realpath.patch b/0101-use-isula_clean_path-rather-than-realpath.patch new file mode 100644 index 0000000..6ac4423 --- /dev/null +++ b/0101-use-isula_clean_path-rather-than-realpath.patch @@ -0,0 +1,26 @@ +From 60a2b15e0090018b7850b37369964bf62e253419 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Wed, 5 Jun 2024 02:07:23 +0000 +Subject: [PATCH 101/108] use isula_clean_path rather than realpath + +Signed-off-by: jikai +--- + src/cmd/isulad-shim/process.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c +index 18fae03f..dd41c77f 100644 +--- a/src/cmd/isulad-shim/process.c ++++ b/src/cmd/isulad-shim/process.c +@@ -489,7 +489,7 @@ static bool attach_fifopath_security_check(process_t *p, const char *fifopath) + return false; + } + +- if (realpath(fifopath, real_path) == NULL) { ++ if (isula_clean_path(fifopath, real_path, sizeof(real_path)) == NULL) { + ERROR("Failed to get realpath for '%s': %d.", real_path, SHIM_SYS_ERR(errno)); + return false; + } +-- +2.25.1 + diff --git a/0102-fix-false-engine-rootpath-reference.patch b/0102-fix-false-engine-rootpath-reference.patch new file mode 100644 index 0000000..a375a44 --- /dev/null +++ b/0102-fix-false-engine-rootpath-reference.patch @@ -0,0 +1,32 @@ +From fa25a8923e47ed6b65e0bcd08954589f5b26092c Mon Sep 17 00:00:00 2001 +From: jikai +Date: Wed, 5 Jun 2024 02:10:48 +0000 +Subject: [PATCH 102/108] fix false engine rootpath reference + +Signed-off-by: jikai +--- + src/daemon/config/isulad_config.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c +index 80689bce..d7b54498 100644 +--- a/src/daemon/config/isulad_config.c ++++ b/src/daemon/config/isulad_config.c +@@ -398,12 +398,12 @@ char *conf_get_sandbox_rootpath(void) + ERROR("Get rootpath failed"); + return epath; + } +- if (strlen(rootpath) > (PATH_MAX - strlen(ENGINE_ROOTPATH_NAME)) - 2) { ++ if (strlen(rootpath) > (PATH_MAX - strlen(SANDBOX_ROOTPATH_NAME)) - 2) { + ERROR("Root path is too long"); + return epath; + } + // rootpath + "/" + SANDBOX_ROOTPATH_NAME + "/0" +- len = strlen(rootpath) + 1 + strlen(ENGINE_ROOTPATH_NAME) + 1; ++ len = strlen(rootpath) + 1 + strlen(SANDBOX_ROOTPATH_NAME) + 1; + epath = util_smart_calloc_s(sizeof(char), len); + if (epath == NULL) { + ERROR("Out of memory"); +-- +2.25.1 + diff --git a/0103-bugfix-add-note.patch b/0103-bugfix-add-note.patch new file mode 100644 index 0000000..1cdc42b --- /dev/null +++ b/0103-bugfix-add-note.patch @@ -0,0 +1,25 @@ +From 8eea40e09aa34da85cfa191f07cfe7e123c9809d Mon Sep 17 00:00:00 2001 +From: liuxu +Date: Wed, 5 Jun 2024 10:18:06 +0800 +Subject: [PATCH 103/108] bugfix:add note + +Signed-off-by: liuxu +--- + src/utils/cutils/utils_array.h | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/src/utils/cutils/utils_array.h b/src/utils/cutils/utils_array.h +index 0c4fd217..553c2c13 100644 +--- a/src/utils/cutils/utils_array.h ++++ b/src/utils/cutils/utils_array.h +@@ -30,6 +30,7 @@ void util_free_array_by_len(char **array, size_t len); + + void util_free_array(char **array); + ++// this function just copies the first len elements of array and does not automatically add NULL element in the end. + char **util_copy_array_by_len(char **array, size_t len); + + int util_grow_array(char ***orig_array, size_t *orig_capacity, size_t size, +-- +2.25.1 + diff --git a/0104-bugfix-adapt-network-name-max-len.patch b/0104-bugfix-adapt-network-name-max-len.patch new file mode 100644 index 0000000..0d7c6a3 --- /dev/null +++ b/0104-bugfix-adapt-network-name-max-len.patch @@ -0,0 +1,64 @@ +From 174f7d9d959bd129675651ccf7ef460794188b63 Mon Sep 17 00:00:00 2001 +From: liuxu +Date: Wed, 5 Jun 2024 10:40:22 +0800 +Subject: [PATCH 104/108] bugfix:adapt network name max len + +Signed-off-by: liuxu +--- + CI/test_cases/network_cases/network_create.sh | 5 +++-- + src/utils/cutils/utils.h | 7 ++++++- + test/cutils/utils_network/utils_network_ut.cc | 4 ++-- + 3 files changed, 11 insertions(+), 5 deletions(-) + +diff --git a/CI/test_cases/network_cases/network_create.sh b/CI/test_cases/network_cases/network_create.sh +index 3107da12..da54fdd0 100755 +--- a/CI/test_cases/network_cases/network_create.sh ++++ b/CI/test_cases/network_cases/network_create.sh +@@ -33,8 +33,9 @@ function test_network_create() + local name1="cni1" + local name2="cni2" + local name3="a" +- for i in $(seq 1 7);do +- name3=${name3}${name3} ++ local basechar="a" ++ for i in $(seq 1 199);do ++ name3=${name3}${basechar} + done + local name4=${name3}b + local invalid_name=".xx" +diff --git a/src/utils/cutils/utils.h b/src/utils/cutils/utils.h +index 4417a165..ce0ca703 100644 +--- a/src/utils/cutils/utils.h ++++ b/src/utils/cutils/utils.h +@@ -99,7 +99,12 @@ int malloc_trim(size_t pad); + #define MAX_IMAGE_REF_LEN 384 + #define MAX_CONTAINER_NAME_LEN 1024 + #define MAX_RUNTIME_NAME_LEN 32 +-#define MAX_NETWORK_NAME_LEN 255 ++/* ++ * Linux limits the length of the file name to 255, ++ * isulad will create file by name "${ISULAD_CNI_NETWORK_CONF_FILE_PRE}${network_name}.conflist" ++ * when create native network,so we limit the length of the network name to 200. ++ */ ++#define MAX_NETWORK_NAME_LEN 200 + + #define LOGIN_USERNAME_LEN 255 + #define LOGIN_PASSWORD_LEN 255 +diff --git a/test/cutils/utils_network/utils_network_ut.cc b/test/cutils/utils_network/utils_network_ut.cc +index 68f6f011..be85e398 100644 +--- a/test/cutils/utils_network/utils_network_ut.cc ++++ b/test/cutils/utils_network/utils_network_ut.cc +@@ -344,8 +344,8 @@ TEST(utils_network, test_net_contain_ip) + TEST(utils_network, test_validate_network_name) + { + ASSERT_EQ(util_validate_network_name(nullptr), false); +- ASSERT_EQ(util_validate_network_name(std::string(255, 'a').c_str()), true); +- ASSERT_EQ(util_validate_network_name(std::string(256, 'a').c_str()), false); ++ ASSERT_EQ(util_validate_network_name(std::string(200, 'a').c_str()), true); ++ ASSERT_EQ(util_validate_network_name(std::string(201, 'a').c_str()), false); + ASSERT_EQ(util_validate_network_name(std::string(".abce").c_str()), false); + } + +-- +2.25.1 + diff --git a/0105-start-sandbox-before-setup-network-by-default.patch b/0105-start-sandbox-before-setup-network-by-default.patch new file mode 100644 index 0000000..df5c44f --- /dev/null +++ b/0105-start-sandbox-before-setup-network-by-default.patch @@ -0,0 +1,140 @@ +From e6b3528acff10fb2bc62e2da0c3754f1e36cbd54 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Wed, 5 Jun 2024 10:04:59 +0800 +Subject: [PATCH 105/108] start sandbox before setup network by default + +Signed-off-by: zhongtao +--- + src/daemon/common/cri/cri_helpers.cc | 12 +++++++ + src/daemon/common/cri/cri_helpers.h | 3 ++ + .../cri_pod_sandbox_manager_service.cc | 34 +++++++++++++------ + src/daemon/modules/spec/specs.c | 2 +- + src/utils/cutils/utils_file.c | 2 +- + 5 files changed, 41 insertions(+), 12 deletions(-) + +diff --git a/src/daemon/common/cri/cri_helpers.cc b/src/daemon/common/cri/cri_helpers.cc +index 68d569cc..8117403c 100644 +--- a/src/daemon/common/cri/cri_helpers.cc ++++ b/src/daemon/common/cri/cri_helpers.cc +@@ -47,6 +47,8 @@ const std::string Constants::DOCKER_IMAGEID_PREFIX { "docker://" }; + const std::string Constants::DOCKER_PULLABLE_IMAGEID_PREFIX { "docker-pullable://" }; + const std::string Constants::RUNTIME_READY { "RuntimeReady" }; + const std::string Constants::NETWORK_READY { "NetworkReady" }; ++// Kata 2.x need create network namespace and setup network befoce run podsandbox ++const std::string Constants::NETWORK_SETUP_ANNOTATION_KEY { "cri.sandbox.network.setup.v2" }; + const std::string Constants::POD_CHECKPOINT_KEY { "cri.sandbox.isulad.checkpoint" }; + const std::string Constants::CONTAINER_TYPE_ANNOTATION_KEY { "io.kubernetes.cri.container-type" }; + const std::string Constants::CONTAINER_NAME_ANNOTATION_KEY { "io.kubernetes.cri.container-name" }; +@@ -1140,4 +1142,14 @@ auto GetPodSELinuxLabelOpts(const std::string &selinuxLabel, Errors &error) + return fmtiSuladOpts(selinuxOpts, securityOptSep); + } + ++bool SetupNetworkFirst(const std::map &annotations) ++{ ++ auto iter = annotations.find(CRIHelpers::Constants::NETWORK_SETUP_ANNOTATION_KEY); ++ if (iter == annotations.end()) { ++ return false; ++ } ++ ++ return iter->second == std::string("true"); ++} ++ + } // namespace CRIHelpers +diff --git a/src/daemon/common/cri/cri_helpers.h b/src/daemon/common/cri/cri_helpers.h +index 5c450b32..11a80b45 100644 +--- a/src/daemon/common/cri/cri_helpers.h ++++ b/src/daemon/common/cri/cri_helpers.h +@@ -49,6 +49,7 @@ public: + static const std::string DOCKER_PULLABLE_IMAGEID_PREFIX; + static const std::string RUNTIME_READY; + static const std::string NETWORK_READY; ++ static const std::string NETWORK_SETUP_ANNOTATION_KEY; + static const std::string POD_CHECKPOINT_KEY; + static const size_t MAX_CHECKPOINT_KEY_LEN { 250 }; + static const std::string CONTAINER_TYPE_ANNOTATION_KEY; +@@ -151,6 +152,8 @@ auto GetPodSELinuxLabelOpts(const std::string &selinuxLabel, Errors &error) -> s + auto GetlegacySeccompiSuladOpts(const std::string &seccompProfile, Errors &error) -> std::vector; + + auto GetSeccompiSuladOptsByPath(const char *dstpath, Errors &error) -> std::vector; ++ ++bool SetupNetworkFirst(const std::map &annotations); + }; // namespace CRIHelpers + + #endif // DAEMON_ENTRY_CRI_CRI_HELPERS_H +diff --git a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +index af6b5fff..f852f4df 100644 +--- a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +@@ -655,19 +655,33 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1alpha2::PodSandbox + } + } + +- // Step 7: Setup networking for the sandbox. +- SetupSandboxNetwork(config, response_id, inspect_data, networkOptions, stdAnnos, network_setting_json, error); +- if (error.NotEmpty()) { +- goto cleanup_ns; +- } ++ // Step 7: According to the annotation and network namespace mode, ++ // determine the order of start sandbox and setup network. ++ if (CRIHelpers::SetupNetworkFirst(stdAnnos)) { ++ // Step 7.1: Setup networking for the sandbox, and then start the sandbox container. ++ SetupSandboxNetwork(config, response_id, inspect_data, networkOptions, stdAnnos, network_setting_json, error); ++ if (error.NotEmpty()) { ++ goto cleanup_ns; ++ } + +- // Step 8: Start the sandbox container. +- StartSandboxContainer(response_id, error); +- if (error.NotEmpty()) { +- goto cleanup_network; ++ StartSandboxContainer(response_id, error); ++ if (error.NotEmpty()) { ++ goto cleanup_network; ++ } ++ } else { ++ // Step 7.2: (Default)Start the sandbox container, and then setup networking for the sandbox. ++ StartSandboxContainer(response_id, error); ++ if (error.NotEmpty()) { ++ goto cleanup_ns; ++ } ++ ++ SetupSandboxNetwork(config, response_id, inspect_data, networkOptions, stdAnnos, network_setting_json, error); ++ if (error.NotEmpty()) { ++ goto cleanup_ns; ++ } + } + +- // Step 9: Save network settings json to disk ++ // Step 8: Save network settings json to disk + if (namespace_is_cni(inspect_data->host_config->network_mode)) { + Errors tmpErr; + UpdatePodSandboxNetworkSettings(response_id, network_setting_json, tmpErr); +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index 122f9992..f0538e26 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -1601,7 +1601,7 @@ static int merge_share_network_namespace(const oci_runtime_spec *oci_spec, const + int ret = 0; + char *ns_path = NULL; + +- if (host_spec->network_mode == NULL) { ++ if (host_spec->network_mode == NULL || strlen(host_spec->network_mode) == 0) { + return 0; + } + +diff --git a/src/utils/cutils/utils_file.c b/src/utils/cutils/utils_file.c +index 6fc6852d..90bb156f 100644 +--- a/src/utils/cutils/utils_file.c ++++ b/src/utils/cutils/utils_file.c +@@ -85,7 +85,7 @@ bool util_file_exists(const char *f) + struct stat buf; + int nret; + +- if (f == NULL) { ++ if (f == NULL || strlen(f) == 0) { + return false; + } + +-- +2.25.1 + diff --git a/0106-Revert-use-isula_clean_path-rather-than-realpath.patch b/0106-Revert-use-isula_clean_path-rather-than-realpath.patch new file mode 100644 index 0000000..6115474 --- /dev/null +++ b/0106-Revert-use-isula_clean_path-rather-than-realpath.patch @@ -0,0 +1,28 @@ +From f690c9a2dff298b41dc607e4ea6dd09113a322fb Mon Sep 17 00:00:00 2001 +From: jikai +Date: Thu, 6 Jun 2024 02:13:36 +0000 +Subject: [PATCH 106/108] Revert "use isula_clean_path rather than realpath" + +This reverts commit 60a2b15e0090018b7850b37369964bf62e253419. + +Signed-off-by: jikai +--- + src/cmd/isulad-shim/process.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c +index dd41c77f..18fae03f 100644 +--- a/src/cmd/isulad-shim/process.c ++++ b/src/cmd/isulad-shim/process.c +@@ -489,7 +489,7 @@ static bool attach_fifopath_security_check(process_t *p, const char *fifopath) + return false; + } + +- if (isula_clean_path(fifopath, real_path, sizeof(real_path)) == NULL) { ++ if (realpath(fifopath, real_path) == NULL) { + ERROR("Failed to get realpath for '%s': %d.", real_path, SHIM_SYS_ERR(errno)); + return false; + } +-- +2.25.1 + diff --git a/0107-bugfix-for-start-sandbox-before-setup-network-by-def.patch b/0107-bugfix-for-start-sandbox-before-setup-network-by-def.patch new file mode 100644 index 0000000..009910f --- /dev/null +++ b/0107-bugfix-for-start-sandbox-before-setup-network-by-def.patch @@ -0,0 +1,28 @@ +From 63f4f9bc9c36825d85a14f6a33102194d30e12a7 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Fri, 7 Jun 2024 02:24:49 +1400 +Subject: [PATCH 107/108] bugfix for start sandbox before setup network by + default + +Signed-off-by: zhongtao +--- + .../entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +index f852f4df..bc3f4031 100644 +--- a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +@@ -677,6 +677,9 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1alpha2::PodSandbox + + SetupSandboxNetwork(config, response_id, inspect_data, networkOptions, stdAnnos, network_setting_json, error); + if (error.NotEmpty()) { ++ Errors stopError; ++ StopContainerHelper(response_id, stopError); ++ WARN("Error stop container: %s: %s", response_id.c_str(), stopError.GetCMessage()); + goto cleanup_ns; + } + } +-- +2.25.1 + diff --git a/0108-skip-test-rely-on-docker.io.patch b/0108-skip-test-rely-on-docker.io.patch new file mode 100644 index 0000000..dd146d7 --- /dev/null +++ b/0108-skip-test-rely-on-docker.io.patch @@ -0,0 +1,59 @@ +From d3e7b0b0d19ca4937716d835e3627714157d6cc3 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Tue, 11 Jun 2024 17:14:58 +0800 +Subject: [PATCH 108/108] skip test rely on docker.io + +Signed-off-by: zhongtao +--- + CI/test_cases/image_cases/image_search.sh | 4 +++- + CI/test_cases/image_cases/integration_check.sh | 2 +- + CI/test_cases/image_cases/registry.sh | 5 +++-- + 3 files changed, 7 insertions(+), 4 deletions(-) + +diff --git a/CI/test_cases/image_cases/image_search.sh b/CI/test_cases/image_cases/image_search.sh +index 11af02f1..4bf0e099 100755 +--- a/CI/test_cases/image_cases/image_search.sh ++++ b/CI/test_cases/image_cases/image_search.sh +@@ -76,7 +76,9 @@ function test_image_search() + + declare -i ans=0 + +-test_image_search || ((ans++)) ++# unable to pull image from docker.io without agent, skip this test ++# registry API v1 is not implemented in https://3laho3y3.mirror.aliyuncs.com and isula search cannot be tested ++# test_image_search || ((ans++)) + + show_result ${ans} "${curr_path}/${0}" + +diff --git a/CI/test_cases/image_cases/integration_check.sh b/CI/test_cases/image_cases/integration_check.sh +index 6ec3ab52..f340348d 100755 +--- a/CI/test_cases/image_cases/integration_check.sh ++++ b/CI/test_cases/image_cases/integration_check.sh +@@ -27,7 +27,7 @@ image="busybox" + function test_image_info() + { + local ret=0 +- local uimage="docker.io/library/nats" ++ local uimage="nats" + local test="list && inspect image info test => (${FUNCNAME[@]})" + local lid + local cid +diff --git a/CI/test_cases/image_cases/registry.sh b/CI/test_cases/image_cases/registry.sh +index e33983d6..7ea9a0c5 100755 +--- a/CI/test_cases/image_cases/registry.sh ++++ b/CI/test_cases/image_cases/registry.sh +@@ -74,8 +74,9 @@ function isula_pull() + isula run --rm -ti busybox echo hello 2>&1 | grep pulling + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - --pull missing failed" && ((ret++)) + +- isula pull docker.io/library/busybox:latest +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - --pull docker.io/library/busybox:latest failed" && ((ret++)) ++ # Unable to pull image from docker.io without agent, skip this test ++ # isula pull docker.io/library/busybox:latest ++ # [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - --pull docker.io/library/busybox:latest failed" && ((ret++)) + + isula pull 3laho3y3.mirror.aliyuncs.com/library/busybox + fn_check_eq "$?" "0" "isula pull 3laho3y3.mirror.aliyuncs.com/library/busybox" +-- +2.25.1 + diff --git a/0109-modify-default-registry-mirrors-in-ci-test.patch b/0109-modify-default-registry-mirrors-in-ci-test.patch new file mode 100644 index 0000000..f0c1a98 --- /dev/null +++ b/0109-modify-default-registry-mirrors-in-ci-test.patch @@ -0,0 +1,26 @@ +From d6284e5e786e1407c2ce5ef098a39c154650bd38 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Wed, 12 Jun 2024 10:57:39 +0800 +Subject: [PATCH 109/121] modify default registry mirrors in ci test + +Signed-off-by: zhongtao +--- + CI/test_cases/container_cases/test_data/daemon.json | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/CI/test_cases/container_cases/test_data/daemon.json b/CI/test_cases/container_cases/test_data/daemon.json +index 2664c6b2..ab7d0360 100644 +--- a/CI/test_cases/container_cases/test_data/daemon.json ++++ b/CI/test_cases/container_cases/test_data/daemon.json +@@ -24,7 +24,7 @@ + "overlay2.override_kernel_check=true" + ], + "registry-mirrors": [ +- "docker.io" ++ "https://3laho3y3.mirror.aliyuncs.com" + ], + "insecure-registries": [ + ], +-- +2.25.1 + diff --git a/0110-add-timestamp-in-PodSandboxStatu-response.patch b/0110-add-timestamp-in-PodSandboxStatu-response.patch new file mode 100644 index 0000000..2d2b276 --- /dev/null +++ b/0110-add-timestamp-in-PodSandboxStatu-response.patch @@ -0,0 +1,25 @@ +From 5087d7501308660970aa9e7c12cf5be7a3d9b063 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Wed, 12 Jun 2024 15:20:17 +0000 +Subject: [PATCH 110/121] add timestamp in PodSandboxStatu response + +Signed-off-by: jikai +--- + src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +index fa726e2c..2a458a6d 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +@@ -910,6 +910,7 @@ void PodSandboxManagerService::PodSandboxStatus(const std::string &podSandboxID, + for (auto &containerStatus : containerStatuses) { + *(reply->add_containers_statuses()) = *containerStatus; + } ++ reply->set_timestamp(util_get_now_time_nanos()); + return; + } + +-- +2.25.1 + diff --git a/0111-bugfix-for-file-param-verify.patch b/0111-bugfix-for-file-param-verify.patch new file mode 100644 index 0000000..2903c4b --- /dev/null +++ b/0111-bugfix-for-file-param-verify.patch @@ -0,0 +1,75 @@ +From d0fd2c2bf87d7befaa8810a70d7eb2061664f02f Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Fri, 14 Jun 2024 09:55:28 +0800 +Subject: [PATCH 111/121] bugfix for file param verify + +Signed-off-by: zhongtao +--- + src/cmd/isula/base/create.c | 19 +++++++++++++++++++ + src/cmd/isula/images/load.c | 6 ++++++ + 2 files changed, 25 insertions(+) + +diff --git a/src/cmd/isula/base/create.c b/src/cmd/isula/base/create.c +index 543b8fd6..b04dddb5 100644 +--- a/src/cmd/isula/base/create.c ++++ b/src/cmd/isula/base/create.c +@@ -292,6 +292,12 @@ static int append_env_variables_to_conf(const char *env_file, isula_container_co + int ret = 0; + size_t file_size; + ++ if (util_dir_exists(env_file)) { ++ COMMAND_ERROR("Env file is a directory: %s", env_file); ++ ret = -1; ++ goto out; ++ } ++ + if (!util_file_exists(env_file)) { + COMMAND_ERROR("env file not exists: %s", env_file); + ret = -1; +@@ -427,6 +433,12 @@ static int append_labels_to_conf(const char *label_file, isula_container_config_ + int ret = 0; + size_t file_size; + ++ if (util_dir_exists(label_file)) { ++ COMMAND_ERROR("Label file is a directory: %s", label_file); ++ ret = -1; ++ goto out; ++ } ++ + if (!util_file_exists(label_file)) { + COMMAND_ERROR("label file not exists: %s", label_file); + ret = -1; +@@ -2357,6 +2369,13 @@ static int create_check_env_target_file(const struct client_arguments *args) + ret = -1; + goto out; + } ++ ++ if (util_dir_exists(env_path)) { ++ COMMAND_ERROR("Env target file is a directory: %s", env_path); ++ ret = -1; ++ goto out; ++ } ++ + if (!util_file_exists(env_path)) { + goto out; + } +diff --git a/src/cmd/isula/images/load.c b/src/cmd/isula/images/load.c +index 314e5d5e..cb39dee7 100644 +--- a/src/cmd/isula/images/load.c ++++ b/src/cmd/isula/images/load.c +@@ -162,6 +162,12 @@ int cmd_load_main(int argc, const char **argv) + g_cmd_load_args.file = file; + } + ++ if (util_dir_exists(g_cmd_load_args.file)) { ++ COMMAND_ERROR("Load file is a directory: %s", g_cmd_load_args.file); ++ ret = -1; ++ exit(exit_code); ++ } ++ + if (!util_file_exists(g_cmd_load_args.file)) { + COMMAND_ERROR("File %s is not exist", g_cmd_load_args.file); + exit(exit_code); +-- +2.25.1 + diff --git a/0112-bugfix-change-cni-log-info.patch b/0112-bugfix-change-cni-log-info.patch new file mode 100644 index 0000000..adc18b1 --- /dev/null +++ b/0112-bugfix-change-cni-log-info.patch @@ -0,0 +1,26 @@ +From 359a6673e01bef937adcc17f99ee94b67caca32e Mon Sep 17 00:00:00 2001 +From: liuxu +Date: Fri, 14 Jun 2024 17:12:58 +0800 +Subject: [PATCH 112/121] bugfix:change cni log info + +Signed-off-by: liuxu +--- + .../modules/network/cni_operator/libcni/invoke/libcni_exec.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.c b/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.c +index 74d6d74a..1e4a7138 100644 +--- a/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.c ++++ b/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.c +@@ -247,7 +247,7 @@ static char *env_stringify(char *(*pargs)[2], size_t len) + bool invalid_arg = (pargs == NULL || len == 0); + + if (invalid_arg) { +- ERROR("Invalid arguments"); ++ DEBUG("Empty arguments"); + return NULL; + } + +-- +2.25.1 + diff --git a/0113-move-shutdown-handle-after-init-module.patch b/0113-move-shutdown-handle-after-init-module.patch new file mode 100644 index 0000000..5ac3acc --- /dev/null +++ b/0113-move-shutdown-handle-after-init-module.patch @@ -0,0 +1,43 @@ +From 0ae6244c6bfed229a46d300888977a4967e1d718 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Wed, 19 Jun 2024 09:50:51 +0800 +Subject: [PATCH 113/121] move shutdown handle after init module + +Signed-off-by: zhongtao +--- + src/cmd/isulad/main.c | 12 +++++++----- + 1 file changed, 7 insertions(+), 5 deletions(-) + +diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c +index 3e2249d7..52ac3172 100644 +--- a/src/cmd/isulad/main.c ++++ b/src/cmd/isulad/main.c +@@ -1669,11 +1669,6 @@ static int start_daemon_threads() + { + int ret = -1; + +- if (new_shutdown_handler()) { +- ERROR("Create new shutdown handler thread failed"); +- goto out; +- } +- + if (events_module_init() != 0) { + goto out; + } +@@ -1825,6 +1820,13 @@ int main(int argc, char **argv) + goto failure; + } + ++ // after all modules are initialized, enable the shutdown handler to ++ // prevent shutdown handler from cleaning up incompletely initialized modules. ++ if (new_shutdown_handler()) { ++ ERROR("Create new shutdown handler thread failed"); ++ goto failure; ++ } ++ + #ifdef ENABLE_PLUGIN + if (start_plugin_manager()) { + ERROR("Failed to init plugin_manager"); +-- +2.25.1 + diff --git a/0114-bugfix-for-null-pointer-reference.patch b/0114-bugfix-for-null-pointer-reference.patch new file mode 100644 index 0000000..a39467e --- /dev/null +++ b/0114-bugfix-for-null-pointer-reference.patch @@ -0,0 +1,56 @@ +From 701180b53d1c52376f753b94c5cf09987ae789b3 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Tue, 18 Jun 2024 16:02:25 +0800 +Subject: [PATCH 114/121] bugfix for null pointer reference + +Signed-off-by: zhongtao +--- + src/daemon/entry/connect/grpc/grpc_service.cc | 14 ++++++++++++-- + 1 file changed, 12 insertions(+), 2 deletions(-) + +diff --git a/src/daemon/entry/connect/grpc/grpc_service.cc b/src/daemon/entry/connect/grpc/grpc_service.cc +index 1d8de922..300af082 100644 +--- a/src/daemon/entry/connect/grpc/grpc_service.cc ++++ b/src/daemon/entry/connect/grpc/grpc_service.cc +@@ -100,7 +100,9 @@ public: + { + // Wait for the server to shutdown. Note that some other thread must be + // responsible for shutting down the server for this call to ever return. +- m_server->Wait(); ++ if (m_server != nullptr) { ++ m_server->Wait(); ++ } + + // Wait for stream server to shutdown + m_criService.Wait(); +@@ -109,7 +111,9 @@ public: + void Shutdown(void) + { + // call CRI to shutdown stream server, shutdown cri first to notify events thread to exit +- m_criService.Shutdown(); ++ if (m_server != nullptr) { ++ m_server->Shutdown(); ++ } + + m_server->Shutdown(); + +@@ -242,10 +246,16 @@ int grpc_server_init(const struct service_arguments *args) + + void grpc_server_wait(void) + { ++ if (g_grpcserver == nullptr) { ++ return; ++ } + g_grpcserver->Wait(); + } + + void grpc_server_shutdown(void) + { ++ if (g_grpcserver == nullptr) { ++ return; ++ } + g_grpcserver->Shutdown(); + } +-- +2.25.1 + diff --git a/0115-bugfix-for-m_criService-shutdown.patch b/0115-bugfix-for-m_criService-shutdown.patch new file mode 100644 index 0000000..8d4d859 --- /dev/null +++ b/0115-bugfix-for-m_criService-shutdown.patch @@ -0,0 +1,33 @@ +From 93b1df1a1d3fcf6d285102f3cc1f79e6241aa393 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Thu, 4 Jul 2024 10:58:38 +0800 +Subject: [PATCH 115/121] bugfix for m_criService shutdown + +Signed-off-by: zhongtao +--- + src/daemon/entry/connect/grpc/grpc_service.cc | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/src/daemon/entry/connect/grpc/grpc_service.cc b/src/daemon/entry/connect/grpc/grpc_service.cc +index 300af082..fb5ec3cb 100644 +--- a/src/daemon/entry/connect/grpc/grpc_service.cc ++++ b/src/daemon/entry/connect/grpc/grpc_service.cc +@@ -111,12 +111,12 @@ public: + void Shutdown(void) + { + // call CRI to shutdown stream server, shutdown cri first to notify events thread to exit ++ m_criService.Shutdown(); ++ + if (m_server != nullptr) { + m_server->Shutdown(); + } +- +- m_server->Shutdown(); +- ++ + // Shutdown daemon, this operation should remove socket file. + for (const auto &address : m_socketPath) { + if (address.find(UNIX_SOCKET_PREFIX) == 0) { +-- +2.25.1 + diff --git a/0116-fix-bug-in-ci-test.patch b/0116-fix-bug-in-ci-test.patch new file mode 100644 index 0000000..181fbc7 --- /dev/null +++ b/0116-fix-bug-in-ci-test.patch @@ -0,0 +1,54 @@ +From c7cf33c432b3d9479b2fe365169d4b9a37cae8f7 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Tue, 9 Jul 2024 12:30:01 +0000 +Subject: [PATCH 116/121] fix bug in ci test + +Signed-off-by: jikai +--- + CI/test_cases/container_cases/run.sh | 6 +++--- + CI/test_cases/helpers.sh | 5 +++-- + 2 files changed, 6 insertions(+), 5 deletions(-) + +diff --git a/CI/test_cases/container_cases/run.sh b/CI/test_cases/container_cases/run.sh +index 1bfd388b..ef04b547 100755 +--- a/CI/test_cases/container_cases/run.sh ++++ b/CI/test_cases/container_cases/run.sh +@@ -26,9 +26,9 @@ source ../helpers.sh + function do_test_t() + { + tid=`isula run --runtime $1 -tid --name hostname busybox` +- chostname=`isula exec -it $tid hostname` +- clean_hostname=$(echo "$hostname" | sed 's/[\x01-\x1F\x7F]//g') +- fn_check_eq "${clean_hostname}" "${tid:0:12}" "default hostname is not id of container" ++ # should not use -it option, otherwise the hostname will containe special characters such as '$' or '\r' ++ hostname=`isula exec $tid hostname` ++ fn_check_eq "${hostname}" "${tid:0:12}" "default hostname is not id of container" + isula exec -it hostname env | grep HOSTNAME + fn_check_eq "$?" "0" "check HOSTNAME env failed" + isula stop -t 0 $tid +diff --git a/CI/test_cases/helpers.sh b/CI/test_cases/helpers.sh +index c5eba8a2..0288b4ea 100755 +--- a/CI/test_cases/helpers.sh ++++ b/CI/test_cases/helpers.sh +@@ -52,15 +52,16 @@ function cut_output_lines() { + return $retval + } + ++# use string compare to check the result + function fn_check_eq() { +- if [[ "$1" -ne "$2" ]];then ++ if [ "x$1" != "x$2" ];then + echo "$3" + TC_RET_T=$(($TC_RET_T+1)) + fi + } + + function fn_check_ne() { +- if [[ "$1" -eq "$2" ]];then ++ if [[ "x$1" == "x$2" ]];then + echo "$3" + TC_RET_T=$(($TC_RET_T+1)) + fi +-- +2.25.1 + diff --git a/0117-NRI-add-nri-head-file-and-common-func.patch b/0117-NRI-add-nri-head-file-and-common-func.patch new file mode 100644 index 0000000..61c287e --- /dev/null +++ b/0117-NRI-add-nri-head-file-and-common-func.patch @@ -0,0 +1,1051 @@ +From d21dc88cb33049ec5d9ce8e16eb1a2f9c070219b Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Tue, 30 Jul 2024 20:17:08 +0800 +Subject: [PATCH 117/121] [NRI] add nri head file and common func + +Signed-off-by: zhongtao +--- + cmake/checker.cmake | 8 ++ + cmake/options.cmake | 13 ++- + src/CMakeLists.txt | 10 +++ + src/daemon/CMakeLists.txt | 9 ++ + src/daemon/common/CMakeLists.txt | 5 ++ + src/daemon/common/nri/CMakeLists.txt | 13 +++ + src/daemon/common/nri/nri_convert.h | 36 ++++++++ + src/daemon/common/nri/nri_spec.h | 24 ++++++ + src/daemon/common/nri/nri_utils.h | 71 ++++++++++++++++ + src/daemon/config/isulad_config.h | 16 ++++ + src/daemon/modules/api/specs_api.h | 10 ++- + src/daemon/modules/spec/specs.c | 67 ++++++++++++++- + src/daemon/nri/CMakeLists.txt | 6 ++ + src/daemon/nri/nri_adaption.h | 123 +++++++++++++++++++++++++++ + src/daemon/nri/nri_helpers.h | 63 ++++++++++++++ + src/daemon/nri/nri_plugin_ops.h | 38 +++++++++ + src/daemon/nri/nri_result.h | 114 +++++++++++++++++++++++++ + src/daemon/nri/plugin.h | 95 +++++++++++++++++++++ + src/utils/cpputils/transform.cc | 16 ++++ + src/utils/cpputils/transform.h | 2 + + src/utils/cutils/utils.c | 7 ++ + src/utils/cutils/utils.h | 2 + + src/utils/tar/util_archive.c | 7 -- + 23 files changed, 744 insertions(+), 11 deletions(-) + create mode 100644 src/daemon/common/nri/CMakeLists.txt + create mode 100644 src/daemon/common/nri/nri_convert.h + create mode 100644 src/daemon/common/nri/nri_spec.h + create mode 100644 src/daemon/common/nri/nri_utils.h + create mode 100644 src/daemon/nri/CMakeLists.txt + create mode 100644 src/daemon/nri/nri_adaption.h + create mode 100644 src/daemon/nri/nri_helpers.h + create mode 100644 src/daemon/nri/nri_plugin_ops.h + create mode 100644 src/daemon/nri/nri_result.h + create mode 100644 src/daemon/nri/plugin.h + +diff --git a/cmake/checker.cmake b/cmake/checker.cmake +index 13f0fd62..b877c1c8 100644 +--- a/cmake/checker.cmake ++++ b/cmake/checker.cmake +@@ -113,6 +113,14 @@ if (ENABLE_SHIM_V2) + _CHECK(LIBSHIM_V2_LIBRARY "LIBSHIM_V2_LIBRARY-NOTFOUND" "libshim_v2.so") + endif() + ++if (ENABLE_NRI) ++ find_path(NRI_INCLUDE_DIR nri_plugin.h) ++ _CHECK(NRI_INCLUDE_DIR "NRI_INCLUDE_DIR-NOTFOUND" "nri_plugin.h") ++ ++ find_library(LIBISULA_NRI_LIBRARY isula_nri) ++ _CHECK(LIBISULA_NRI_LIBRARY "LIBISULA_NRI_LIBRARY-NOTFOUND" "libisula_nri.so") ++endif() ++ + if (OPENSSL_VERIFY) + find_path(OPENSSL_INCLUDE_DIR openssl/x509.h) + _CHECK(OPENSSL_INCLUDE_DIR "OPENSSL_INCLUDE_DIR-NOTFOUND" "openssl/x509.h") +diff --git a/cmake/options.cmake b/cmake/options.cmake +index 5b17f631..41177fe0 100644 +--- a/cmake/options.cmake ++++ b/cmake/options.cmake +@@ -59,12 +59,23 @@ if (ENABLE_SANDBOXER STREQUAL "ON") + endif() + + option(ENABLE_OOM_MONITOR "Enable oom monitor" ON) +-IF (ENABLE_OOM_MONITOR STREQUAL "ON") ++if (ENABLE_OOM_MONITOR STREQUAL "ON") + add_definitions(-DENABLE_OOM_MONITOR) + set(ENABLE_OOM_MONITOR 1) + message("${Green}-- Enable oom monitor${ColourReset}") + endif() + ++option(ENABLE_NRI "Enable NRI API" OFF) ++if (ENABLE_NRI STREQUAL "ON") ++ if (ENABLE_CRI_API_V1) ++ add_definitions(-DENABLE_NRI) ++ set(ENABLE_NRI 1) ++ message("${Green}-- Enable NRI API${ColourReset}") ++ else() ++ message("${Yellow}-- Can not enable NRI, NRI need enable CRI API V1 first ${ColourReset}") ++ endif() ++endif() ++ + option(ENABLE_SYSTEMD_NOTIFY "Enable systemd notify" ON) + if (ENABLE_SYSTEMD_NOTIFY STREQUAL "ON") + add_definitions(-DSYSTEMD_NOTIFY) +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index 48c1bad0..70a8ac91 100644 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -15,6 +15,12 @@ if (ENABLE_SHIM_V2) + ) + endif() + ++if (ENABLE_NRI) ++ list(APPEND CHECKED_INCLUDE_DIRS ++ ${NRI_INCLUDE_DIR} ++ ) ++endif() ++ + if (GRPC_CONNECTOR) + list(APPEND CHECKED_INCLUDE_DIRS + ${GRPC_INCLUDE_DIR} +@@ -168,6 +174,10 @@ if (ENABLE_SHIM_V2) + target_link_libraries(isulad ${LIBSHIM_V2_LIBRARY}) + endif() + ++if (ENABLE_NRI) ++ target_link_libraries(isulad ${LIBISULA_NRI_LIBRARY}) ++endif() ++ + if (ENABLE_EMBEDDED_IMAGE) + target_link_libraries(isulad ${SQLITE3_LIBRARY}) + endif() +diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt +index 29af3dca..801f5797 100644 +--- a/src/daemon/CMakeLists.txt ++++ b/src/daemon/CMakeLists.txt +@@ -36,6 +36,15 @@ if (ENABLE_CRI_API_V1) + list (APPEND local_daemon_incs + ${SANDBOX_INCS} + ) ++ if (ENABLE_NRI) ++ add_subdirectory(nri) ++ list (APPEND local_daemon_srcs ++ ${NRI_SRCS} ++ ) ++ list (APPEND local_daemon_incs ++ ${NRI_INCS} ++ ) ++ endif() + endif() + + set(DAEMON_SRCS +diff --git a/src/daemon/common/CMakeLists.txt b/src/daemon/common/CMakeLists.txt +index e88578dd..ffeed4b3 100644 +--- a/src/daemon/common/CMakeLists.txt ++++ b/src/daemon/common/CMakeLists.txt +@@ -10,12 +10,16 @@ if (GRPC_CONNECTOR) + endif() + + add_subdirectory(cgroup) ++if (ENABLE_NRI) ++ add_subdirectory(nri) ++endif() + + set(local_daemon_common_srcs ${daemon_common_top_srcs}) + + set(DAEMON_COMMON_SRCS + ${COMMON_CRI_SRCS} + ${COMMON_CGROUP_SRCS} ++ ${COMMON_NRI_SRCS} + ${local_daemon_common_srcs} + PARENT_SCOPE + ) +@@ -23,6 +27,7 @@ set(DAEMON_COMMON_SRCS + set(DAEMON_COMMON_INCS + ${COMMON_CRI_INCS} + ${COMMON_CGROUP_INCS} ++ ${COMMON_NRI_INCS} + ${CMAKE_CURRENT_SOURCE_DIR} + PARENT_SCOPE + ) +diff --git a/src/daemon/common/nri/CMakeLists.txt b/src/daemon/common/nri/CMakeLists.txt +new file mode 100644 +index 00000000..512ba694 +--- /dev/null ++++ b/src/daemon/common/nri/CMakeLists.txt +@@ -0,0 +1,13 @@ ++# get current directory sources files ++aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_common_nri_srcs) ++set(local_common_nri_incs ${CMAKE_CURRENT_SOURCE_DIR}) ++ ++set(COMMON_NRI_SRCS ++ ${local_common_nri_srcs} ++ PARENT_SCOPE ++) ++ ++set(COMMON_NRI_INCS ++ ${local_common_nri_incs} ++ PARENT_SCOPE ++) +diff --git a/src/daemon/common/nri/nri_convert.h b/src/daemon/common/nri/nri_convert.h +new file mode 100644 +index 00000000..883f7c41 +--- /dev/null ++++ b/src/daemon/common/nri/nri_convert.h +@@ -0,0 +1,36 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-03-16 ++ * Description: provide nri convert functions ++ *********************************************************************************/ ++#ifndef DAEMON_COMMON_NRI_NRI_CONVERT_H ++#define DAEMON_COMMON_NRI_NRI_CONVERT_H ++#include ++#include ++#include ++#include ++ ++#include "isula_libutils/nri_pod_sandbox.h" ++#include "isula_libutils/nri_container.h" ++#include "isula_libutils/nri_mount.h" ++#include "isula_libutils/nri_linux_resources.h" ++ ++#include "sandbox.h" ++#include "api_v1.pb.h" ++ ++auto PodSandboxToNRI(const std::shared_ptr &sandbox, nri_pod_sandbox *pod) -> bool; ++auto ContainerToNRIByConConfig(const runtime::v1::ContainerConfig &containerConfig, nri_container *con) -> bool; ++auto ContainerToNRIByID(const std::string &id, nri_container *con) -> bool; ++auto PodSandboxesToNRI(const std::vector> &arrs, nri_pod_sandbox **pod, int pod_len) -> bool; ++ ++auto LinuxResourcesFromNRI(const nri_linux_resources *src, runtime::v1::LinuxContainerResources &resources) -> bool; ++#endif // DAEMON_COMMON_NRI_NRI_CONVERT_H +diff --git a/src/daemon/common/nri/nri_spec.h b/src/daemon/common/nri/nri_spec.h +new file mode 100644 +index 00000000..e7c5035d +--- /dev/null ++++ b/src/daemon/common/nri/nri_spec.h +@@ -0,0 +1,24 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-07-17 ++ * Description: provide nri oci functions ++ *********************************************************************************/ ++ ++#ifndef DAEMON_COMMON_NRI_NRI_SPEC_H ++#define DAEMON_COMMON_NRI_NRI_SPEC_H ++ ++#include ++#include ++ ++int nri_adjust_oci_spec(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec); ++ ++#endif // DAEMON_COMMON_NRI_NRI_SPEC_H +\ No newline at end of file +diff --git a/src/daemon/common/nri/nri_utils.h b/src/daemon/common/nri/nri_utils.h +new file mode 100644 +index 00000000..3aa50ae4 +--- /dev/null ++++ b/src/daemon/common/nri/nri_utils.h +@@ -0,0 +1,71 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-07-17 ++ * Description: provide nri utils functions ++ *********************************************************************************/ ++#ifndef DAEMON_COMMON_NRI_NRI_UTILS_H ++#define DAEMON_COMMON_NRI_NRI_UTILS_H ++ ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++typedef enum { ++ UNKNOWN = 0, ++ RUN_POD_SANDBOX = 1, ++ STOP_POD_SANDBOX = 2, ++ REMOVE_POD_SANDBOX = 3, ++ CREATE_CONTAINER = 4, ++ POST_CREATE_CONTAINER = 5, ++ START_CONTAINER = 6, ++ POST_START_CONTAINER = 7, ++ UPDATE_CONTAINER = 8, ++ POST_UPDATE_CONTAINER = 9, ++ STOP_CONTAINER = 10, ++ REMOVE_CONTAINER = 11, ++ LAST = 12, ++} NRI_Event; ++ ++bool copy_nri_mount(const nri_mount *src, nri_mount **dest); ++bool copy_nri_key_value(const nri_key_value *src, nri_key_value **dest); ++bool copy_nri_hook(const nri_hook *src, nri_hook **dest); ++bool copy_nri_posix_rlimit(const nri_posix_rlimit *src, nri_posix_rlimit **dest); ++bool copy_nri_linux_resources(const nri_linux_resources *src, nri_linux_resources **dest); ++bool copy_nri_linux_cpu(const nri_linux_cpu *src, nri_linux_cpu **dest); ++ ++bool is_marked_for_removal(const char* key, char **out); ++ ++bool realloc_and_copy_nri_hooks(nri_hook **targetHooks, size_t targetSize, const nri_hook **sourceHooks, size_t sourceLen); ++ ++bool init_nri_container_adjust(nri_container_adjustment **adjust); ++bool init_nri_container_update(nri_container_update **update, const char *id, uint8_t ignore_failure); ++bool init_nri_linux_resources(nri_linux_resources **resources); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif // DAEMON_COMMON_NRI_NRI_UTILS_H +\ No newline at end of file +diff --git a/src/daemon/config/isulad_config.h b/src/daemon/config/isulad_config.h +index f29cd564..c57ca8d6 100644 +--- a/src/daemon/config/isulad_config.h ++++ b/src/daemon/config/isulad_config.h +@@ -40,6 +40,22 @@ struct isulad_conf { + #define DEFAULT_SANDBOXER_NAME "shim" + char *conf_get_sandbox_rootpath(void); + char *conf_get_sandbox_statepath(void); ++ ++#ifdef ENABLE_NRI ++#define DEFAULT_SOCKET_PATH "/var/run/nri/nri.sock" ++#define DEFAULT_MUX_SOCKET_PATH "/var/run/nri/nri_mux.sock" ++#define DEFAULT_PLUGIN_PATH "/opt/nri/plugins" ++#define DEFAULT_PLUGIN_CONFIG_PATH "/etc/nri/conf.d" ++#define DEFAULT_PLUGIN_REGISTRY_TIMEOUT 5 ++#define DEFAULT_PLUGIN_REQUST_TIMEOUT 2 ++bool conf_get_nri_support(void); ++bool conf_get_nri_external_support(void); ++char *conf_get_nri_plugin_config_path(void); ++char *conf_get_nri_plugin_path(void); ++char *conf_get_socket_path(void); ++uint64_t conf_get_nri_plugin_registration_timeout(void); ++uint64_t conf_get_nri_plugin_requst_timeout(void); ++#endif + #endif + + char *conf_get_isulad_pidfile(void); +diff --git a/src/daemon/modules/api/specs_api.h b/src/daemon/modules/api/specs_api.h +index 0999836b..6a1cd776 100644 +--- a/src/daemon/modules/api/specs_api.h ++++ b/src/daemon/modules/api/specs_api.h +@@ -58,6 +58,9 @@ int spec_module_init(void); + #ifdef ENABLE_CDI + int defs_process_add_multiple_env(defs_process *dp, const char **envs, size_t env_len); + int spec_add_multiple_process_env(oci_runtime_spec *oci_spec, const char **envs, size_t env_len); ++#endif/* ENABLE_CDI */ ++ ++#if defined (ENABLE_CDI) || defined(ENABLE_NRI) + int spec_add_device(oci_runtime_spec *oci_spec, defs_device *device); + int spec_add_linux_resources_device(oci_runtime_spec *oci_spec, bool allow, const char *dev_type, + int64_t major, int64_t minor, const char *access); +@@ -66,7 +69,12 @@ int spec_add_mount(oci_runtime_spec *oci_spec, defs_mount *mnt); + int spec_add_prestart_hook(oci_runtime_spec *oci_spec, defs_hook *prestart_hook); + int spec_add_poststart_hook(oci_runtime_spec *oci_spec, defs_hook *poststart_hook); + int spec_add_poststop_hook(oci_runtime_spec *oci_spec, defs_hook *poststop_hook); +-#endif /* ENABLE_CDI */ ++#endif /* ENABLE_CDI || ENABLE_NRI */ ++ ++#ifdef ENABLE_NRI ++int spec_add_linux_resources_hugepage_limit(oci_runtime_spec *oci_spec, const char *page_size, uint64_t limit); ++int spec_add_linux_resources_rlimit(oci_runtime_spec *oci_spec, const char *type, uint64_t hard, uint64_t soft); ++#endif /* ENABLE_NRI */ + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index f0538e26..1fd9e5a8 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -2608,7 +2608,7 @@ int spec_module_init(void) + static int add_env(defs_process *dp, const char *env, const char *key) + { + size_t i; +- ++ + for (i = 0; i < dp->env_len; i++) { + __isula_auto_free char *oci_key = NULL; + if (util_valid_split_env(dp->env[i], &oci_key, NULL) < 0) { +@@ -2684,7 +2684,9 @@ int spec_add_multiple_process_env(oci_runtime_spec *oci_spec, const char **envs, + + return ret; + } ++#endif /* ENABLE_CDI */ + ++#if defined (ENABLE_CDI) || defined(ENABLE_NRI) + int spec_add_device(oci_runtime_spec *oci_spec, defs_device *device) + { + int ret = 0; +@@ -2817,4 +2819,65 @@ SPEC_ADD_HOOKS_ITEM_DEF(prestart) + SPEC_ADD_HOOKS_ITEM_DEF(poststart) + SPEC_ADD_HOOKS_ITEM_DEF(poststop) + +-#endif /* ENABLE_CDI */ +\ No newline at end of file ++#endif /* ENABLE_CDI || ENABLE_NRI */ ++ ++#ifdef ENABLE_NRI ++int spec_add_linux_resources_hugepage_limit(oci_runtime_spec *oci_spec, const char *page_size, uint64_t limit) ++{ ++ int ret = 0; ++ defs_resources_hugepage_limits_element *hugepage_limit = NULL; ++ ++ ret = make_sure_oci_spec_linux_resources(oci_spec); ++ if (ret < 0) { ++ return -1; ++ } ++ ++ hugepage_limit = util_common_calloc_s(sizeof(*hugepage_limit)); ++ if (hugepage_limit == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ hugepage_limit->page_size = util_strdup_s(page_size); ++ hugepage_limit->limit = limit; ++ ++ if (util_mem_realloc((void **)&oci_spec->linux->resources->hugepage_limits, ++ (oci_spec->linux->resources->hugepage_limits_len + 1) * sizeof(defs_resources_hugepage_limits_element *), ++ (void *)oci_spec->linux->resources->hugepage_limits, ++ oci_spec->linux->resources->hugepage_limits_len * sizeof(defs_resources_hugepage_limits_element *)) != 0) { ++ ERROR("Out of memory"); ++ free_defs_resources_hugepage_limits_element(hugepage_limit); ++ return -1; ++ } ++ oci_spec->linux->resources->hugepage_limits[oci_spec->linux->resources->hugepage_limits_len] = hugepage_limit; ++ oci_spec->linux->resources->hugepage_limits_len++; ++ ++ return 0; ++} ++ ++int spec_add_linux_resources_rlimit(oci_runtime_spec *oci_spec, const char *type, uint64_t hard, uint64_t soft) ++{ ++ int ret = 0; ++ defs_process_rlimits_element *rlimit = NULL; ++ ++ ret = make_sure_oci_spec_linux_resources(oci_spec); ++ if (ret < 0) { ++ return -1; ++ } ++ ++ rlimit = util_common_calloc_s(sizeof(*rlimit)); ++ if (rlimit == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ rlimit->type = util_strdup_s(type); ++ rlimit->hard = hard; ++ rlimit->soft = soft; ++ ++ if (do_merge_one_ulimit_override(oci_spec, rlimit) != 0) { ++ ERROR("Failed to merge one nri ulimit to oci spec"); ++ return -1; ++ } ++ ++ return 0; ++} ++#endif /* ENABLE_NRI */ +\ No newline at end of file +diff --git a/src/daemon/nri/CMakeLists.txt b/src/daemon/nri/CMakeLists.txt +new file mode 100644 +index 00000000..361d79f8 +--- /dev/null ++++ b/src/daemon/nri/CMakeLists.txt +@@ -0,0 +1,6 @@ ++# get current directory sources files ++aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} nri_top_srcs) ++ ++set(NRI_INCS ${CMAKE_CURRENT_SOURCE_DIR} PARENT_SCOPE) ++ ++set(NRI_SRCS ${nri_top_srcs} PARENT_SCOPE) +diff --git a/src/daemon/nri/nri_adaption.h b/src/daemon/nri/nri_adaption.h +new file mode 100644 +index 00000000..7f0640df +--- /dev/null ++++ b/src/daemon/nri/nri_adaption.h +@@ -0,0 +1,123 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-03-15 ++ * Description: provide plugin manager(NRI adaption) class definition ++ *********************************************************************************/ ++ ++#ifndef DAEMON_NRI_PLUGIN_NRI_ADAPTION_H ++#define DAEMON_NRI_PLUGIN_NRI_ADAPTION_H ++ ++// #include "read_write_lock.h" ++#include ++#include ++ ++#include "plugin.h" ++#include "sandbox.h" ++#include "v1_cri_container_manager_service.h" ++ ++const std::string PluginNameEnv = "NRI_PLUGIN_NAME"; ++const std::string PluginIdxEnv = "NRI_PLUGIN_IDX"; ++const std::string PluginSocketEnv = "NRI_PLUGIN_SOCKET"; ++ ++struct nri_plugin_exec_args_t { ++ const char *workdir; ++ const char *cmd; ++ const char *name; ++ const char *index; ++ const uint32_t sockFd; ++}; ++ ++class NRIAdaptation { ++public: ++ // Singleton ++ static NRIAdaptation *GetInstance() noexcept; ++ ++ // initialize value ++ auto Init(Errors &error) -> bool; ++ ++ auto GetSockpath(std::vector &paths) -> bool; ++ ++ // Stop plugins. ++ auto StopPlugins() -> bool; ++ ++ void RemoveClosedPlugins(); ++ ++ auto GetPluginByIndex(const std::string &index) -> std::shared_ptr; ++ void AddPluginByIndex(const std::string &index, std::shared_ptr plugin); ++ void RemovePluginByIndex(const std::string &index); ++ ++ auto RunPodSandbox(std::shared_ptr sandbox, Errors &error) ->bool; ++ auto StopPodSandbox(std::shared_ptr sandbox, Errors &error) ->bool; ++ auto RemovePodSandbox(std::shared_ptr sandbox, Errors &error) ->bool; ++ auto CreateContainer(std::shared_ptr sandbox, const std::string &conId, const runtime::v1::ContainerConfig &containerConfig, nri_container_adjustment **adjust, Errors &error) -> bool; ++ auto PostCreateContainer(const std::string &conId, Errors &error) ->bool; ++ auto UndoCreateContainer(std::shared_ptr sandbox, const std::string &conId, Errors &error) -> bool; ++ auto StartContainer(const std::string &conId, Errors &error) ->bool; ++ auto PostStartContainer(const std::string &conId, Errors &error) ->bool; ++ auto UpdateContainer(const std::string &conId, Errors &error) ->bool; ++ auto PostUpdateContainer(const std::string &conId, Errors &error) ->bool; ++ auto StopContainer(const std::string &conId, Errors &error) ->bool; ++ auto RemoveContainer(const std::string &conId, Errors &error) ->bool; ++ auto StateChange(nri_state_change_event *evt, Errors &error) ->bool; ++ auto updateContainers(const nri_update_containers_request *req, nri_update_containers_response **resp) ->bool; ++ ++ auto NewExternalPlugin(int fd) -> bool; ++private: ++ NRIAdaptation() = default; ++ NRIAdaptation(const NRIAdaptation &other) = delete; ++ NRIAdaptation &operator=(const NRIAdaptation &) = delete; ++ virtual ~NRIAdaptation(); ++ ++ auto StartPlugin() -> bool; ++ auto NewLaunchedPlugin(const std::shared_ptr &) -> bool; ++ auto DiscoverPlugins(std::map> &map) -> bool; ++ // Synchronizing NRI (plugin) with current runtime state ++ auto SyncPlugin() -> bool; ++ ++ auto SortPlugins() -> bool; ++ void GetClosedPlugins(std::vector &closedPlugin); ++ ++ auto IsSupport() -> bool; ++ ++ auto ApplyUpdates(const std::vector &update, std::vector &failed, bool getFailed, ++ Errors &error) -> bool; ++ ++ auto NRIPodSandbox(const std::shared_ptr &sandbox, Errors& error) -> std::unique_ptr>; ++ auto NRIContainerByConConfig(const std::shared_ptr &sandbox, const runtime::v1::ContainerConfig &containerConfig, Errors& error) -> std::unique_ptr>; ++ auto NRIContainerByID(const std::string &id, Errors& error) -> std::unique_ptr>; ++ ++ auto GetNRIPluginConfigPath(void) -> std::string; ++ auto GetNRIPluginPath(void) -> std::string; ++ auto GetNRISockPath(void) -> std::string; ++private: ++ RWMutex m_mutex; ++ static std::atomic m_instance; ++ bool m_support; ++ bool m_external_support; ++ std::string m_version; ++ std::string m_sock_path; ++ std::string m_pluginConfigPath; ++ std::string m_pluginPath; ++ std::vector m_socketPathArr; ++ std::string m_disableConnections; ++ // id --> NRIPlugin map ++ std::map> m_storeMap; ++ // TODO:plugin monitor thread id?? ++ // shutdown() to clean resource ++ // init to create thread ++ // todo: if Singleton? ++ std::unique_ptr m_containerManager; ++ int64_t m_plugin_registration_timeout; ++ int64_t m_plugin_requst_timeout; ++}; ++ ++#endif // DAEMON_NRI_PLUGIN_NRI_ADAPTION_H +\ No newline at end of file +diff --git a/src/daemon/nri/nri_helpers.h b/src/daemon/nri/nri_helpers.h +new file mode 100644 +index 00000000..06ee8419 +--- /dev/null ++++ b/src/daemon/nri/nri_helpers.h +@@ -0,0 +1,63 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2017-2019. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-07-13 ++ * Description: provide nri helpers functions ++ *********************************************************************************/ ++#ifndef DAEMON_NRI_PLUGIN_NRI_HELPERS_H ++#define DAEMON_NRI_PLUGIN_NRI_HELPERS_H ++ ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++ ++#include "errors.h" ++ ++namespace NRIHelpers { ++std::string MarkForRemoval(const std::string &key); ++ ++auto GetPluginConfig(std::string &idx, std::string &name, std::string &config) -> bool; ++ ++std::string GenerateRandomExternalName(void); ++ ++bool CheckPluginIndex(const std::string &idx); ++ ++template ++void freeArray(T ** &arr, int size) ++{ ++ if (arr == NULL) { ++ return; ++ } ++ ++ for (int i = 0; i < size; i++) { ++ if (arr[i] == NULL) { ++ return; ++ } ++ free(arr[i]); ++ } ++ ++ free(arr); ++ arr = NULL; ++} ++}; // namespace NRIHelpers ++ ++#endif // DAEMON_NRI_PLUGIN_NRI_HELPERS_H +diff --git a/src/daemon/nri/nri_plugin_ops.h b/src/daemon/nri/nri_plugin_ops.h +new file mode 100644 +index 00000000..37d437d2 +--- /dev/null ++++ b/src/daemon/nri/nri_plugin_ops.h +@@ -0,0 +1,38 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtaoo ++ * Create: 2024-03-26 ++ * Description: provide nri plugin api definition ++ ******************************************************************************/ ++ ++#ifndef DAEMON_NRI_PLUGIN_OPS_H ++#define DAEMON_NRI_PLUGIN_OPS_H ++ ++#include ++#include ++#include ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++bool nri_adaption_init(void); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++int nri_update_containers(const nri_update_containers_request *request, nri_update_containers_response **response); ++int nri_registry_containers(const nri_register_plugin_request *request); ++ ++int nri_external_plugin_connect(int fd); ++ ++#endif // DAEMON_NRI_PLUGIN_OPS_H +diff --git a/src/daemon/nri/nri_result.h b/src/daemon/nri/nri_result.h +new file mode 100644 +index 00000000..f2896ea0 +--- /dev/null ++++ b/src/daemon/nri/nri_result.h +@@ -0,0 +1,114 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-06-29 ++ * Description: provide nri result definition ++ *********************************************************************************/ ++ ++#ifndef DAEMON_NRI_PLUGIN_NRI_RESULT_H ++#define DAEMON_NRI_PLUGIN_NRI_RESULT_H ++ ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "api_v1.pb.h" ++#include "nri_helpers.h" ++#include "nri_utils.h" ++ ++using EventMask = std::int32_t; ++ ++const EventMask ValidEvents = (1 << (LAST - 1)) - 1; ++ ++struct owners { ++ std::map annotations; ++ std::map mounts; ++ std::map devices; ++ std::map env; ++ std::string memLimit; ++ std::string memReservation; ++ std::string memSwapLimit; ++ std::string memKernelLimit; ++ std::string memTCPLimit; ++ std::string memSwappiness; ++ std::string memDisableOomKiller; ++ std::string memUseHierarchy; ++ std::string cpuShares; ++ std::string cpuQuota; ++ std::string cpuPeriod; ++ std::string cpuRealtimeRuntime; ++ std::string cpuRealtimePeriod; ++ std::string cpusetCpus; ++ std::string cpusetMems; ++ std::map hugepageLimits; ++ std::string blockioClass; ++ std::string rdtClass; ++ std::map unified; ++ std::string cgroupsPath; ++ std::map rlimits; ++}; ++ ++struct resultReply { ++ nri_container_adjustment* adjust; ++ std::vector update; ++}; ++ ++using resultOwners = std::map; ++ ++class pluginResult { ++public: ++ pluginResult() = default; ++ ++ ~pluginResult() = default; ++ ++ auto InitByConId(std::string conId) -> bool; ++ auto InitByUpdateReq(nri_update_container_request *req) -> bool; ++ ++ auto GetReplyUpdate() -> std::vector; ++ auto GetReplyAdjust() -> nri_container_adjustment *; ++ ++ auto Apply(int32_t event, nri_container_adjustment *adjust, nri_container_update **update, size_t update_len, ++ const std::string &plugin) -> bool; ++ auto Update(nri_container_update **updates, size_t update_len, const std::string &plugin) -> bool; ++ ++private: ++ auto GetContainerUpdate(nri_container_update *update, const std::string &plugin, nri_container_update **out) -> bool; ++ auto UpdateResources(nri_container_update *reply, nri_container_update *u, const std::string &plugin) -> bool; ++ ++ auto InitReply(void) -> bool; ++ ++ auto Adjust(nri_container_adjustment *adjust, const std::string &plugin) -> bool; ++ ++ auto AdjustAnnotations(json_map_string_string *annos, const std::string &plugin) -> bool; ++ auto AdjustMounts(nri_mount **mounts, size_t mounts_size, const std::string &plugin) -> bool; ++ auto AdjustEnv(nri_key_value **envs, size_t envs_size, const std::string &plugin) -> bool; ++ auto AdjustHooks(nri_hooks *hooks, const std::string &plugin) -> bool; ++ auto AdjustDevices(nri_linux_device **devices, size_t devices_size, const std::string &plugin) -> bool; ++ auto AdjustResources(nri_linux_resources *resources, const std::string &plugin) -> bool; ++ bool ClaimAndCopyResources(nri_linux_resources *src, std::string &id, const std::string &plugin, ++ nri_linux_resources *dest); ++ auto AdjustCgroupsPath(char *path, const std::string &plugin) -> bool; ++ auto AdjustRlimits(nri_posix_rlimit **rlimits, size_t rlimits_len, const std::string &plugin) -> bool; ++ ++private: ++ std::string m_conId; ++ nri_linux_resources *m_update_req; ++ resultReply m_reply; ++ std::map m_updates; ++ resultOwners m_owners; ++}; ++ ++#endif +\ No newline at end of file +diff --git a/src/daemon/nri/plugin.h b/src/daemon/nri/plugin.h +new file mode 100644 +index 00000000..f60a9b3d +--- /dev/null ++++ b/src/daemon/nri/plugin.h +@@ -0,0 +1,95 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-03-15 ++ * Description: provide plugin class definition ++ *********************************************************************************/ ++ ++#ifndef DAEMON_NRI_PLUGIN_PLUGIN_H ++#define DAEMON_NRI_PLUGIN_PLUGIN_H ++ ++#include ++ ++#include ++#include ++#include ++ ++#include "errors.h" ++#include "read_write_lock.h" ++#include "nri_result.h" ++ ++const std::string NRIRruntime = "v2"; ++const std::string NRIVersion = "2.0.0-beta.2+unknown"; ++ ++class NRIPlugin { ++public: ++ // init client conn ++ NRIPlugin(std::string &idx, std::string &name, std::string &config); ++ NRIPlugin(int fd, std::string &name); ++ // todo: close client conn ?? or single close func? ++ virtual ~NRIPlugin() = default; ++ // wait for plugin to register, then configure it. ++ auto Start(int64_t registry_timeout, int64_t request_timeout) -> bool; ++ // close a plugin shutting down its multiplexed ttrpc connections. ++ auto Close(void) -> bool; ++ // stop a plugin (if it was launched by us) ++ auto Stop(void) -> bool; ++ ++ // Name returns a string indentication for the plugin. ++ auto GetName(void) -> const std::string &; ++ auto GetIndex(void) -> const std::string &; ++ auto GetPeerSockFd(void) -> uint32_t; ++ auto GetQualifiedName(void) -> std::string; ++ ++ void SetReady(void); ++ void SetPid(int pid); ++ ++ auto IsClose(void) -> bool; ++ ++ auto CreateSocketPair(void) -> bool; ++ ++ auto Configure(Errors &error) -> bool; ++ // Only called in external plugin scenario ++ auto Synchronize(std::vector> pods, ++ std::vector> &containers, nri_container_update ***update, size_t update_len, ++ Errors &error) -> bool; ++ auto CreateContainer(nri_create_container_request *req, nri_create_container_response **resp, Errors &error) -> bool; ++ auto UpdateContainer(nri_update_container_request *req, nri_update_container_response **resp, Errors &error) -> bool; ++ auto StopContainer(nri_stop_container_request *req, nri_stop_container_response **resp, Errors &error) -> bool; ++ auto StateChange(nri_state_change_event *evt, Errors &error) -> bool; ++ ++private: ++ auto Connect(int64_t timeout) -> bool; ++ ++ auto WaitForReady(int64_t timeout) -> bool; ++ auto IsSetEvent(EventMask e) -> bool; ++ ++private: ++ RWMutex m_mutex; ++ bool m_external; ++ std::string m_idx; ++ std::string m_name; ++ std::string m_config; ++ int m_pid; ++ std::string m_cmd; ++ std::vector m_sockFds; ++ std::string m_localFileName; ++ std::string m_peerFileName; ++ // TODO:zhontao monitor? ++ bool m_closed; ++ std::mutex m_readyMutex; ++ bool m_ready; ++ std::condition_variable m_condition; ++ EventMask m_events; ++}; ++ ++ ++#endif // DAEMON_NRI_PLUGIN_PLUGIN_H +\ No newline at end of file +diff --git a/src/utils/cpputils/transform.cc b/src/utils/cpputils/transform.cc +index 74c178a9..51c154fb 100644 +--- a/src/utils/cpputils/transform.cc ++++ b/src/utils/cpputils/transform.cc +@@ -89,4 +89,20 @@ void CharArrayToStringVector(const char **src, size_t len, std::vector &ptrs) -> char ** ++{ ++ size_t len = ptrs.size(); ++ char **result = (char **)util_smart_calloc_s(sizeof(char *), (len + 1)); ++ if (result == nullptr) { ++ return nullptr; ++ } ++ size_t i {}; ++ for (const auto &it : ptrs) { ++ result[i++] = util_strdup_s(it.c_str()); ++ } ++ ++ return result; ++} ++ + } // namespace Transform +diff --git a/src/utils/cpputils/transform.h b/src/utils/cpputils/transform.h +index 476f39a6..57c58d9e 100644 +--- a/src/utils/cpputils/transform.h ++++ b/src/utils/cpputils/transform.h +@@ -35,6 +35,8 @@ auto StringVectorToCharArray(std::vector &strVec) -> char **; + + void CharArrayToStringVector(const char **src, size_t len, std::vector &dest); + ++auto RepeatedPtrFieldToCharArray(const google::protobuf::RepeatedPtrField &ptrs) -> char **; ++ + }; // namespace Transform + + #endif // UTILS_CPPUTILS_TRANSFORM_H +\ No newline at end of file +diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c +index 9a33f935..f81a9141 100644 +--- a/src/utils/cutils/utils.c ++++ b/src/utils/cutils/utils.c +@@ -1699,3 +1699,10 @@ int util_chown_for_shm(const char *shm_path, const char *user_remap) + + return 0; + } ++ ++void set_child_process_pdeathsig(void) ++{ ++ if (prctl(PR_SET_PDEATHSIG, SIGKILL) < 0) { ++ SYSERROR("Failed to set child process pdeathsig"); ++ } ++} +\ No newline at end of file +diff --git a/src/utils/cutils/utils.h b/src/utils/cutils/utils.h +index ce0ca703..76684ed3 100644 +--- a/src/utils/cutils/utils.h ++++ b/src/utils/cutils/utils.h +@@ -409,6 +409,8 @@ int util_create_shm_path(const char *spath, const int64_t shm_size); + + int util_chown_for_shm(const char *shm_path, const char *user_remap); + ++void set_child_process_pdeathsig(void); ++ + /** + * retry_cnt: max count of call cb; + * interval_us: how many us to sleep, after call cb; +diff --git a/src/utils/tar/util_archive.c b/src/utils/tar/util_archive.c +index 204dab83..985e0f16 100644 +--- a/src/utils/tar/util_archive.c ++++ b/src/utils/tar/util_archive.c +@@ -812,13 +812,6 @@ static void close_archive_pipes_fd(int *pipes, size_t pipe_size) + } + } + +-static void set_child_process_pdeathsig(void) +-{ +- if (prctl(PR_SET_PDEATHSIG, SIGKILL) < 0) { +- SYSERROR("Failed to set child process pdeathsig"); +- } +-} +- + int archive_unpack(const struct io_read_wrapper *content, const char *dstdir, const struct archive_options *options, + const char *root_dir, char **errmsg) + { +-- +2.25.1 + diff --git a/0118-skip-calling-cni-plugin-cleanup-when-network-namespa.patch b/0118-skip-calling-cni-plugin-cleanup-when-network-namespa.patch new file mode 100644 index 0000000..4b33de5 --- /dev/null +++ b/0118-skip-calling-cni-plugin-cleanup-when-network-namespa.patch @@ -0,0 +1,51 @@ +From ee720f966fdf14a99b8ebc685f3948bb8b29ba73 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Tue, 13 Aug 2024 10:56:44 +0800 +Subject: [PATCH 118/121] skip calling cni plugin cleanup when network + namespace is not mounted + +Signed-off-by: zhongtao +--- + .../entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc | 7 +++++++ + .../entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc | 7 +++++++ + 2 files changed, 14 insertions(+) + +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +index 2a458a6d..77faf48a 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +@@ -435,6 +435,13 @@ void PodSandboxManagerService::ClearCniNetwork(const std::shared_ptrGetSandboxConfig(); + std::map stdAnnos; + CRIHelpers::ProtobufAnnoMapToStd(config.annotations(), stdAnnos); +diff --git a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +index bc3f4031..5590827e 100644 +--- a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +@@ -848,6 +848,13 @@ auto PodSandboxManagerService::ClearCniNetwork(const std::string &realSandboxID, + goto cleanup; + } + ++ // If the network namespace is not mounted, the network has been cleaned up ++ // and there is no need to call the cni plugin. ++ if (!util_detect_mounted(netnsPath.c_str())) { ++ WARN("Network namespace %s not exist", netnsPath.c_str()); ++ goto cleanup; ++ } ++ + stdAnnos.insert(std::pair(CRIHelpers::Constants::POD_SANDBOX_KEY, netnsPath)); + pluginErr.Clear(); + m_pluginManager->TearDownPod(ns, name, Network::DEFAULT_NETWORK_INTERFACE_NAME, realSandboxID, stdAnnos, +-- +2.25.1 + diff --git a/0119-nri-add-convert-and-utils-impl-for-nri.patch b/0119-nri-add-convert-and-utils-impl-for-nri.patch new file mode 100644 index 0000000..ba76c2e --- /dev/null +++ b/0119-nri-add-convert-and-utils-impl-for-nri.patch @@ -0,0 +1,2254 @@ +From c0d4b523c24e88b8c70cd3b121a46b7b3c841c17 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Tue, 13 Aug 2024 20:33:28 +0800 +Subject: [PATCH 119/121] [nri] add convert and utils impl for nri + +Signed-off-by: zhongtao +--- + src/daemon/common/nri/nri_convert.cc | 539 ++++++++++++++++++++++++ + src/daemon/common/nri/nri_convert.h | 9 +- + src/daemon/common/nri/nri_spec.c | 602 +++++++++++++++++++++++++++ + src/daemon/common/nri/nri_utils.c | 520 +++++++++++++++++++++++ + src/daemon/common/nri/nri_utils.h | 5 +- + src/daemon/config/isulad_config.c | 178 ++++++++ + src/daemon/modules/api/specs_api.h | 5 + + src/daemon/modules/spec/specs.c | 32 +- + src/daemon/nri/nri_adaption.h | 46 +- + src/daemon/nri/nri_helpers.cc | 93 +++++ + src/daemon/nri/nri_helpers.h | 2 +- + src/utils/cpputils/transform.cc | 2 +- + 12 files changed, 2002 insertions(+), 31 deletions(-) + create mode 100644 src/daemon/common/nri/nri_convert.cc + create mode 100644 src/daemon/common/nri/nri_spec.c + create mode 100644 src/daemon/common/nri/nri_utils.c + create mode 100644 src/daemon/nri/nri_helpers.cc + +diff --git a/src/daemon/common/nri/nri_convert.cc b/src/daemon/common/nri/nri_convert.cc +new file mode 100644 +index 00000000..7cce64ec +--- /dev/null ++++ b/src/daemon/common/nri/nri_convert.cc +@@ -0,0 +1,539 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-03-16 ++ * Description: provide nri convert functions ++ *********************************************************************************/ ++ ++#include "nri_convert.h" ++ ++#include "container_api.h" ++#include "v1_cri_helpers.h" ++#include "path.h" ++#include "transform.h" ++#include "nri_utils.h" ++ ++static int64_t DefaultOOMScoreAdj = 0; ++ ++static bool NRILinuxCpuFromCRI(const runtime::v1::LinuxContainerResources &config, nri_linux_cpu &cpu) ++{ ++ if (!config.cpuset_cpus().empty()) { ++ cpu.cpus = util_strdup_s(config.cpuset_cpus().c_str()); ++ } ++ ++ if (!config.cpuset_mems().empty()) { ++ cpu.mems = util_strdup_s(config.cpuset_mems().c_str()); ++ } ++ ++ cpu.period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if (cpu.period == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(cpu.period) = config.cpu_period(); ++ ++ cpu.quota = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (cpu.quota == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(cpu.quota) = config.cpu_quota(); ++ ++ cpu.shares = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if (cpu.shares == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(cpu.shares) = config.cpu_shares(); ++ ++ // consistent with other container engines, ++ // not obtained cpu.realtime_period & cpu.realtime_runtime ++ return true; ++} ++ ++static bool NRILinuxMemoryFromCRI(const runtime::v1::LinuxContainerResources &config, nri_linux_memory &memory) ++{ ++ memory.limit = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (memory.limit == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(memory.limit) = config.memory_limit_in_bytes(); ++ ++ // consistent with other container engines, ++ // not obtained other memory info ++ ++ return true; ++} ++ ++static bool NRIHugePageLimitFromCRI(const runtime::v1::LinuxContainerResources &config, nri_linux_resources &resources) ++{ ++ int i; ++ nri_hugepage_limit *tmp = nullptr; ++ ++ if (config.hugepage_limits_size() == 0) { ++ return true; ++ } ++ ++ resources.hugepage_limits = (nri_hugepage_limit **)util_smart_calloc_s(sizeof(nri_hugepage_limit *), ++ config.hugepage_limits_size()); ++ if (resources.hugepage_limits == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ for (i = 0; i < config.hugepage_limits_size(); i++) { ++ tmp = (nri_hugepage_limit *)util_common_calloc_s(sizeof(nri_hugepage_limit)); ++ if (tmp == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ tmp->page_size = util_strdup_s(config.hugepage_limits(i).page_size().c_str()); ++ tmp->limit = config.hugepage_limits(i).limit(); ++ resources.hugepage_limits[i] = tmp; ++ resources.hugepage_limits_len++; ++ tmp = nullptr; ++ } ++ return true; ++} ++ ++static auto NRILinuxResourcesFromCRI(const runtime::v1::LinuxContainerResources &config, ++ nri_linux_resources &resources) -> bool ++{ ++ if (!NRILinuxMemoryFromCRI(config, *resources.memory)) { ++ ERROR("Failed to transform memory to nri for container"); ++ return false; ++ } ++ ++ if (!NRILinuxCpuFromCRI(config, *resources.cpu)) { ++ ERROR("Failed to transform cpu to nri for container"); ++ return false; ++ } ++ ++ if (!NRIHugePageLimitFromCRI(config, resources)) { ++ ERROR("Failed to transform hugepage limits to nri for container"); ++ return false; ++ } ++ ++ // resources.blockio_class is not support ++ // resources.rdt_class is not support ++ // They are not standard fields in oci spec ++ ++ Errors tmpError; ++ ++ resources.unified = Transform::ProtobufMapToJsonMapForString(config.unified(), tmpError); ++ if (resources.unified == nullptr) { ++ ERROR("Failed to transform unified to nri for container : %s", tmpError.GetMessage().c_str()); ++ return false; ++ } ++ ++ // resources.devices is not set in pod ++ ++ return true; ++} ++ ++static auto NRILinuxFromCRI(const runtime::v1::LinuxPodSandboxConfig &config, nri_linux_pod_sandbox &linux) -> bool ++{ ++ if (!init_nri_linux_resources(&linux.pod_overhead)) { ++ ERROR("Failed to init nri linux overhead resources for pod"); ++ return false; ++ } ++ if (!init_nri_linux_resources(&linux.pod_resources)) { ++ ERROR("Failed to init nri linux resources resources for pod"); ++ return false; ++ } ++ if (config.has_overhead() && !NRILinuxResourcesFromCRI(config.overhead(), *linux.pod_overhead)) { ++ ERROR("Failed to transform overhead to nri for pod"); ++ return false; ++ } ++ ++ if (config.has_resources() && !NRILinuxResourcesFromCRI(config.resources(), *linux.pod_resources)) { ++ ERROR("Failed to transform resources to nri for pod"); ++ return false; ++ } ++ ++ linux.cgroup_parent = util_strdup_s(config.cgroup_parent().c_str()); ++ ++ // todo: other container engines get linux.cgroups_path/linux.resourses/linux.namespace from spec.linux, ++ // How does isulad get these values ​​from CRI module? ++ return true; ++} ++ ++auto PodSandboxToNRI(const std::shared_ptr &sandbox, nri_pod_sandbox &pod) -> bool ++{ ++ container_t *cont = nullptr; ++ Errors tmpError; ++ ++ cont = containers_store_get(sandbox->GetName().c_str()); ++ if (cont != nullptr) { ++ pod.pid = container_state_get_pid(cont->state); ++ container_unref(cont); ++ } ++ ++ pod.id = util_strdup_s(sandbox->GetId().c_str()); ++ pod.name = util_strdup_s(sandbox->GetName().c_str()); ++ if (sandbox->GetSandboxConfig().has_metadata()) { ++ pod.uid = util_strdup_s(sandbox->GetSandboxConfig().metadata().uid().c_str()); ++ pod._namespace = util_strdup_s(sandbox->GetSandboxConfig().metadata().namespace_().c_str()); ++ } ++ ++ ++ pod.labels = Transform::ProtobufMapToJsonMapForString(sandbox->GetSandboxConfig().labels(), tmpError); ++ if (pod.labels == nullptr) { ++ ERROR("Failed to transform labels to nri for pod : %s, : %s", pod.name, tmpError.GetMessage().c_str()); ++ return false; ++ } ++ ++ pod.annotations = Transform::ProtobufMapToJsonMapForString(sandbox->GetSandboxConfig().annotations(), tmpError); ++ if (pod.annotations == nullptr) { ++ ERROR("Failed to transform annotations to nri for pod : %s, : %s", pod.name, tmpError.GetMessage().c_str()); ++ return false; ++ } ++ ++ if (sandbox->GetSandboxConfig().has_linux()) { ++ pod.linux = (nri_linux_pod_sandbox *)util_common_calloc_s(sizeof(nri_linux_pod_sandbox)); ++ if (pod.linux == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ if (!NRILinuxFromCRI(sandbox->GetSandboxConfig().linux(), *pod.linux)) { ++ ERROR("Failed to transform linux to nri for pod : %s", pod.name); ++ return false; ++ } ++ } ++ ++ pod.runtime_handler = util_strdup_s(sandbox->GetRuntimeHandle().c_str()); ++ ++ return true; ++} ++ ++static auto CRIMountArrToNRI(const runtime::v1::ContainerConfig &containerConfig, nri_container &con) -> bool ++{ ++ size_t i, len; ++ ++ // get mount from cont ++ len = containerConfig.mounts_size(); ++ if (len == 0) { ++ return true; ++ } ++ con.mounts = (nri_mount **)util_smart_calloc_s(sizeof(nri_mount *), len); ++ if (con.mounts == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ nri_mount *tmp = nullptr; ++ ++ for (i = 0; i < len; i++) { ++ tmp = (nri_mount *)util_common_calloc_s(sizeof(nri_mount)); ++ if (tmp == nullptr) { ++ ERROR("Out of memory"); ++ goto error_out; ++ } ++ ++ if (containerConfig.mounts()[i].container_path().empty() || containerConfig.mounts()[i].host_path().empty()) { ++ ERROR("Mount path is empty"); ++ goto error_out; ++ } ++ ++ char path[PATH_MAX] = { 0 }; ++ if (!util_clean_path(containerConfig.mounts()[i].container_path().c_str(), path, sizeof(path))) { ++ ERROR("Failed to get clean path for mount src path: %s", containerConfig.mounts()[i].container_path().c_str()); ++ goto error_out; ++ } ++ ++ tmp->destination = util_strdup_s(path); ++ ++ if (!util_clean_path(containerConfig.mounts()[i].host_path().c_str(), path, sizeof(path))) { ++ ERROR("Failed to get clean path for mount src path: %s", containerConfig.mounts()[i].host_path().c_str()); ++ goto error_out; ++ } ++ tmp->source = util_strdup_s(path); ++ ++ if (util_array_append(&(tmp->options), "rbind") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ ++ if (containerConfig.mounts()[i].propagation() == runtime::v1::PROPAGATION_PRIVATE) { ++ DEBUG("noop, private is default"); ++ if (util_array_append(&(tmp->options), "rprivate") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ } else if (containerConfig.mounts()[i].propagation() == runtime::v1::PROPAGATION_BIDIRECTIONAL) { ++ if (util_array_append(&(tmp->options), "rshared") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ } else if (containerConfig.mounts()[i].propagation() == runtime::v1::PROPAGATION_HOST_TO_CONTAINER) { ++ if (util_array_append(&(tmp->options), "rslave") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ } else { ++ WARN("unknown propagation mode for hostPath %s", containerConfig.mounts()[i].host_path().c_str()); ++ if (util_array_append(&(tmp->options), "rprivate") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ } ++ ++ if (containerConfig.mounts()[i].readonly()) { ++ if (util_array_append(&(tmp->options), "ro") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ } else { ++ if (util_array_append(&(tmp->options), "rw") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ } ++ ++ tmp->type = util_strdup_s("bind"); ++ ++ con.mounts[i] = tmp; ++ tmp = nullptr; ++ con.mounts_len++; ++ } ++ return true; ++ ++error_out: ++ free_nri_mount(tmp); ++ return false; ++} ++ ++static auto MountPointsElementToNRI(container_config_v2_common_config_mount_points *mp, nri_container &con) -> bool ++{ ++ size_t i, len; ++ nri_mount *tmp = nullptr; ++ ++ if (mp == nullptr || mp->len == 0) { ++ return true; ++ } ++ len = mp->len; ++ ++ con.mounts = (nri_mount **)util_smart_calloc_s(sizeof(nri_mount *), len); ++ if (con.mounts == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ for (i = 0; i < len; i++) { ++ tmp = (nri_mount *)util_common_calloc_s(sizeof(nri_mount)); ++ char path[PATH_MAX] = { 0 }; ++ ++ if (!util_clean_path(mp->values[i]->destination, path, sizeof(path))) { ++ ERROR("Failed to get clean path for mount dest path: %s", mp->values[i]->destination); ++ goto error_out; ++ } ++ tmp->destination = util_strdup_s(path); ++ ++ if (!util_clean_path(mp->values[i]->source, path, sizeof(path))) { ++ ERROR("Failed to get clean path for mount src path: %s", mp->values[i]->source); ++ goto error_out; ++ } ++ tmp->source = util_strdup_s(path); ++ ++ if (util_array_append(&(tmp->options), "rbind") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ if (util_array_append(&(tmp->options), mp->values[i]->propagation) != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ ++ if (mp->values[i]->rw) { ++ if (util_array_append(&(tmp->options), "rw") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ } else { ++ if (util_array_append(&(tmp->options), "ro") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ } ++ ++ tmp->type = util_strdup_s("bind"); ++ con.mounts[i] = tmp; ++ con.mounts_len++; ++ tmp = nullptr; ++ } ++ ++ return true; ++ ++error_out: ++ free_nri_mount(tmp); ++ return false; ++} ++ ++// container info is incomplete because container in excution is not created ++auto ContainerToNRIByConConfig(const runtime::v1::ContainerConfig &containerConfig, nri_container &con) -> bool ++{ ++ // todo: can not get container id and state from containerConfig ++ if (containerConfig.has_metadata() && !containerConfig.metadata().name().empty()) { ++ con.name = util_strdup_s(containerConfig.metadata().name().c_str()); ++ } ++ ++ Errors tmpError; ++ ++ con.labels = Transform::ProtobufMapToJsonMapForString(containerConfig.labels(), tmpError); ++ if (con.labels == nullptr) { ++ ERROR("Failed to transform labels to nri for con : %s, : %s", con.name, tmpError.GetMessage().c_str()); ++ return false; ++ } ++ ++ con.annotations = Transform::ProtobufMapToJsonMapForString(containerConfig.annotations(), tmpError); ++ if (con.annotations == nullptr) { ++ ERROR("Failed to transform annotations to nri for con : %s, : %s", con.name, tmpError.GetMessage().c_str()); ++ return false; ++ } ++ ++ con.args = Transform::RepeatedPtrFieldToCharArray(containerConfig.args()); ++ if (con.args == nullptr) { ++ ERROR("Failed to transform args to nri for con : %s, : %s", con.name, tmpError.GetMessage().c_str()); ++ return false; ++ } ++ con.args_len = containerConfig.args_size(); ++ ++ auto envVect = CRIHelpersV1::GenerateEnvList(containerConfig.envs()); ++ con.env = Transform::StringVectorToCharArray(envVect); ++ if (con.env == nullptr) { ++ ERROR("Failed to transform env to nri for con : %s", con.name); ++ return false; ++ } ++ con.env_len = containerConfig.envs_size(); ++ ++ if (!CRIMountArrToNRI(containerConfig, con)) { ++ ERROR("Failed to transform mounts to nri for con : %s", con.name); ++ return false; ++ } ++ return true; ++ ++ // todo: can not get container hooks and pid from containerConfig ++} ++ ++// container info is incomplete because container in excution is not created ++auto ContainerToNRIByID(const std::string &id, nri_container &con) -> bool ++{ ++ container_t *cont = nullptr; ++ bool ret = false; ++ ++ cont = containers_store_get(id.c_str()); ++ if (cont == nullptr || cont->common_config == nullptr) { ++ ERROR("No such container:%s", id.c_str()); ++ goto out; ++ } ++ ++ con.id = util_strdup_s(id.c_str()); ++ ++ con.name = util_strdup_s(cont->common_config->name); ++ ++ con.labels = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); ++ if (con.labels == nullptr) { ++ ERROR("Out of memory"); ++ goto out; ++ } ++ con.annotations = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); ++ if (con.annotations == nullptr) { ++ ERROR("Out of memory"); ++ goto out; ++ } ++ // state ++ if (dup_json_map_string_string(cont->common_config->config->labels, con.labels) != 0) { ++ ERROR("Failed to copy labels for con: %s", cont->common_config->name); ++ goto out; ++ } ++ if (dup_json_map_string_string(cont->common_config->config->annotations, con.annotations) != 0) { ++ ERROR("Failed to copy labels for con: %s", cont->common_config->name); ++ goto out; ++ } ++ ++ con.args = util_copy_array_by_len(cont->common_config->args, cont->common_config->args_len); ++ if (cont->common_config->args_len != 0 && con.args == nullptr) { ++ ERROR("Failed to copy args for con: %s", cont->common_config->name); ++ goto out; ++ } ++ con.args_len = cont->common_config->args_len; ++ ++ con.env = util_copy_array_by_len(cont->common_config->config->env, cont->common_config->config->env_len); ++ if (cont->common_config->config->env_len != 0 && con.env == nullptr) { ++ ERROR("Failed to copy env for con: %s", cont->common_config->name); ++ goto out; ++ } ++ con.env_len = cont->common_config->config->env_len; ++ ++ if (!MountPointsElementToNRI(cont->common_config->mount_points, con)) { ++ ERROR("Failed to transform mounts to nri for con : %s", con.name); ++ goto out; ++ } ++ ++ // todo: can convert hostconfig's hook_spec to nri spec ++ ++ con.pid = container_state_get_pid(cont->state); ++ if (con.pid < 0) { ++ ERROR("Container %s pid %d invalid", cont->common_config->name, con.pid); ++ goto out; ++ } ++ ++ con.pod_sandbox_id = util_strdup_s(cont->common_config->sandbox_info->id); ++ ret = true; ++ ++out: ++ container_unref(cont); ++ return ret; ++} ++ ++auto LinuxResourcesFromNRI(const nri_linux_resources *src, runtime::v1::LinuxContainerResources &resources) -> bool ++{ ++ if (src == nullptr) { ++ return false; ++ } ++ ++ if (src->memory != nullptr) { ++ resources.set_memory_limit_in_bytes(*src->memory->limit); ++ resources.set_oom_score_adj(DefaultOOMScoreAdj); ++ } ++ ++ if (src->cpu != nullptr) { ++ if (src->cpu->shares != NULL) { ++ resources.set_cpu_shares(*src->cpu->shares); ++ } ++ if (src->cpu->quota != NULL) { ++ resources.set_cpu_quota(*src->cpu->quota); ++ } ++ if (src->cpu->period != NULL) { ++ resources.set_cpu_period(*src->cpu->period); ++ } ++ ++ resources.set_cpuset_cpus(src->cpu->cpus); ++ resources.set_cpuset_mems(src->cpu->mems); ++ } ++ ++ if (src->hugepage_limits != nullptr && src->hugepage_limits_len > 0) { ++ for (size_t i = 0; i < src->hugepage_limits_len; i++) { ++ if (src->hugepage_limits[i] != nullptr) { ++ auto limit = resources.add_hugepage_limits(); ++ limit->set_page_size(src->hugepage_limits[i]->page_size); ++ limit->set_limit(src->hugepage_limits[i]->limit); ++ } ++ } ++ } ++ ++ if (src->unified != nullptr) { ++ Transform::JsonMapToProtobufMapForString(src->unified, *resources.mutable_unified()); ++ } ++ ++ return true; ++} +\ No newline at end of file +diff --git a/src/daemon/common/nri/nri_convert.h b/src/daemon/common/nri/nri_convert.h +index 883f7c41..c04b14e4 100644 +--- a/src/daemon/common/nri/nri_convert.h ++++ b/src/daemon/common/nri/nri_convert.h +@@ -27,10 +27,11 @@ + #include "sandbox.h" + #include "api_v1.pb.h" + +-auto PodSandboxToNRI(const std::shared_ptr &sandbox, nri_pod_sandbox *pod) -> bool; +-auto ContainerToNRIByConConfig(const runtime::v1::ContainerConfig &containerConfig, nri_container *con) -> bool; +-auto ContainerToNRIByID(const std::string &id, nri_container *con) -> bool; +-auto PodSandboxesToNRI(const std::vector> &arrs, nri_pod_sandbox **pod, int pod_len) -> bool; ++auto PodSandboxToNRI(const std::shared_ptr &sandbox, nri_pod_sandbox &pod) -> bool; ++auto ContainerToNRIByConConfig(const runtime::v1::ContainerConfig &containerConfig, nri_container &con) -> bool; ++auto ContainerToNRIByID(const std::string &id, nri_container &con) -> bool; ++auto PodSandboxesToNRI(const std::vector> &arrs, nri_pod_sandbox **pod, ++ int pod_len) -> bool; + + auto LinuxResourcesFromNRI(const nri_linux_resources *src, runtime::v1::LinuxContainerResources &resources) -> bool; + #endif // DAEMON_COMMON_NRI_NRI_CONVERT_H +diff --git a/src/daemon/common/nri/nri_spec.c b/src/daemon/common/nri/nri_spec.c +new file mode 100644 +index 00000000..855fe3b3 +--- /dev/null ++++ b/src/daemon/common/nri/nri_spec.c +@@ -0,0 +1,602 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-07-17 ++ * Description: provide nri oci functions ++ *********************************************************************************/ ++ ++#include "nri_spec.h" ++ ++#include ++ ++#include "map.h" ++#include "utils.h" ++#include "utils_string.h" ++#include "nri_utils.h" ++#include "specs_api.h" ++#include "sysinfo.h" ++#include "verify.h" ++#include "specs_extend.h" ++ ++static defs_hook *nri_hook_to_oci(const nri_hook *h) ++{ ++ defs_hook *oci_hook = NULL; ++ ++ if (h == NULL) { ++ return NULL; ++ } ++ ++ oci_hook = util_common_calloc_s(sizeof(*oci_hook)); ++ if (oci_hook == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ oci_hook->path = util_strdup_s(h->path); ++ if (h->args_len != 0) { ++ oci_hook->args = util_copy_array_by_len(h->args, h->args_len); ++ if (oci_hook->args == NULL) { ++ ERROR("Failed to copy args"); ++ goto error_out; ++ } ++ oci_hook->args_len = h->args_len; ++ } ++ if (h->env_len != 0) { ++ oci_hook->env = util_copy_array_by_len(h->env, h->env_len); ++ if (oci_hook->env == NULL) { ++ ERROR("Failed to copy env"); ++ goto error_out; ++ } ++ oci_hook->env_len = h->env_len; ++ } ++ if (h->timeout != NULL) { ++ oci_hook->timeout = *(h->timeout); ++ } ++ return oci_hook; ++ ++error_out: ++ free_defs_hook(oci_hook); ++ return NULL; ++} ++ ++static defs_device *nri_device_to_oci(nri_linux_device *dev) ++{ ++ if (dev == NULL) { ++ return NULL; ++ } ++ ++ defs_device *oci_dev = util_common_calloc_s(sizeof(defs_device)); ++ if (oci_dev == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ oci_dev->path = util_strdup_s(dev->path); ++ oci_dev->type = util_strdup_s(dev->type); ++ oci_dev->major = dev->major; ++ oci_dev->minor = dev->minor; ++ if (dev->file_mode != NULL) { ++ oci_dev->file_mode = *dev->file_mode; ++ } ++ if (dev->uid != NULL) { ++ oci_dev->uid = *dev->uid; ++ } ++ if (dev->gid != NULL) { ++ oci_dev->gid = *dev->gid; ++ } ++ ++ return oci_dev; ++} ++ ++static defs_mount *nri_mount_to_oci(nri_mount *mount) ++{ ++ if (mount == NULL) { ++ return NULL; ++ } ++ ++ defs_mount *oci_mount = util_common_calloc_s(sizeof(defs_mount)); ++ if (oci_mount == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ oci_mount->destination = util_strdup_s(mount->destination); ++ oci_mount->type = util_strdup_s(mount->type); ++ oci_mount->source = util_strdup_s(mount->source); ++ if (mount->options_len != 0) { ++ oci_mount->options = util_copy_array_by_len(mount->options, mount->options_len); ++ if (oci_mount->options == NULL) { ++ ERROR("Failed to copy options"); ++ free_defs_mount(oci_mount); ++ return NULL; ++ } ++ oci_mount->options_len = mount->options_len; ++ } ++ ++ return oci_mount; ++} ++ ++static int nri_adjust_annotation(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ int ret = -1; ++ size_t i; ++ ++ if (adjust == NULL || adjust->annotations == NULL || adjust->annotations->len == 0) { ++ return 0; ++ } ++ ++ if (make_sure_oci_spec_annotations(oci_spec) != 0) { ++ ERROR("Failed to make sure oci spec annotations"); ++ return -1; ++ } ++ ++ json_map_string_string *cleard = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); ++ if (cleard == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ ++ map_t *del = map_new(MAP_STR_STR, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ if (del == NULL) { ++ ERROR("Out of memory"); ++ goto free_out; ++ } ++ ++ for (i = 0; i < adjust->annotations->len; i++) { ++ __isula_auto_free char *out = NULL; ++ if (is_marked_for_removal(adjust->annotations->keys[i], &out)) { ++ if (!map_insert(del, out, "")) { ++ ERROR("Failed to insert del map"); ++ goto free_out; ++ } ++ continue; ++ } ++ if (append_json_map_string_string(cleard, adjust->annotations->keys[i], ++ adjust->annotations->values[i]) != 0) { ++ ERROR("Failed to append annotation"); ++ goto free_out; ++ } ++ } ++ ++ for (i = 0; i < oci_spec->annotations->len; i++) { ++ if (map_search(del, oci_spec->annotations->keys[i]) != NULL) { ++ continue; ++ } ++ append_json_map_string_string(cleard, oci_spec->annotations->keys[i], ++ oci_spec->annotations->values[i]); ++ } ++ ++ free_json_map_string_string(oci_spec->annotations); ++ oci_spec->annotations = cleard; ++ ret = 0; ++ ++free_out: ++ free_json_map_string_string(cleard); ++ map_free(del); ++ return ret; ++} ++ ++static void nri_key_value_map_kvfree(void *key, void *value) ++{ ++ free(key); ++ ++ // no need to free nri_key_value ++ // nri_key_value *value will be free in nri_container_adjustment *adjust ++} ++ ++ ++static int nri_adjust_env(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ int ret = -1; ++ size_t i; ++ char **old_env = NULL; ++ size_t old_env_len = 0; ++ __isula_auto_array_t char **adjust_env = NULL; ++ size_t adjust_env_len = 0; ++ ++ if (adjust->env == NULL || adjust->env_len == 0) { ++ return 0; ++ } ++ ++ map_t *mod = map_new(MAP_STR_PTR, MAP_DEFAULT_CMP_FUNC, nri_key_value_map_kvfree); ++ if (mod == NULL) { ++ ERROR("Out of memory"); ++ goto free_out; ++ } ++ ++ for (i = 0; i < adjust->env_len; i++) { ++ nri_key_value *e = adjust->env[i]; ++ char *out = NULL; ++ (void)is_marked_for_removal(e->key, &out); ++ ++ if (!map_insert(mod, out, e) == false) { ++ ERROR("Failed to insert mod map"); ++ goto free_out; ++ } ++ } ++ ++ if (map_size(mod) <= 0 || oci_spec == NULL || oci_spec->process == NULL) { ++ ret = 0; ++ goto free_out; ++ } ++ ++ // modify existing environment ++ old_env = oci_spec->process->env; ++ old_env_len = oci_spec->process->env_len; ++ oci_spec->process->env = NULL; ++ oci_spec->process->env_len = 0; ++ ++ for (i = 0; i < old_env_len; i++) { ++ __isula_auto_array_t char **envArr = util_string_split_n(old_env[i], '=', 2); ++ if (envArr == NULL) { ++ continue; ++ } ++ ++ nri_key_value *target = map_search(mod, envArr[0]); ++ if (target != NULL) { ++ __isula_auto_free char *out = NULL; ++ if (!is_marked_for_removal(envArr[0], &out)) { ++ // If not marked for removal, append modified value ++ __isula_auto_free char *tmp_str = util_string_append(target->key, "="); ++ __isula_auto_free char *final_str = util_string_append(tmp_str, target->value); ++ ++ if (util_array_append(&adjust_env, final_str) != 0) { ++ ERROR("Failed to append env"); ++ goto free_out; ++ } ++ adjust_env_len++; ++ continue; ++ } ++ } ++ // If not found in mod map, append original value ++ if (util_array_append(&adjust_env, old_env[i]) != 0) { ++ ERROR("Failed to append env"); ++ goto free_out; ++ } ++ adjust_env_len++; ++ } ++ ++ ret = 0; ++free_out: ++ if (merge_env(oci_spec, (const char **)adjust_env, adjust_env_len) != 0) { ++ ERROR("Failed to merge env"); ++ goto free_out; ++ } ++ for (i = 0; i < old_env_len; i++) { ++ free(old_env[i]); ++ } ++ free(old_env); ++ map_free(mod); ++ return ret; ++} ++ ++static int nri_adjust_hooks(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ if (adjust->hooks == NULL) { ++ return 0; ++ } ++ ++ size_t i; ++ int ret = 0; ++ ++ if (make_sure_oci_spec_hooks(oci_spec) != 0) { ++ ERROR("Failed to make sure oci spec hooks"); ++ return -1; ++ } ++ ++ // todo: change to macro definition function call ++ for (i = 0; i < adjust->hooks->prestart_len; i++) { ++ defs_hook *oci_hook = nri_hook_to_oci(adjust->hooks->prestart[i]); ++ ret = spec_add_prestart_hook(oci_spec, oci_hook); ++ if (ret != 0) { ++ ERROR("Failed add hook %s", adjust->hooks->prestart[i]->path); ++ free_defs_hook(oci_hook); ++ return -1; ++ } ++ } ++ ++ for (i = 0; i < adjust->hooks->poststart_len; i++) { ++ defs_hook *oci_hook = nri_hook_to_oci(adjust->hooks->poststart[i]); ++ ret = spec_add_poststart_hook(oci_spec, oci_hook); ++ if (ret != 0) { ++ ERROR("Failed add hook %s", adjust->hooks->poststart[i]->path); ++ free_defs_hook(oci_hook); ++ return -1; ++ } ++ } ++ ++ for (i = 0; i < adjust->hooks->poststop_len; i++) { ++ defs_hook *oci_hook = nri_hook_to_oci(adjust->hooks->poststop[i]); ++ ret = spec_add_poststop_hook(oci_spec, oci_hook); ++ if (ret != 0) { ++ ERROR("Failed add hook %s", adjust->hooks->poststop[i]->path); ++ free_defs_hook(oci_hook); ++ return -1; ++ } ++ } ++ /* ++ * The OCI being used by the iSulad not supportes ++ * createRuntime/createContainer/startContainer currently. ++ */ ++ ++ return ret; ++} ++ ++static int nri_adjust_devices(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ if (adjust->linux == NULL || adjust->linux->devices == NULL || adjust->linux->devices_len == 0) { ++ return 0; ++ } ++ ++ size_t i; ++ ++ for (i = 0; i < adjust->linux->devices_len; i++) { ++ nri_linux_device *dev = adjust->linux->devices[i]; ++ if (spec_add_device(oci_spec, nri_device_to_oci(dev)) != 0) { ++ ERROR("Failed to add device %s", dev->path); ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ ++static int nri_adjust_cgroup_path(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ if (adjust->linux == NULL || adjust->linux->cgroups_path == NULL) { ++ return 0; ++ } ++ ++ free(oci_spec->linux->cgroups_path); ++ oci_spec->linux->cgroups_path = util_strdup_s(adjust->linux->cgroups_path); ++ ++ return 0; ++} ++ ++static void nri_adjust_cpu_memory(nri_linux_resources *resource, oci_runtime_spec *oci_spec) ++{ ++ if (resource->cpu == NULL) { ++ return; ++ } ++ if (make_sure_oci_spec_linux_resources_cpu(oci_spec) != 0) { ++ ERROR("Failed to make sure oci spec linux resources cpu"); ++ return; ++ } ++ if (resource->cpu->shares != NULL) { ++ oci_spec->linux->resources->cpu->shares = *resource->cpu->shares; ++ } ++ if (resource->cpu->quota != NULL) { ++ oci_spec->linux->resources->cpu->quota = *resource->cpu->quota; ++ } ++ if (resource->cpu->period != NULL) { ++ oci_spec->linux->resources->cpu->period = *resource->cpu->period; ++ } ++ if (resource->cpu->realtime_runtime != NULL) { ++ oci_spec->linux->resources->cpu->realtime_runtime = *resource->cpu->realtime_runtime; ++ } ++ if (resource->cpu->realtime_period != NULL) { ++ oci_spec->linux->resources->cpu->realtime_period = *resource->cpu->realtime_period; ++ } ++} ++ ++static void nri_adjust_memory_resource(nri_linux_resources *resource, oci_runtime_spec *oci_spec) ++{ ++ if (resource->memory == NULL) { ++ return; ++ } ++ ++ if (make_sure_oci_spec_linux_resources_mem(oci_spec) != 0) { ++ ERROR("Failed to make sure oci spec linux resources memory"); ++ return; ++ } ++ if (resource->memory->limit != NULL) { ++ oci_spec->linux->resources->memory->limit = *resource->memory->limit; ++ } ++ if (resource->memory->reservation != NULL) { ++ oci_spec->linux->resources->memory->reservation = *resource->memory->reservation; ++ } ++ if (resource->memory->swap != NULL) { ++ oci_spec->linux->resources->memory->swap = *resource->memory->swap; ++ } ++ if (resource->memory->kernel != NULL) { ++ oci_spec->linux->resources->memory->kernel = *resource->memory->kernel; ++ } ++ if (resource->memory->kernel_tcp != NULL) { ++ oci_spec->linux->resources->memory->kernel_tcp = *resource->memory->kernel_tcp; ++ } ++ if (resource->memory->swappiness != NULL) { ++ oci_spec->linux->resources->memory->swappiness = *resource->memory->swappiness; ++ } ++ if (resource->memory->disable_oom_killer != NULL) { ++ oci_spec->linux->resources->memory->disable_oom_killer = *resource->memory->disable_oom_killer; ++ } ++} ++ ++static int nri_adjust_hugepage_resource(nri_linux_resources *resource, oci_runtime_spec *oci_spec) ++{ ++ size_t i; ++ if (resource->hugepage_limits != NULL) { ++ for (i = 0; i < resource->hugepage_limits_len; i++) { ++ nri_hugepage_limit *limit = resource->hugepage_limits[i]; ++ if (limit->page_size != NULL) { ++ if (spec_add_linux_resources_hugepage_limit(oci_spec, limit->page_size, limit->limit) != 0) { ++ ERROR("Failed to add hugepage limit"); ++ return -1; ++ } ++ } ++ } ++ } ++ return 0; ++} ++ ++static int nri_adjust_unified_resource(nri_linux_resources *resource, oci_runtime_spec *oci_spec) ++{ ++ size_t i; ++ if (resource->unified != NULL) { ++ for (i = 0; i < resource->unified->len; i++) { ++ if (append_json_map_string_string(oci_spec->linux->resources->unified, resource->unified->keys[i], ++ resource->unified->values[i]) != 0) { ++ ERROR("Failed to append unified resource"); ++ return -1; ++ } ++ } ++ } ++ return 0; ++} ++ ++static int nri_adjust_resources(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ if (adjust->linux == NULL || adjust->linux->resources == NULL) { ++ return 0; ++ } ++ ++ nri_linux_resources *resource = adjust->linux->resources; ++ ++ nri_adjust_memory_resource(resource, oci_spec); ++ nri_adjust_cpu_memory(resource, oci_spec); ++ ++ if (nri_adjust_hugepage_resource(resource, oci_spec) != 0) { ++ ERROR("Failed to adjust hugepage resource"); ++ return -1; ++ } ++ ++ if (nri_adjust_unified_resource(resource, oci_spec) != 0) { ++ ERROR("Failed to adjust unified resource"); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++static int nri_adjust_mounts(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ if (adjust->mounts == NULL || adjust->mounts_len == 0) { ++ return 0; ++ } ++ ++ size_t i; ++ for (i = 0; i < adjust->mounts_len; i++) { ++ nri_mount *mount = adjust->mounts[i]; ++ defs_mount *oci_mount = nri_mount_to_oci(mount); ++ if (oci_mount == NULL) { ++ ERROR("Failed to convert nri mount to oci mount"); ++ return -1; ++ } ++ if (spec_add_mount(oci_spec, oci_mount) != 0) { ++ ERROR("Failed to add mount"); ++ free_defs_mount(oci_mount); ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ ++static int nri_adjust_rlimit(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ if (adjust->rlimits == NULL || adjust->rlimits_len == 0) { ++ return 0; ++ } ++ ++ size_t i; ++ for (i = 0; i < adjust->rlimits_len; i++) { ++ nri_posix_rlimit *rlimit = adjust->rlimits[i]; ++ if (rlimit->type == NULL) { ++ ERROR("Invalid rlimit type"); ++ return -1; ++ } ++ if (spec_add_linux_resources_rlimit(oci_spec, rlimit->type, rlimit->soft, rlimit->hard) != 0) { ++ ERROR("Failed to add rlimit"); ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ ++// todo: we do not support it blockio_class ++static int nri_adjust_blockio_class(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ if (adjust->linux == NULL || adjust->linux->resources->blockio_class == NULL) { ++ return 0; ++ } ++ ++ return 0; ++} ++ ++int nri_adjust_oci_spec(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ if (oci_spec == NULL || adjust == NULL) { ++ ERROR("Invalid input arguments"); ++ return -1; ++ } ++ ++ if (nri_adjust_annotation(adjust, oci_spec) != 0) { ++ ERROR("Failed to do nri adjust annotation in oci spec"); ++ return -1; ++ } ++ ++ if (nri_adjust_env(adjust, oci_spec) != 0) { ++ ERROR("Failed to do nri adjust env in oci spec"); ++ return -1; ++ } ++ ++ if (nri_adjust_hooks(adjust, oci_spec) != 0) { ++ ERROR("Failed to do nri adjust hooks in oci spec"); ++ return -1; ++ } ++ ++ if (nri_adjust_devices(adjust, oci_spec) != 0) { ++ ERROR("Failed to do nri adjust devices in oci spec"); ++ return -1; ++ } ++ ++ if (nri_adjust_cgroup_path(adjust, oci_spec) != 0) { ++ ERROR("Failed to do nri adjust cgroup path in oci spec"); ++ return -1; ++ } ++ ++ if (nri_adjust_resources(adjust, oci_spec) != 0) { ++ ERROR("Failed to do nri adjust resources in oci spec"); ++ return -1; ++ } ++ ++ if (nri_adjust_blockio_class(adjust, oci_spec) != 0) { ++ ERROR("Failed to do nri adjust blockio class in oci spec"); ++ return -1; ++ } ++ ++ // iSuald is not support IntelRdt ++ if (nri_adjust_mounts(adjust, oci_spec) != 0) { ++ ERROR("Failed to do nri adjust mount in oci spec"); ++ return -1; ++ } ++ ++ if (nri_adjust_rlimit(adjust, oci_spec) != 0) { ++ ERROR("Failed to do nri adjust rlimit in oci spec"); ++ return -1; ++ } ++ ++ __isula_auto_sysinfo_t sysinfo_t *sysinfo = NULL; ++ ++ sysinfo = get_sys_info(true); ++ if (sysinfo == NULL) { ++ ERROR("Failed to get system info"); ++ return -1; ++ } ++ ++ if (verify_container_settings(oci_spec, sysinfo) != 0) { ++ ERROR("Failed to verify oci runtime spec settings after adjust by nri"); ++ return -1; ++ } ++ ++ return 0; ++} +\ No newline at end of file +diff --git a/src/daemon/common/nri/nri_utils.c b/src/daemon/common/nri/nri_utils.c +new file mode 100644 +index 00000000..51054e32 +--- /dev/null ++++ b/src/daemon/common/nri/nri_utils.c +@@ -0,0 +1,520 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-07-17 ++ * Description: provide nri utils functions ++ *********************************************************************************/ ++ ++#include "nri_utils.h" ++ ++#include ++ ++#include "utils.h" ++ ++static bool copy_nri_hugepage_limit(const nri_hugepage_limit* src, nri_hugepage_limit** dest) ++{ ++ if (src == NULL || dest == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ *dest = (nri_hugepage_limit *)util_common_calloc_s(sizeof(nri_hugepage_limit)); ++ if (*dest == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ (*dest)->limit = src->limit; ++ (*dest)->page_size = util_strdup_s(src->page_size); ++ return true; ++} ++ ++static bool copy_nri_hook(const nri_hook *src, nri_hook **dest) ++{ ++ if (src == NULL || dest == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ *dest = (nri_hook *)util_common_calloc_s(sizeof(nri_hook)); ++ if (dest == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ (*dest)->args = util_copy_array_by_len(src->args, src->args_len); ++ (*dest)->args_len = src->args_len; ++ (*dest)->env = util_copy_array_by_len(src->env, src->env_len); ++ (*dest)->env_len = src->env_len; ++ (*dest)->path = util_strdup_s(src->path); ++ return true; ++} ++ ++static bool copy_nri_linux_device_cgroup(const nri_linux_device_cgroup *src, nri_linux_device_cgroup **dest) ++{ ++ if (src == NULL || dest == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ *dest = (nri_linux_device_cgroup *)util_common_calloc_s(sizeof(nri_linux_device_cgroup)); ++ if (dest == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ (*dest)->allow = src->allow; ++ (*dest)->type = util_strdup_s(src->type); ++ (*dest)->major = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if ((*dest)->major == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ (*dest)->minor = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if ((*dest)->minor == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ (*dest)->access = util_strdup_s(src->access); ++ return true; ++} ++ ++static bool copy_nri_linux_cpu(const nri_linux_cpu *src, nri_linux_cpu **dest) ++{ ++ if (src == NULL || dest == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ (*dest) = (nri_linux_cpu *)util_common_calloc_s(sizeof(nri_linux_cpu)); ++ if ((*dest) == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ (*dest)->cpus = util_strdup_s(src->cpus); ++ (*dest)->mems = util_strdup_s(src->mems); ++ if (src->period != NULL) { ++ (*dest)->period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if ((*dest)->period == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->period = *src->period; ++ } ++ ++ if (src->quota != NULL) { ++ (*dest)->quota = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if ((*dest)->quota == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->quota = *src->quota; ++ } ++ ++ if (src->realtime_period != NULL) { ++ (*dest)->realtime_period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if ((*dest)->realtime_period == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->realtime_period = *src->realtime_period; ++ } ++ ++ if (src->realtime_runtime != NULL) { ++ (*dest)->realtime_runtime = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if ((*dest)->realtime_runtime == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->realtime_runtime = *src->realtime_runtime; ++ } ++ ++ if (src->shares != NULL) { ++ (*dest)->shares = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if ((*dest)->shares == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->shares = *src->shares; ++ } ++ ++ return true; ++} ++ ++static bool copy_nri_linux_memory(const nri_linux_memory *src, nri_linux_memory **dest) ++{ ++ if (src == NULL || dest == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ *dest = (nri_linux_memory *)util_common_calloc_s(sizeof(nri_linux_memory)); ++ if (dest == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ if (src->limit != NULL) { ++ (*dest)->limit = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if ((*dest)->limit == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->limit = *src->limit; ++ } ++ ++ if (src->reservation != NULL) { ++ (*dest)->reservation = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if ((*dest)->reservation == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->reservation = *src->reservation; ++ } ++ ++ if (src->swap != NULL) { ++ (*dest)->swap = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if ((*dest)->swap == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->swap = *src->swap; ++ } ++ ++ if (src->kernel != NULL) { ++ (*dest)->kernel = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if ((*dest)->kernel == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->kernel = *src->kernel; ++ } ++ ++ ++ if (src->kernel_tcp != NULL) { ++ (*dest)->kernel_tcp = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if ((*dest)->kernel_tcp == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->kernel_tcp = *src->kernel_tcp; ++ } ++ ++ if (src->swappiness != NULL) { ++ (*dest)->swappiness = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if ((*dest)->swappiness == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->swappiness = *src->swappiness; ++ } ++ ++ if (src->disable_oom_killer != NULL) { ++ (*dest)->disable_oom_killer = (uint8_t *)util_common_calloc_s(sizeof(uint8_t)); ++ if ((*dest)->disable_oom_killer == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->disable_oom_killer = *src->disable_oom_killer; ++ } ++ ++ if (src->use_hierarchy != NULL) { ++ (*dest)->use_hierarchy = (uint8_t *)util_common_calloc_s(sizeof(uint8_t)); ++ if ((*dest)->use_hierarchy == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->use_hierarchy = *src->use_hierarchy; ++ } ++ return true; ++} ++ ++bool is_marked_for_removal(const char* key, char **out) ++{ ++ if (key == NULL || out == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ if (!util_has_prefix(key, "-")) { ++ *out = (char*)key; ++ return false; ++ } ++ ++ *out = util_sub_string(key, 1, strlen(key) - 1); ++ if (*out == NULL) { ++ ERROR("Failed to sub string"); ++ return false; ++ } ++ ++ return true; ++} ++ ++bool copy_nri_mount(const nri_mount *src, nri_mount **dest) ++{ ++ if (src == NULL || dest == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ *dest = (nri_mount *)util_common_calloc_s(sizeof(nri_mount)); ++ if (dest == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ (*dest)->destination = util_strdup_s(src->destination); ++ (*dest)->options = util_copy_array_by_len(src->options, src->options_len); ++ (*dest)->options_len = src->options_len; ++ (*dest)->source = util_strdup_s(src->source); ++ (*dest)->type = util_strdup_s(src->type); ++ return true; ++} ++ ++bool copy_nri_key_value(const nri_key_value *src, nri_key_value **dest) ++{ ++ if (src == NULL || dest == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ *dest = (nri_key_value *)util_common_calloc_s(sizeof(nri_key_value)); ++ if (dest == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ (*dest)->key = util_strdup_s(src->key); ++ (*dest)->value = util_strdup_s(src->value); ++ return true; ++} ++ ++bool copy_nri_posix_rlimit(const nri_posix_rlimit *src, nri_posix_rlimit **dest) ++{ ++ if (src == NULL || dest == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ *dest = (nri_posix_rlimit *)util_common_calloc_s(sizeof(nri_posix_rlimit)); ++ if (dest == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ (*dest)->hard = src->hard; ++ (*dest)->soft = src->soft; ++ (*dest)->type = util_strdup_s(src->type); ++ return true; ++} ++ ++bool copy_nri_linux_resources(const nri_linux_resources *src, nri_linux_resources **dest) ++{ ++ if (src == NULL || dest == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ *dest = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); ++ if (*dest == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ if (!init_nri_linux_resources(dest)) { ++ ERROR("Failed to init dest nri linux resources"); ++ goto free_out; ++ } ++ ++ if (!copy_nri_linux_cpu(src->cpu, &(*dest)->cpu)) { ++ ERROR("Failed to copy nri_linux_cpu"); ++ goto free_out; ++ } ++ ++ if (!copy_nri_linux_memory(src->memory, &(*dest)->memory)) { ++ ERROR("Failed to copy nri_linux_memory"); ++ goto free_out; ++ } ++ ++ (*dest)->blockio_class = util_strdup_s(src->blockio_class); ++ (*dest)->rdt_class = util_strdup_s(src->rdt_class); ++ ++ if (src->hugepage_limits_len > 0) { ++ (*dest)->hugepage_limits = (nri_hugepage_limit**)util_smart_calloc_s(sizeof(nri_hugepage_limit*), ++ src->hugepage_limits_len); ++ for (size_t i = 0; i < src->hugepage_limits_len; ++i) { ++ if (!copy_nri_hugepage_limit(src->hugepage_limits[i], &((*dest)->hugepage_limits[i]))) { ++ ERROR("Failed to copy nri_hugepage_limit"); ++ goto free_out; ++ } ++ } ++ } ++ ++ if (src->devices_len > 0) { ++ (*dest)->devices = (nri_linux_device_cgroup**)util_smart_calloc_s(sizeof(nri_linux_device_cgroup*), src->devices_len); ++ for (size_t i = 0; i < src->devices_len; ++i) { ++ if (!copy_nri_linux_device_cgroup(src->devices[i], &((*dest)->devices[i]))) { ++ ERROR("Failed to copy nri_linux_device_cgroup"); ++ goto free_out; ++ } ++ } ++ } ++ ++ if (dup_json_map_string_string(src->unified, (*dest)->unified)) { ++ ERROR("Failed to copy json_map_string_string"); ++ goto free_out; ++ } ++ ++ return true; ++ ++free_out: ++ free_nri_linux_resources(*dest); ++ return false; ++} ++ ++bool merge_nri_hooks(nri_hook **targetHooks, size_t targetSize, const nri_hook **sourceHooks, ++ size_t sourceLen) ++{ ++ size_t oldSize = targetSize * sizeof(nri_hook *); ++ size_t newSize = oldSize + sourceLen * sizeof(nri_hook *); ++ ++ if (sourceHooks == NULL || targetHooks == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ if (util_mem_realloc((void**)&targetHooks, newSize, (void**)&targetHooks, oldSize) != 0) { ++ ERROR("Failed to realloc and assign hook array"); ++ return false; ++ } ++ ++ for (size_t i = 0; i < sourceLen; i++) { ++ if (!copy_nri_hook(sourceHooks[i], &targetHooks[targetSize++])) { ++ ERROR("Failed to copy hook"); ++ return false; ++ } ++ } ++ ++ return true; ++} ++ ++bool init_nri_container_adjust(nri_container_adjustment **adjust) ++{ ++ if (adjust == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ *adjust = (nri_container_adjustment *)util_common_calloc_s(sizeof(nri_container_adjustment)); ++ if (*adjust == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ (*adjust)->annotations = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); ++ if ((*adjust)->annotations == NULL) { ++ goto free_out; ++ } ++ ++ (*adjust)->env = (nri_key_value **)util_common_calloc_s(sizeof(nri_key_value *)); ++ if ((*adjust)->env == NULL) { ++ goto free_out; ++ } ++ (*adjust)->env_len = 0; ++ ++ (*adjust)->hooks = (nri_hooks *)util_common_calloc_s(sizeof(nri_hooks)); ++ if ((*adjust)->hooks == NULL) { ++ goto free_out; ++ } ++ ++ (*adjust)->linux = (nri_linux_container_adjustment *)util_common_calloc_s(sizeof(nri_linux_container_adjustment)); ++ if ((*adjust)->linux == NULL) { ++ goto free_out; ++ } ++ ++ (*adjust)->linux->resources = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); ++ if ((*adjust)->linux->resources == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ (*adjust)->mounts = (nri_mount **)util_common_calloc_s(sizeof(nri_mount *)); ++ if ((*adjust)->mounts == NULL) { ++ goto free_out; ++ } ++ (*adjust)->mounts_len = 0; ++ ++ (*adjust)->rlimits = (nri_posix_rlimit **)util_common_calloc_s(sizeof(nri_posix_rlimit *)); ++ if ((*adjust)->rlimits == NULL) { ++ goto free_out; ++ } ++ (*adjust)->rlimits_len = 0; ++ ++ return true; ++ ++free_out: ++ ERROR("Out of memory"); ++ free_nri_container_adjustment(*adjust); ++ return false; ++} ++ ++bool init_nri_container_update(nri_container_update **update, const char *id, uint8_t ignore_failure) ++{ ++ if (update == NULL || id == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ *update = (nri_container_update *)util_common_calloc_s(sizeof(nri_container_update)); ++ if (*update == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ (*update)->container_id = util_strdup_s(id); ++ (*update)->linux = (nri_linux_container_update *)util_common_calloc_s(sizeof(nri_linux_container_update)); ++ if ((*update)->linux == NULL) { ++ goto free_out; ++ } ++ ++ (*update)->ignore_failure = ignore_failure; ++ return true; ++ ++free_out: ++ ERROR("Out of memory"); ++ free_nri_container_update(*update); ++ return false; ++} ++ ++bool init_nri_linux_resources(nri_linux_resources **resources) ++{ ++ if (resources == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ *resources = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); ++ if (*resources == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ (*resources)->cpu = (nri_linux_cpu *)util_common_calloc_s(sizeof(nri_linux_cpu)); ++ if ((*resources)->cpu == NULL) { ++ goto free_out; ++ } ++ ++ (*resources)->memory = (nri_linux_memory *)util_common_calloc_s(sizeof(nri_linux_memory)); ++ if ((*resources)->memory == NULL) { ++ goto free_out; ++ } ++ ++ (*resources)->unified = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); ++ if ((*resources)->unified == NULL) { ++ goto free_out; ++ } ++ return true; ++ ++free_out: ++ ERROR("Out of memory"); ++ free_nri_linux_resources(*resources); ++ return false; ++} +\ No newline at end of file +diff --git a/src/daemon/common/nri/nri_utils.h b/src/daemon/common/nri/nri_utils.h +index 3aa50ae4..7bf54a71 100644 +--- a/src/daemon/common/nri/nri_utils.h ++++ b/src/daemon/common/nri/nri_utils.h +@@ -51,14 +51,13 @@ typedef enum { + + bool copy_nri_mount(const nri_mount *src, nri_mount **dest); + bool copy_nri_key_value(const nri_key_value *src, nri_key_value **dest); +-bool copy_nri_hook(const nri_hook *src, nri_hook **dest); + bool copy_nri_posix_rlimit(const nri_posix_rlimit *src, nri_posix_rlimit **dest); + bool copy_nri_linux_resources(const nri_linux_resources *src, nri_linux_resources **dest); +-bool copy_nri_linux_cpu(const nri_linux_cpu *src, nri_linux_cpu **dest); + + bool is_marked_for_removal(const char* key, char **out); + +-bool realloc_and_copy_nri_hooks(nri_hook **targetHooks, size_t targetSize, const nri_hook **sourceHooks, size_t sourceLen); ++bool merge_nri_hooks(nri_hook **targetHooks, size_t targetSize, const nri_hook **sourceHooks, ++ size_t sourceLen); + + bool init_nri_container_adjust(nri_container_adjustment **adjust); + bool init_nri_container_update(nri_container_update **update, const char *id, uint8_t ignore_failure); +diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c +index d7b54498..9ba1c8a0 100644 +--- a/src/daemon/config/isulad_config.c ++++ b/src/daemon/config/isulad_config.c +@@ -456,6 +456,175 @@ out: + (void)isulad_server_conf_unlock(); + return path; + } ++ ++#ifdef ENABLE_NRI ++bool conf_get_nri_support(void) ++{ ++ bool nri_support = false; ++ struct service_arguments *conf = NULL; ++ ++ if (isulad_server_conf_rdlock() != 0) { ++ return false; ++ } ++ ++ conf = conf_get_server_conf(); ++ if (conf == NULL || conf->json_confs == NULL) { ++ goto out; ++ } ++ ++ nri_support = conf->json_confs->nri_support; ++ ++out: ++ (void)isulad_server_conf_unlock(); ++ return nri_support; ++} ++ ++bool conf_get_nri_external_support(void) ++{ ++ bool nri_external_support = false; ++ struct service_arguments *conf = NULL; ++ ++ if (isulad_server_conf_rdlock() != 0) { ++ return false; ++ } ++ ++ conf = conf_get_server_conf(); ++ if (conf == NULL || conf->json_confs == NULL) { ++ goto out; ++ } ++ ++ nri_external_support = conf->json_confs->disable_connections; ++ ++out: ++ (void)isulad_server_conf_unlock(); ++ return !nri_external_support; ++} ++ ++char *conf_get_nri_plugin_config_path(void) ++{ ++ char *path = NULL; ++ struct service_arguments *conf = NULL; ++ ++ if (isulad_server_conf_rdlock() != 0) { ++ return NULL; ++ } ++ ++ conf = conf_get_server_conf(); ++ if (conf == NULL || conf->json_confs == NULL || conf->json_confs->plugin_config_path == NULL) { ++ path = util_strdup_s(DEFAULT_PLUGIN_CONFIG_PATH); ++ goto out; ++ } ++ ++ path = util_strdup_s(conf->json_confs->plugin_config_path); ++ ++out: ++ (void)isulad_server_conf_unlock(); ++ return path; ++} ++ ++char *conf_get_nri_plugin_path(void) ++{ ++ char *path = NULL; ++ struct service_arguments *conf = NULL; ++ ++ if (isulad_server_conf_rdlock() != 0) { ++ return NULL; ++ } ++ ++ conf = conf_get_server_conf(); ++ if (conf == NULL || conf->json_confs == NULL) { ++ goto out; ++ } ++ ++ if (conf->json_confs->plugin_path == NULL) { ++ path = util_strdup_s(DEFAULT_PLUGIN_PATH); ++ goto out; ++ } ++ ++ path = util_strdup_s(conf->json_confs->plugin_path); ++ ++out: ++ (void)isulad_server_conf_unlock(); ++ return path; ++} ++ ++char *conf_get_socket_path(void) ++{ ++ char *path = NULL; ++ struct service_arguments *conf = NULL; ++ ++ if (isulad_server_conf_rdlock() != 0) { ++ return NULL; ++ } ++ ++ conf = conf_get_server_conf(); ++ if (conf == NULL || conf->json_confs == NULL) { ++ goto out; ++ } ++ ++ if (conf->json_confs->nri_socket_path == NULL) { ++ path = util_strdup_s(DEFAULT_SOCKET_PATH); ++ goto out; ++ } ++ ++ path = util_strdup_s(conf->json_confs->nri_socket_path); ++ ++out: ++ (void)isulad_server_conf_unlock(); ++ return path; ++} ++ ++uint64_t conf_get_nri_plugin_registration_timeout(void) ++{ ++ uint64_t timeout = false; ++ struct service_arguments *conf = NULL; ++ ++ if (isulad_server_conf_rdlock() != 0) { ++ return false; ++ } ++ ++ conf = conf_get_server_conf(); ++ if (conf == NULL || conf->json_confs == NULL) { ++ goto out; ++ } ++ ++ if (conf->json_confs->plugin_registration_timeout == 0) { ++ timeout = DEFAULT_PLUGIN_REGISTRY_TIMEOUT; ++ goto out; ++ } ++ ++ timeout = conf->json_confs->plugin_registration_timeout; ++ ++out: ++ (void)isulad_server_conf_unlock(); ++ return timeout; ++} ++uint64_t conf_get_nri_plugin_requst_timeout(void) ++{ ++ uint64_t timeout = false; ++ struct service_arguments *conf = NULL; ++ ++ if (isulad_server_conf_rdlock() != 0) { ++ return false; ++ } ++ ++ conf = conf_get_server_conf(); ++ if (conf == NULL || conf->json_confs == NULL) { ++ goto out; ++ } ++ ++ if (conf->json_confs->plugin_requst_timeout == 0) { ++ timeout = DEFAULT_PLUGIN_REQUST_TIMEOUT; ++ goto out; ++ } ++ ++ timeout = conf->json_confs->plugin_requst_timeout; ++ ++out: ++ (void)isulad_server_conf_unlock(); ++ return timeout; ++} ++#endif + #endif + + /* conf get isulad rootdir */ +@@ -1762,6 +1931,15 @@ int merge_json_confs_into_global(struct service_arguments *args) + tmp_json_confs->cri_sandboxers = NULL; + #endif + args->json_confs->enable_cri_v1 = tmp_json_confs->enable_cri_v1; ++#ifdef ENABLE_NRI ++ args->json_confs->nri_support = tmp_json_confs->nri_support; ++ args->json_confs->disable_connections = tmp_json_confs->disable_connections; ++ override_string_value(&args->json_confs->plugin_config_path, &tmp_json_confs->plugin_config_path); ++ override_string_value(&args->json_confs->plugin_path, &tmp_json_confs->plugin_path); ++ args->json_confs->plugin_registration_timeout = tmp_json_confs->plugin_registration_timeout; ++ args->json_confs->plugin_requst_timeout = tmp_json_confs->plugin_requst_timeout; ++ override_string_value(&args->json_confs->nri_socket_path, &tmp_json_confs->nri_socket_path); ++#endif + args->json_confs->enable_pod_events = tmp_json_confs->enable_pod_events; + #endif + +diff --git a/src/daemon/modules/api/specs_api.h b/src/daemon/modules/api/specs_api.h +index 6a1cd776..d5ea0c7c 100644 +--- a/src/daemon/modules/api/specs_api.h ++++ b/src/daemon/modules/api/specs_api.h +@@ -76,6 +76,11 @@ int spec_add_linux_resources_hugepage_limit(oci_runtime_spec *oci_spec, const ch + int spec_add_linux_resources_rlimit(oci_runtime_spec *oci_spec, const char *type, uint64_t hard, uint64_t soft); + #endif /* ENABLE_NRI */ + ++int make_sure_oci_spec_annotations(oci_runtime_spec *oci_spec); ++int make_sure_oci_spec_linux_resources_cpu(oci_runtime_spec *oci_spec); ++int make_sure_oci_spec_linux_resources_mem(oci_runtime_spec *oci_spec); ++int make_sure_oci_spec_hooks(oci_runtime_spec *oci_spec); ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index 1fd9e5a8..002431d8 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -87,8 +87,11 @@ struct readonly_default_oci_spec { + + static struct readonly_default_oci_spec g_rdspec; + +-static int make_sure_oci_spec_annotations(oci_runtime_spec *oci_spec) ++int make_sure_oci_spec_annotations(oci_runtime_spec *oci_spec) + { ++ if (oci_spec == NULL) { ++ return -1; ++ } + if (oci_spec->annotations == NULL) { + oci_spec->annotations = util_common_calloc_s(sizeof(json_map_string_string)); + if (oci_spec->annotations == NULL) { +@@ -464,10 +467,14 @@ out: + return ret; + } + +-static int make_sure_oci_spec_linux_resources_cpu(oci_runtime_spec *oci_spec) ++int make_sure_oci_spec_linux_resources_cpu(oci_runtime_spec *oci_spec) + { + int ret = 0; + ++ if (oci_spec == NULL) { ++ return -1; ++ } ++ + ret = make_sure_oci_spec_linux_resources(oci_spec); + if (ret < 0) { + return -1; +@@ -589,10 +596,14 @@ out: + return ret; + } + +-static int make_sure_oci_spec_linux_resources_mem(oci_runtime_spec *oci_spec) ++int make_sure_oci_spec_linux_resources_mem(oci_runtime_spec *oci_spec) + { + int ret = 0; + ++ if (oci_spec == NULL) { ++ return -1; ++ } ++ + ret = make_sure_oci_spec_linux_resources(oci_spec); + if (ret < 0) { + return -1; +@@ -731,8 +742,11 @@ out: + return ret; + } + +-static int make_sure_oci_spec_hooks(oci_runtime_spec *oci_spec) ++int make_sure_oci_spec_hooks(oci_runtime_spec *oci_spec) + { ++ if (oci_spec == NULL) { ++ return -1; ++ } + if (oci_spec->hooks == NULL) { + oci_spec->hooks = util_common_calloc_s(sizeof(oci_runtime_spec_hooks)); + if (oci_spec->hooks == NULL) { +@@ -2827,6 +2841,11 @@ int spec_add_linux_resources_hugepage_limit(oci_runtime_spec *oci_spec, const ch + int ret = 0; + defs_resources_hugepage_limits_element *hugepage_limit = NULL; + ++ if (oci_spec == NULL || page_size == NULL) { ++ ERROR("Invalid arguments"); ++ return -1; ++ } ++ + ret = make_sure_oci_spec_linux_resources(oci_spec); + if (ret < 0) { + return -1; +@@ -2859,6 +2878,11 @@ int spec_add_linux_resources_rlimit(oci_runtime_spec *oci_spec, const char *type + int ret = 0; + defs_process_rlimits_element *rlimit = NULL; + ++ if (oci_spec == NULL || type == NULL) { ++ ERROR("Invalid arguments"); ++ return -1; ++ } ++ + ret = make_sure_oci_spec_linux_resources(oci_spec); + if (ret < 0) { + return -1; +diff --git a/src/daemon/nri/nri_adaption.h b/src/daemon/nri/nri_adaption.h +index 7f0640df..874662cf 100644 +--- a/src/daemon/nri/nri_adaption.h ++++ b/src/daemon/nri/nri_adaption.h +@@ -46,7 +46,6 @@ public: + + auto GetSockpath(std::vector &paths) -> bool; + +- // Stop plugins. + auto StopPlugins() -> bool; + + void RemoveClosedPlugins(); +@@ -58,19 +57,23 @@ public: + auto RunPodSandbox(std::shared_ptr sandbox, Errors &error) ->bool; + auto StopPodSandbox(std::shared_ptr sandbox, Errors &error) ->bool; + auto RemovePodSandbox(std::shared_ptr sandbox, Errors &error) ->bool; +- auto CreateContainer(std::shared_ptr sandbox, const std::string &conId, const runtime::v1::ContainerConfig &containerConfig, nri_container_adjustment **adjust, Errors &error) -> bool; +- auto PostCreateContainer(const std::string &conId, Errors &error) ->bool; +- auto UndoCreateContainer(std::shared_ptr sandbox, const std::string &conId, Errors &error) -> bool; +- auto StartContainer(const std::string &conId, Errors &error) ->bool; +- auto PostStartContainer(const std::string &conId, Errors &error) ->bool; +- auto UpdateContainer(const std::string &conId, Errors &error) ->bool; +- auto PostUpdateContainer(const std::string &conId, Errors &error) ->bool; +- auto StopContainer(const std::string &conId, Errors &error) ->bool; +- auto RemoveContainer(const std::string &conId, Errors &error) ->bool; +- auto StateChange(nri_state_change_event *evt, Errors &error) ->bool; +- auto updateContainers(const nri_update_containers_request *req, nri_update_containers_response **resp) ->bool; ++ auto CreateContainer(std::shared_ptr sandbox, const std::string &conId, ++ const runtime::v1::ContainerConfig &containerConfig, nri_container_adjustment **adjust, ++ Errors &error) -> bool; ++ auto PostCreateContainer(const std::string &conId, Errors &error) -> bool; ++ auto UndoCreateContainer(std::shared_ptr sandbox, const std::string &conId, ++ Errors &error) -> bool; ++ auto StartContainer(const std::string &conId, Errors &error) -> bool; ++ auto PostStartContainer(const std::string &conId, Errors &error) -> bool; ++ auto UpdateContainer(const std::string &conId, Errors &error) -> bool; ++ auto PostUpdateContainer(const std::string &conId, Errors &error) -> bool; ++ auto StopContainer(const std::string &conId, Errors &error) -> bool; ++ auto RemoveContainer(const std::string &conId, Errors &error) -> bool; ++ auto StateChange(nri_state_change_event *evt, Errors &error) -> bool; ++ auto updateContainers(const nri_update_containers_request *req, nri_update_containers_response **resp) -> bool; + + auto NewExternalPlugin(int fd) -> bool; ++ + private: + NRIAdaptation() = default; + NRIAdaptation(const NRIAdaptation &other) = delete; +@@ -86,18 +89,25 @@ private: + auto SortPlugins() -> bool; + void GetClosedPlugins(std::vector &closedPlugin); + +- auto IsSupport() -> bool; ++ auto ApplyUpdates(const std::vector &update, std::vector &failed, ++ bool getFailed, Errors &error) -> bool; + +- auto ApplyUpdates(const std::vector &update, std::vector &failed, bool getFailed, +- Errors &error) -> bool; ++ auto IsSupport() -> bool; + +- auto NRIPodSandbox(const std::shared_ptr &sandbox, Errors& error) -> std::unique_ptr>; +- auto NRIContainerByConConfig(const std::shared_ptr &sandbox, const runtime::v1::ContainerConfig &containerConfig, Errors& error) -> std::unique_ptr>; +- auto NRIContainerByID(const std::string &id, Errors& error) -> std::unique_ptr>; ++ auto NRIPodSandbox(const std::shared_ptr &sandbox, ++ Errors &error) -> std::unique_ptr>; ++ auto NRIContainerByConConfig(const std::shared_ptr &sandbox, ++ const runtime::v1::ContainerConfig &containerConfig, Errors &error) -> std::unique_ptr>; ++ auto NRIContainerByID(const std::string &id, Errors &error) -> std::unique_ptr>; + + auto GetNRIPluginConfigPath(void) -> std::string; + auto GetNRIPluginPath(void) -> std::string; + auto GetNRISockPath(void) -> std::string; ++ ++ void PluginsStateChange(nri_state_change_event *evt); ++ bool PluginsCreateContainer(nri_create_container_request *req, const std::string &conId, pluginResult &result); ++ bool PluginsUpdateContainer(nri_update_container_request *req, const std::string &conId, pluginResult &result); ++ + private: + RWMutex m_mutex; + static std::atomic m_instance; +diff --git a/src/daemon/nri/nri_helpers.cc b/src/daemon/nri/nri_helpers.cc +new file mode 100644 +index 00000000..ff9d67c1 +--- /dev/null ++++ b/src/daemon/nri/nri_helpers.cc +@@ -0,0 +1,93 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-07-13 ++ * Description: provide nri helpers functions ++ *********************************************************************************/ ++ ++#include "nri_helpers.h" ++ ++#include ++ ++#include "utils.h" ++#include "isulad_config.h" ++ ++namespace NRIHelpers { ++std::string MarkForRemoval(const std::string &key) ++{ ++ return "-" + key; ++} ++ ++auto GetPluginConfig(std::string &idx, std::string &name, std::string &config) -> bool ++{ ++ __isula_auto_free char *plugin_path = NULL; ++ ++ plugin_path = conf_get_nri_plugin_config_path(); ++ if (plugin_path == NULL) { ++ return false; ++ } ++ std::string compleName = idx + "-" + name; ++ std::vector dropIns = { ++ std::string(plugin_path) + "/" + compleName + ".conf", ++ std::string(plugin_path) + "/" + name + ".conf" ++ }; ++ ++ for (const std::string &path : dropIns) { ++ char buf[MAX_BUFFER_SIZE + 1] = { 0 }; ++ __isula_auto_close int fd = util_open(path.c_str(), O_RDONLY, 0); ++ if (fd < 0) { ++ ERROR("Failed to open '%s'", path.c_str()); ++ return false; ++ } ++ int len = util_read_nointr(fd, buf, sizeof(buf) - 1); ++ if (len < 0) { ++ SYSERROR("Failed to read nri plugin config : %s", path.c_str()); ++ return false; ++ } ++ config = std::string(buf); ++ return true; ++ } ++ return true; ++} ++ ++void GenerateRandomExternalName(std::string &ret) ++{ ++ __isula_auto_free char *external_name = NULL; ++ ++ external_name = (char *)util_smart_calloc_s(sizeof(char), (CONTAINER_ID_MAX_LEN + 1)); ++ if (external_name == NULL) { ++ ERROR("Out of memory"); ++ return; ++ } ++ ++ if (util_generate_random_str(external_name, (size_t)CONTAINER_ID_MAX_LEN)) { ++ ERROR("Generate exec suffix failed"); ++ return; ++ } ++ ++ ret = std::string(external_name); ++} ++ ++bool CheckPluginIndex(const std::string &idx) ++{ ++ if (idx.length() != 2) { ++ ERROR("Invalid plugin index \"%s\", must be 2 digits", idx.c_str()); ++ return false; ++ } ++ ++ if (!std::isdigit(idx[0]) || !std::isdigit(idx[1])) { ++ ERROR("Invalid plugin index \"%s\", (not [0-9][0-9])", idx.c_str()); ++ return false; ++ } ++ ++ return true; ++} ++}// namespace NRIHelpers +\ No newline at end of file +diff --git a/src/daemon/nri/nri_helpers.h b/src/daemon/nri/nri_helpers.h +index 06ee8419..1a2f488e 100644 +--- a/src/daemon/nri/nri_helpers.h ++++ b/src/daemon/nri/nri_helpers.h +@@ -37,7 +37,7 @@ std::string MarkForRemoval(const std::string &key); + + auto GetPluginConfig(std::string &idx, std::string &name, std::string &config) -> bool; + +-std::string GenerateRandomExternalName(void); ++void GenerateRandomExternalName(std::string &ret); + + bool CheckPluginIndex(const std::string &idx); + +diff --git a/src/utils/cpputils/transform.cc b/src/utils/cpputils/transform.cc +index 51c154fb..ba8c1f7a 100644 +--- a/src/utils/cpputils/transform.cc ++++ b/src/utils/cpputils/transform.cc +@@ -97,7 +97,7 @@ auto RepeatedPtrFieldToCharArray(const google::protobuf::RepeatedPtrField +Date: Thu, 15 Aug 2024 19:21:19 +1400 +Subject: [PATCH 120/121] get realpath before ns mountpoint verification + +Signed-off-by: zhongtao +--- + .../entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc | 9 +++++++-- + .../entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc | 9 +++++++-- + 2 files changed, 14 insertions(+), 4 deletions(-) + +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +index 77faf48a..3ece885f 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +@@ -424,6 +424,7 @@ cleanup_sandbox: + + void PodSandboxManagerService::ClearCniNetwork(const std::shared_ptr sandbox, Errors &error) + { ++ char real_path[PATH_MAX] = { 0 }; + std::string networkMode = sandbox->GetNetMode(); + if (!namespace_is_cni(networkMode.c_str()) || !sandbox->GetNetworkReady()) { + return; +@@ -435,10 +436,14 @@ void PodSandboxManagerService::ClearCniNetwork(const std::shared_ptr int + { + Errors networkErr; ++ char real_path[PATH_MAX] = { 0 }; + + bool ready = GetNetworkReady(realSandboxID, networkErr); + if (hostNetwork || (!ready && networkErr.Empty())) { +@@ -848,10 +849,14 @@ auto PodSandboxManagerService::ClearCniNetwork(const std::string &realSandboxID, + goto cleanup; + } + ++ if (realpath(netnsPath.c_str(), real_path) == NULL) { ++ ERROR("Failed to get %s realpath", netnsPath.c_str()); ++ } ++ + // If the network namespace is not mounted, the network has been cleaned up + // and there is no need to call the cni plugin. +- if (!util_detect_mounted(netnsPath.c_str())) { +- WARN("Network namespace %s not exist", netnsPath.c_str()); ++ if (strlen(real_path) != 0 && !util_detect_mounted(real_path)) { ++ ERROR("Network namespace %s not exist", real_path); + goto cleanup; + } + +-- +2.25.1 + diff --git a/0121-nri-impl-for-nri-plugin-and-adaption.patch b/0121-nri-impl-for-nri-plugin-and-adaption.patch new file mode 100644 index 0000000..fc13950 --- /dev/null +++ b/0121-nri-impl-for-nri-plugin-and-adaption.patch @@ -0,0 +1,4724 @@ +From 343be112a3d5cfd3857f166fb5dc2473534e64d7 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Thu, 15 Aug 2024 15:19:25 +0800 +Subject: [PATCH 121/121] [nri] impl for nri plugin and adaption + +Signed-off-by: zhongtao +--- + src/cmd/isulad/main.c | 15 + + src/daemon/common/nri/nri_convert.cc | 148 ++- + src/daemon/common/nri/nri_convert.h | 7 +- + src/daemon/common/nri/nri_spec.c | 223 +++- + src/daemon/common/nri/nri_spec.h | 6 + + src/daemon/common/nri/nri_utils.c | 476 +++---- + src/daemon/common/nri/nri_utils.h | 14 +- + src/daemon/config/isulad_config.c | 2 +- + .../v1/v1_cri_container_manager_service.cc | 104 +- + .../v1/v1_cri_pod_sandbox_manager_service.cc | 37 +- + .../executor/container_cb/execution_create.c | 21 + + .../modules/service/service_container.c | 12 + + src/daemon/modules/spec/specs.c | 7 +- + src/daemon/nri/nri_adaption.cc | 1165 +++++++++++++++++ + src/daemon/nri/nri_adaption.h | 13 +- + src/daemon/nri/nri_helpers.cc | 21 + + src/daemon/nri/nri_helpers.h | 4 + + src/daemon/nri/nri_plugin_ops.cc | 123 ++ + src/daemon/nri/nri_plugin_ops.h | 6 +- + src/daemon/nri/nri_result.cc | 977 ++++++++++++++ + src/daemon/nri/nri_result.h | 17 +- + src/daemon/nri/plugin.cc | 417 ++++++ + src/daemon/nri/plugin.h | 20 +- + src/utils/cutils/utils.c | 2 +- + 24 files changed, 3527 insertions(+), 310 deletions(-) + create mode 100644 src/daemon/nri/nri_adaption.cc + create mode 100644 src/daemon/nri/nri_plugin_ops.cc + create mode 100644 src/daemon/nri/nri_result.cc + create mode 100644 src/daemon/nri/plugin.cc + +diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c +index 52ac3172..0228caa8 100644 +--- a/src/cmd/isulad/main.c ++++ b/src/cmd/isulad/main.c +@@ -86,6 +86,9 @@ + #ifdef ENABLE_CDI + #include "cdi_operate_api.h" + #endif /* ENABLE_CDI */ ++#ifdef ENABLE_NRI ++#include "nri_plugin_ops.h" ++#endif + + sem_t g_daemon_shutdown_sem; + sem_t g_daemon_wait_shutdown_sem; +@@ -289,6 +292,11 @@ static void daemon_shutdown() + EVENT("Network module exit completed"); + #endif + ++#ifdef ENABLE_NRI ++ nri_adaption_shutdown(); ++ EVENT("nri module exit completed"); ++#endif ++ + clean_residual_files(); + EVENT("Clean residual files completed"); + +@@ -1834,6 +1842,13 @@ int main(int argc, char **argv) + } + #endif + ++#ifdef ENABLE_NRI ++ if (!nri_adaption_init()) { ++ ERROR("Failed to init nri adaption"); ++ goto failure; ++ } ++#endif ++ + clock_gettime(CLOCK_MONOTONIC, &t_end); + use_time = (double)(t_end.tv_sec - t_start.tv_sec) * (double)1000000000 + (double)(t_end.tv_nsec - t_start.tv_nsec); + use_time /= 1000000000; +diff --git a/src/daemon/common/nri/nri_convert.cc b/src/daemon/common/nri/nri_convert.cc +index 7cce64ec..30caf1dd 100644 +--- a/src/daemon/common/nri/nri_convert.cc ++++ b/src/daemon/common/nri/nri_convert.cc +@@ -20,6 +20,7 @@ + #include "path.h" + #include "transform.h" + #include "nri_utils.h" ++#include "cstruct_wrapper.h" + + static int64_t DefaultOOMScoreAdj = 0; + +@@ -142,22 +143,28 @@ static auto NRILinuxResourcesFromCRI(const runtime::v1::LinuxContainerResources + + static auto NRILinuxFromCRI(const runtime::v1::LinuxPodSandboxConfig &config, nri_linux_pod_sandbox &linux) -> bool + { +- if (!init_nri_linux_resources(&linux.pod_overhead)) { +- ERROR("Failed to init nri linux overhead resources for pod"); +- return false; +- } +- if (!init_nri_linux_resources(&linux.pod_resources)) { +- ERROR("Failed to init nri linux resources resources for pod"); +- return false; +- } +- if (config.has_overhead() && !NRILinuxResourcesFromCRI(config.overhead(), *linux.pod_overhead)) { +- ERROR("Failed to transform overhead to nri for pod"); +- return false; ++ if (config.has_overhead()) { ++ linux.pod_overhead = init_nri_linux_resources(); ++ if (linux.pod_overhead == nullptr) { ++ ERROR("Failed to init nri linux overhead resources for pod"); ++ return false; ++ } ++ if (!NRILinuxResourcesFromCRI(config.overhead(), *linux.pod_overhead)) { ++ ERROR("Failed to transform overhead to nri for pod"); ++ return false; ++ } + } + +- if (config.has_resources() && !NRILinuxResourcesFromCRI(config.resources(), *linux.pod_resources)) { +- ERROR("Failed to transform resources to nri for pod"); +- return false; ++ if (config.has_resources()) { ++ linux.pod_resources = init_nri_linux_resources(); ++ if (linux.pod_resources == nullptr) { ++ ERROR("Failed to init nri linux resources resources for pod"); ++ return false; ++ } ++ if (!NRILinuxResourcesFromCRI(config.resources(), *linux.pod_resources)) { ++ ERROR("Failed to transform resources to nri for pod"); ++ return false; ++ } + } + + linux.cgroup_parent = util_strdup_s(config.cgroup_parent().c_str()); +@@ -507,13 +514,13 @@ auto LinuxResourcesFromNRI(const nri_linux_resources *src, runtime::v1::LinuxCon + } + + if (src->cpu != nullptr) { +- if (src->cpu->shares != NULL) { ++ if (src->cpu->shares != nullptr) { + resources.set_cpu_shares(*src->cpu->shares); + } +- if (src->cpu->quota != NULL) { ++ if (src->cpu->quota != nullptr) { + resources.set_cpu_quota(*src->cpu->quota); + } +- if (src->cpu->period != NULL) { ++ if (src->cpu->period != nullptr) { + resources.set_cpu_period(*src->cpu->period); + } + +@@ -535,5 +542,112 @@ auto LinuxResourcesFromNRI(const nri_linux_resources *src, runtime::v1::LinuxCon + Transform::JsonMapToProtobufMapForString(src->unified, *resources.mutable_unified()); + } + ++ return true; ++} ++ ++auto LinuxResourcesToNRI(const runtime::v1::LinuxContainerResources &src) -> nri_linux_resources * ++{ ++ nri_linux_resources *resources = nullptr; ++ ++ resources = init_nri_linux_resources(); ++ if (resources == nullptr) { ++ ERROR("Failed to init nri linux resources"); ++ return nullptr; ++ } ++ ++ resources->cpu->shares = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if (resources->cpu->shares == nullptr) { ++ ERROR("Out of memory"); ++ goto error_out; ++ } ++ *(resources->cpu->shares) = src.cpu_shares(); ++ ++ resources->cpu->quota = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (resources->cpu->quota == nullptr) { ++ ERROR("Out of memory"); ++ goto error_out; ++ } ++ *(resources->cpu->quota) = src.cpu_quota(); ++ ++ resources->cpu->period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if (resources->cpu->period == nullptr) { ++ ERROR("Out of memory"); ++ goto error_out; ++ } ++ *(resources->cpu->period) = src.cpu_period(); ++ ++ resources->cpu->cpus = util_strdup_s(src.cpuset_cpus().c_str()); ++ resources->cpu->mems = util_strdup_s(src.cpuset_mems().c_str()); ++ ++ resources->memory->limit = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (resources->memory->limit == nullptr) { ++ ERROR("Out of memory"); ++ goto error_out; ++ } ++ *(resources->memory->limit) = src.memory_limit_in_bytes(); ++ ++ resources->hugepage_limits = (nri_hugepage_limit **)util_smart_calloc_s(sizeof(nri_hugepage_limit *), ++ src.hugepage_limits_size()); ++ if (resources->hugepage_limits == nullptr) { ++ ERROR("Out of memory"); ++ goto error_out; ++ } ++ ++ for (int i = 0; i < src.hugepage_limits_size(); i++) { ++ resources->hugepage_limits[i] = (nri_hugepage_limit *)util_common_calloc_s(sizeof(nri_hugepage_limit)); ++ if (resources->hugepage_limits[i] == nullptr) { ++ ERROR("Out of memory"); ++ goto error_out; ++ } ++ resources->hugepage_limits[i]->page_size = util_strdup_s(src.hugepage_limits(i).page_size().c_str()); ++ resources->hugepage_limits[i]->limit = src.hugepage_limits(i).limit(); ++ resources->hugepage_limits_len++; ++ } ++ ++ return resources; ++ ++error_out: ++ free_nri_linux_resources(resources); ++ resources = nullptr; ++ return resources; ++} ++ ++auto PodSandboxesToNRI(const std::vector> &arrs, ++ std::vector &pods) -> bool ++{ ++ size_t i = 0; ++ for (i = 0; i < arrs.size(); i++) { ++ nri_pod_sandbox *pod = (nri_pod_sandbox *)util_common_calloc_s(sizeof(nri_pod_sandbox)); ++ if (pod == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ if (!PodSandboxToNRI(arrs[i], *pod)) { ++ ERROR("Failed to transform pod to nri for pod : %s", arrs[i]->GetName().c_str()); ++ return false; ++ } ++ pods.push_back(pod); ++ } ++ ++ return true; ++} ++ ++auto ContainersToNRI(std::vector> &containers, ++ std::vector &cons) -> bool ++{ ++ size_t i = 0; ++ for (i = 0; i < containers.size(); i++) { ++ nri_container *con = (nri_container *)util_common_calloc_s(sizeof(nri_container)); ++ if (con == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ if (!ContainerToNRIByID(containers[i].get()->id(), *con)) { ++ ERROR("Failed to transform container to nri for container : %s", containers[i]->metadata().name().c_str()); ++ return false; ++ } ++ cons.push_back(con); ++ } ++ + return true; + } +\ No newline at end of file +diff --git a/src/daemon/common/nri/nri_convert.h b/src/daemon/common/nri/nri_convert.h +index c04b14e4..a0caf6ce 100644 +--- a/src/daemon/common/nri/nri_convert.h ++++ b/src/daemon/common/nri/nri_convert.h +@@ -30,8 +30,11 @@ + auto PodSandboxToNRI(const std::shared_ptr &sandbox, nri_pod_sandbox &pod) -> bool; + auto ContainerToNRIByConConfig(const runtime::v1::ContainerConfig &containerConfig, nri_container &con) -> bool; + auto ContainerToNRIByID(const std::string &id, nri_container &con) -> bool; +-auto PodSandboxesToNRI(const std::vector> &arrs, nri_pod_sandbox **pod, +- int pod_len) -> bool; ++auto PodSandboxesToNRI(const std::vector> &arrs, ++ std::vector &pods) -> bool; ++auto ContainersToNRI(std::vector> &containers, ++ std::vector &cons) -> bool; + + auto LinuxResourcesFromNRI(const nri_linux_resources *src, runtime::v1::LinuxContainerResources &resources) -> bool; ++auto LinuxResourcesToNRI(const runtime::v1::LinuxContainerResources &src) -> nri_linux_resources *; + #endif // DAEMON_COMMON_NRI_NRI_CONVERT_H +diff --git a/src/daemon/common/nri/nri_spec.c b/src/daemon/common/nri/nri_spec.c +index 855fe3b3..327d31e1 100644 +--- a/src/daemon/common/nri/nri_spec.c ++++ b/src/daemon/common/nri/nri_spec.c +@@ -176,6 +176,7 @@ static int nri_adjust_annotation(const nri_container_adjustment *adjust, oci_run + + free_json_map_string_string(oci_spec->annotations); + oci_spec->annotations = cleard; ++ cleard = NULL; + ret = 0; + + free_out: +@@ -330,7 +331,34 @@ static int nri_adjust_hooks(const nri_container_adjustment *adjust, oci_runtime_ + return ret; + } + +-static int nri_adjust_devices(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++static int host_spec_add_device(host_config *spec, defs_device *device) ++{ ++ size_t i; ++ ++ if (device == NULL) { ++ return -1; ++ } ++ ++ for (i = 0; i < spec->nri_devices_len; i++) { ++ if (strcmp(spec->nri_devices[i]->path, device->path) == 0) { ++ free_defs_device(spec->nri_devices[i]); ++ spec->nri_devices[i] = device; ++ return 0; ++ } ++ } ++ ++ if (util_mem_realloc((void **)&spec->nri_devices, (spec->nri_devices_len + 1) * sizeof(defs_device *), ++ (void *)spec->nri_devices, spec->nri_devices_len * sizeof(defs_device *)) != 0) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ spec->nri_devices[spec->nri_devices_len] = device; ++ spec->nri_devices_len++; ++ ++ return 0; ++} ++ ++static int nri_adjust_oci_spec_devices(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) + { + if (adjust->linux == NULL || adjust->linux->devices == NULL || adjust->linux->devices_len == 0) { + return 0; +@@ -349,6 +377,25 @@ static int nri_adjust_devices(const nri_container_adjustment *adjust, oci_runtim + return 0; + } + ++static int nri_adjust_host_spec_devices(const nri_container_adjustment *adjust, host_config *spec) ++{ ++ if (adjust->linux == NULL || adjust->linux->devices == NULL || adjust->linux->devices_len == 0) { ++ return 0; ++ } ++ ++ size_t i; ++ ++ for (i = 0; i < adjust->linux->devices_len; i++) { ++ nri_linux_device *dev = adjust->linux->devices[i]; ++ if (host_spec_add_device(spec, nri_device_to_oci(dev)) != 0) { ++ ERROR("Failed to add device %s", dev->path); ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ + static int nri_adjust_cgroup_path(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) + { + if (adjust->linux == NULL || adjust->linux->cgroups_path == NULL) { +@@ -500,7 +547,50 @@ static int nri_adjust_mounts(const nri_container_adjustment *adjust, oci_runtime + return 0; + } + +-static int nri_adjust_rlimit(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++static int host_spec_add_linux_resources_rlimit(host_config *spec, const char *type, uint64_t hard, uint64_t soft) ++{ ++ size_t j; ++ bool exists = false; ++ host_config_nri_rlimits_element *rlimit = NULL; ++ ++ if (spec == NULL || type == NULL) { ++ ERROR("Invalid arguments"); ++ return -1; ++ } ++ ++ rlimit = util_common_calloc_s(sizeof(host_config_nri_rlimits_element)); ++ if (rlimit == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ rlimit->type = util_strdup_s(type); ++ rlimit->hard = hard; ++ rlimit->soft = soft; ++ ++ for (j = 0; j < spec->nri_rlimits_len; j++) { ++ if (spec->nri_rlimits[j]->type == NULL) { ++ ERROR("rlimit type is empty"); ++ free_host_config_nri_rlimits_element(rlimit); ++ return -1; ++ } ++ if (strcmp(spec->nri_rlimits[j]->type, rlimit->type) == 0) { ++ exists = true; ++ break; ++ } ++ } ++ if (exists) { ++ /* override ulimit */ ++ free_host_config_nri_rlimits_element(spec->nri_rlimits[j]); ++ spec->nri_rlimits[j] = rlimit; ++ } else { ++ spec->nri_rlimits[spec->nri_rlimits_len] = rlimit; ++ spec->nri_rlimits_len++; ++ } ++ ++ return 0; ++} ++ ++static int nri_adjust_oci_spec_rlimit(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) + { + if (adjust->rlimits == NULL || adjust->rlimits_len == 0) { + return 0; +@@ -522,6 +612,36 @@ static int nri_adjust_rlimit(const nri_container_adjustment *adjust, oci_runtime + return 0; + } + ++static int nri_adjust_host_spec_rlimit(const nri_container_adjustment *adjust, host_config *spec) ++{ ++ if (adjust->rlimits == NULL || adjust->rlimits_len == 0) { ++ return 0; ++ } ++ ++ if (spec->nri_rlimits == NULL) { ++ spec->nri_rlimits = util_common_calloc_s(sizeof(host_config_nri_rlimits_element *) * adjust->rlimits_len); ++ if (spec->nri_rlimits == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ } ++ ++ size_t i; ++ for (i = 0; i < adjust->rlimits_len; i++) { ++ nri_posix_rlimit *rlimit = adjust->rlimits[i]; ++ if (rlimit->type == NULL) { ++ ERROR("Invalid rlimit type"); ++ return -1; ++ } ++ if (host_spec_add_linux_resources_rlimit(spec, rlimit->type, rlimit->soft, rlimit->hard) != 0) { ++ ERROR("Failed to add rlimit"); ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ + // todo: we do not support it blockio_class + static int nri_adjust_blockio_class(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) + { +@@ -554,7 +674,7 @@ int nri_adjust_oci_spec(const nri_container_adjustment *adjust, oci_runtime_spec + return -1; + } + +- if (nri_adjust_devices(adjust, oci_spec) != 0) { ++ if (nri_adjust_oci_spec_devices(adjust, oci_spec) != 0) { + ERROR("Failed to do nri adjust devices in oci spec"); + return -1; + } +@@ -580,7 +700,7 @@ int nri_adjust_oci_spec(const nri_container_adjustment *adjust, oci_runtime_spec + return -1; + } + +- if (nri_adjust_rlimit(adjust, oci_spec) != 0) { ++ if (nri_adjust_oci_spec_rlimit(adjust, oci_spec) != 0) { + ERROR("Failed to do nri adjust rlimit in oci spec"); + return -1; + } +@@ -598,5 +718,100 @@ int nri_adjust_oci_spec(const nri_container_adjustment *adjust, oci_runtime_spec + return -1; + } + ++ return 0; ++} ++ ++int nri_adjust_host_spec(const nri_container_adjustment *adjust, host_config *host_spec) ++{ ++ if (adjust == NULL || host_spec == NULL) { ++ ERROR("Invalid input arguments"); ++ return -1; ++ } ++ ++ if (nri_adjust_host_spec_devices(adjust, host_spec) != 0) { ++ ERROR("Failed to do nri adjust devices in host config"); ++ return -1; ++ } ++ ++ if (nri_adjust_host_spec_rlimit(adjust, host_spec) != 0) { ++ ERROR("Failed to do nri adjust rlimit in host config"); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++static defs_device *copy_def_devices(const defs_device *dev) ++{ ++ defs_device *tmp_dev = util_common_calloc_s(sizeof(defs_device)); ++ if (tmp_dev == NULL) { ++ return NULL; ++ } ++ tmp_dev->type = util_strdup_s(dev->type); ++ tmp_dev->path = util_strdup_s(dev->path); ++ tmp_dev->file_mode = dev->file_mode; ++ tmp_dev->major = dev->major; ++ tmp_dev->minor = dev->minor; ++ tmp_dev->uid = dev->uid; ++ tmp_dev->gid = dev->gid; ++ return tmp_dev; ++} ++ ++static int merge_nri_devices(oci_runtime_spec *oci_spec, host_config *host_spec) ++{ ++ size_t i; ++ ++ if (oci_spec == NULL || host_spec == NULL) { ++ ERROR("Invalid input arguments"); ++ return -1; ++ } ++ ++ for (i = 0; i < host_spec->nri_devices_len; i++) { ++ if (spec_add_device(oci_spec, copy_def_devices(host_spec->nri_devices[i])) != 0) { ++ ERROR("Failed to add device %s", host_spec->nri_devices[i]->path); ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ ++static int merge_nri_ulimits(oci_runtime_spec *oci_spec, host_config *host_spec) ++{ ++ size_t i; ++ ++ if (oci_spec == NULL || host_spec == NULL) { ++ ERROR("Invalid input arguments"); ++ return -1; ++ } ++ ++ for (i = 0; i < host_spec->nri_rlimits_len; i++) { ++ host_config_nri_rlimits_element *rlimit = host_spec->nri_rlimits[i]; ++ if (spec_add_linux_resources_rlimit(oci_spec, rlimit->type, rlimit->hard, rlimit->soft) != 0) { ++ ERROR("Failed to add rlimit %s", rlimit->type); ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ ++int update_oci_nri(oci_runtime_spec *oci_spec, host_config *host_spec) ++{ ++ if (oci_spec == NULL || host_spec == NULL) { ++ ERROR("Invalid input arguments"); ++ return -1; ++ } ++ ++ if (merge_nri_devices(oci_spec, host_spec) != 0) { ++ ERROR("Failed to merge nri devices"); ++ return -1; ++ } ++ ++ if (merge_nri_ulimits(oci_spec, host_spec) != 0) { ++ ERROR("Failed to merge nri ulimits"); ++ return -1; ++ } ++ + return 0; + } +\ No newline at end of file +diff --git a/src/daemon/common/nri/nri_spec.h b/src/daemon/common/nri/nri_spec.h +index e7c5035d..1a622284 100644 +--- a/src/daemon/common/nri/nri_spec.h ++++ b/src/daemon/common/nri/nri_spec.h +@@ -18,7 +18,13 @@ + + #include + #include ++#include + + int nri_adjust_oci_spec(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec); + ++// the device and ulimit will be updated when starting, so they need to be stored in host-spec ++int nri_adjust_host_spec(const nri_container_adjustment *adjust, host_config *host_spec); ++ ++int update_oci_nri(oci_runtime_spec *oci_spec, host_config *host_spec); ++ + #endif // DAEMON_COMMON_NRI_NRI_SPEC_H +\ No newline at end of file +diff --git a/src/daemon/common/nri/nri_utils.c b/src/daemon/common/nri/nri_utils.c +index 51054e32..d2b62117 100644 +--- a/src/daemon/common/nri/nri_utils.c ++++ b/src/daemon/common/nri/nri_utils.c +@@ -19,218 +19,235 @@ + + #include "utils.h" + +-static bool copy_nri_hugepage_limit(const nri_hugepage_limit* src, nri_hugepage_limit** dest) ++static nri_hugepage_limit*copy_nri_hugepage_limit(const nri_hugepage_limit *src) + { +- if (src == NULL || dest == NULL) { ++ nri_hugepage_limit *dest = NULL; ++ if (src == NULL) { + ERROR("Invalid input arguments"); + return false; + } + +- *dest = (nri_hugepage_limit *)util_common_calloc_s(sizeof(nri_hugepage_limit)); +- if (*dest == NULL) { ++ dest = (nri_hugepage_limit *)util_common_calloc_s(sizeof(nri_hugepage_limit)); ++ if (dest == NULL) { + ERROR("Out of memory"); + return false; + } + +- (*dest)->limit = src->limit; +- (*dest)->page_size = util_strdup_s(src->page_size); +- return true; ++ dest->limit = src->limit; ++ dest->page_size = util_strdup_s(src->page_size); ++ return dest; + } + +-static bool copy_nri_hook(const nri_hook *src, nri_hook **dest) ++static nri_hook *copy_nri_hook(const nri_hook *src) + { +- if (src == NULL || dest == NULL) { ++ nri_hook *dest = NULL; ++ if (src == NULL) { + ERROR("Invalid input arguments"); + return false; + } + +- *dest = (nri_hook *)util_common_calloc_s(sizeof(nri_hook)); ++ dest = (nri_hook *)util_common_calloc_s(sizeof(nri_hook)); + if (dest == NULL) { + ERROR("Out of memory"); + return false; + } +- (*dest)->args = util_copy_array_by_len(src->args, src->args_len); +- (*dest)->args_len = src->args_len; +- (*dest)->env = util_copy_array_by_len(src->env, src->env_len); +- (*dest)->env_len = src->env_len; +- (*dest)->path = util_strdup_s(src->path); +- return true; ++ dest->args = util_copy_array_by_len(src->args, src->args_len); ++ dest->args_len = src->args_len; ++ dest->env = util_copy_array_by_len(src->env, src->env_len); ++ dest->env_len = src->env_len; ++ dest->path = util_strdup_s(src->path); ++ return dest; + } + +-static bool copy_nri_linux_device_cgroup(const nri_linux_device_cgroup *src, nri_linux_device_cgroup **dest) ++static nri_linux_device_cgroup *copy_nri_linux_device_cgroup(const nri_linux_device_cgroup *src) + { +- if (src == NULL || dest == NULL) { ++ nri_linux_device_cgroup *dest = NULL; ++ if (src == NULL) { + ERROR("Invalid input arguments"); + return false; + } + +- *dest = (nri_linux_device_cgroup *)util_common_calloc_s(sizeof(nri_linux_device_cgroup)); ++ dest = (nri_linux_device_cgroup *)util_common_calloc_s(sizeof(nri_linux_device_cgroup)); + if (dest == NULL) { + ERROR("Out of memory"); + return false; + } +- (*dest)->allow = src->allow; +- (*dest)->type = util_strdup_s(src->type); +- (*dest)->major = (int64_t *)util_common_calloc_s(sizeof(int64_t)); +- if ((*dest)->major == NULL) { ++ dest->allow = src->allow; ++ dest->type = util_strdup_s(src->type); ++ dest->major = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (dest->major == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- (*dest)->minor = (int64_t *)util_common_calloc_s(sizeof(int64_t)); +- if ((*dest)->minor == NULL) { ++ dest->minor = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (dest->minor == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- (*dest)->access = util_strdup_s(src->access); +- return true; ++ dest->access = util_strdup_s(src->access); ++ return dest; ++free_out: ++ free_nri_linux_device_cgroup(dest); ++ return NULL; + } + +-static bool copy_nri_linux_cpu(const nri_linux_cpu *src, nri_linux_cpu **dest) ++static nri_linux_cpu *copy_nri_linux_cpu(const nri_linux_cpu *src) + { +- if (src == NULL || dest == NULL) { ++ nri_linux_cpu *dest = NULL; ++ if (src == NULL) { + ERROR("Invalid input arguments"); + return false; + } + +- (*dest) = (nri_linux_cpu *)util_common_calloc_s(sizeof(nri_linux_cpu)); +- if ((*dest) == NULL) { ++ dest = (nri_linux_cpu *)util_common_calloc_s(sizeof(nri_linux_cpu)); ++ if (dest == NULL) { + ERROR("Out of memory"); + return false; + } +- (*dest)->cpus = util_strdup_s(src->cpus); +- (*dest)->mems = util_strdup_s(src->mems); ++ dest->cpus = util_strdup_s(src->cpus); ++ dest->mems = util_strdup_s(src->mems); + if (src->period != NULL) { +- (*dest)->period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); +- if ((*dest)->period == NULL) { ++ dest->period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if (dest->period == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->period = *src->period; ++ *dest->period = *src->period; + } + + if (src->quota != NULL) { +- (*dest)->quota = (int64_t *)util_common_calloc_s(sizeof(int64_t)); +- if ((*dest)->quota == NULL) { ++ dest->quota = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (dest->quota == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->quota = *src->quota; ++ *dest->quota = *src->quota; + } + + if (src->realtime_period != NULL) { +- (*dest)->realtime_period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); +- if ((*dest)->realtime_period == NULL) { ++ dest->realtime_period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if (dest->realtime_period == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->realtime_period = *src->realtime_period; ++ *dest->realtime_period = *src->realtime_period; + } + + if (src->realtime_runtime != NULL) { +- (*dest)->realtime_runtime = (int64_t *)util_common_calloc_s(sizeof(int64_t)); +- if ((*dest)->realtime_runtime == NULL) { ++ dest->realtime_runtime = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (dest->realtime_runtime == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->realtime_runtime = *src->realtime_runtime; ++ *dest->realtime_runtime = *src->realtime_runtime; + } + + if (src->shares != NULL) { +- (*dest)->shares = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); +- if ((*dest)->shares == NULL) { ++ dest->shares = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if (dest->shares == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->shares = *src->shares; ++ *dest->shares = *src->shares; + } + +- return true; ++ return dest; ++ ++free_out: ++ free_nri_linux_cpu(dest); ++ return NULL; + } + +-static bool copy_nri_linux_memory(const nri_linux_memory *src, nri_linux_memory **dest) ++static nri_linux_memory *copy_nri_linux_memory(const nri_linux_memory *src) + { +- if (src == NULL || dest == NULL) { ++ nri_linux_memory *dest = NULL; ++ if (src == NULL) { + ERROR("Invalid input arguments"); + return false; + } +- *dest = (nri_linux_memory *)util_common_calloc_s(sizeof(nri_linux_memory)); ++ ++ dest = (nri_linux_memory *)util_common_calloc_s(sizeof(nri_linux_memory)); + if (dest == NULL) { + ERROR("Out of memory"); + return false; + } + if (src->limit != NULL) { +- (*dest)->limit = (int64_t *)util_common_calloc_s(sizeof(int64_t)); +- if ((*dest)->limit == NULL) { ++ dest->limit = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (dest->limit == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->limit = *src->limit; ++ *dest->limit = *src->limit; + } + + if (src->reservation != NULL) { +- (*dest)->reservation = (int64_t *)util_common_calloc_s(sizeof(int64_t)); +- if ((*dest)->reservation == NULL) { ++ dest->reservation = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (dest->reservation == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->reservation = *src->reservation; ++ *dest->reservation = *src->reservation; + } + + if (src->swap != NULL) { +- (*dest)->swap = (int64_t *)util_common_calloc_s(sizeof(int64_t)); +- if ((*dest)->swap == NULL) { ++ dest->swap = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (dest->swap == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->swap = *src->swap; ++ *dest->swap = *src->swap; + } + + if (src->kernel != NULL) { +- (*dest)->kernel = (int64_t *)util_common_calloc_s(sizeof(int64_t)); +- if ((*dest)->kernel == NULL) { ++ dest->kernel = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (dest->kernel == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->kernel = *src->kernel; ++ *dest->kernel = *src->kernel; + } + + + if (src->kernel_tcp != NULL) { +- (*dest)->kernel_tcp = (int64_t *)util_common_calloc_s(sizeof(int64_t)); +- if ((*dest)->kernel_tcp == NULL) { ++ dest->kernel_tcp = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (dest->kernel_tcp == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->kernel_tcp = *src->kernel_tcp; ++ *dest->kernel_tcp = *src->kernel_tcp; + } + + if (src->swappiness != NULL) { +- (*dest)->swappiness = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); +- if ((*dest)->swappiness == NULL) { ++ dest->swappiness = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if (dest->swappiness == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->swappiness = *src->swappiness; ++ *dest->swappiness = *src->swappiness; + } + + if (src->disable_oom_killer != NULL) { +- (*dest)->disable_oom_killer = (uint8_t *)util_common_calloc_s(sizeof(uint8_t)); +- if ((*dest)->disable_oom_killer == NULL) { ++ dest->disable_oom_killer = (uint8_t *)util_common_calloc_s(sizeof(uint8_t)); ++ if (dest->disable_oom_killer == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->disable_oom_killer = *src->disable_oom_killer; ++ *dest->disable_oom_killer = *src->disable_oom_killer; + } + + if (src->use_hierarchy != NULL) { +- (*dest)->use_hierarchy = (uint8_t *)util_common_calloc_s(sizeof(uint8_t)); +- if ((*dest)->use_hierarchy == NULL) { ++ dest->use_hierarchy = (uint8_t *)util_common_calloc_s(sizeof(uint8_t)); ++ if (dest->use_hierarchy == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->use_hierarchy = *src->use_hierarchy; ++ *dest->use_hierarchy = *src->use_hierarchy; + } +- return true; ++ return dest; ++ ++free_out: ++ free_nri_linux_memory(dest); ++ return NULL; + } + + bool is_marked_for_removal(const char* key, char **out) +@@ -241,7 +258,7 @@ bool is_marked_for_removal(const char* key, char **out) + } + + if (!util_has_prefix(key, "-")) { +- *out = (char*)key; ++ *out = util_strdup_s(key); + return false; + } + +@@ -254,94 +271,153 @@ bool is_marked_for_removal(const char* key, char **out) + return true; + } + +-bool copy_nri_mount(const nri_mount *src, nri_mount **dest) ++nri_mount *copy_nri_mount(const nri_mount *src) + { +- if (src == NULL || dest == NULL) { ++ nri_mount *dest = NULL; ++ if (src == NULL) { + ERROR("Invalid input arguments"); + return false; + } +- *dest = (nri_mount *)util_common_calloc_s(sizeof(nri_mount)); ++ ++ dest = (nri_mount *)util_common_calloc_s(sizeof(nri_mount)); + if (dest == NULL) { + ERROR("Out of memory"); + return false; + } +- (*dest)->destination = util_strdup_s(src->destination); +- (*dest)->options = util_copy_array_by_len(src->options, src->options_len); +- (*dest)->options_len = src->options_len; +- (*dest)->source = util_strdup_s(src->source); +- (*dest)->type = util_strdup_s(src->type); +- return true; ++ dest->destination = util_strdup_s(src->destination); ++ dest->options = util_copy_array_by_len(src->options, src->options_len); ++ dest->options_len = src->options_len; ++ dest->source = util_strdup_s(src->source); ++ dest->type = util_strdup_s(src->type); ++ return dest; + } + +-bool copy_nri_key_value(const nri_key_value *src, nri_key_value **dest) ++nri_linux_device *copy_nri_device(const nri_linux_device *src) + { +- if (src == NULL || dest == NULL) { ++ nri_linux_device *dest = NULL; ++ ++ if (src == NULL) { + ERROR("Invalid input arguments"); + return false; + } +- *dest = (nri_key_value *)util_common_calloc_s(sizeof(nri_key_value)); ++ ++ dest = (nri_linux_device *)util_common_calloc_s(sizeof(nri_linux_device)); + if (dest == NULL) { + ERROR("Out of memory"); + return false; + } +- (*dest)->key = util_strdup_s(src->key); +- (*dest)->value = util_strdup_s(src->value); +- return true; ++ if (src->file_mode != NULL) { ++ dest->file_mode = (uint32_t *)util_common_calloc_s(sizeof(uint32_t)); ++ if (dest->file_mode == NULL) { ++ ERROR("Out of memory"); ++ goto free_out; ++ } ++ *dest->file_mode = *src->file_mode; ++ } ++ ++ if (src->uid != NULL) { ++ dest->uid = (uint32_t *)util_common_calloc_s(sizeof(uint32_t)); ++ if (dest->uid == NULL) { ++ ERROR("Out of memory"); ++ goto free_out; ++ } ++ *dest->uid = *src->uid; ++ } ++ ++ if (src->gid != NULL) { ++ dest->gid = (uint32_t *)util_common_calloc_s(sizeof(uint32_t)); ++ if (dest->gid == NULL) { ++ ERROR("Out of memory"); ++ goto free_out; ++ } ++ *dest->gid = *src->gid; ++ } ++ ++ dest->major = src->major; ++ dest->minor = src->minor; ++ dest->path = util_strdup_s(src->path); ++ dest->type = util_strdup_s(src->type); ++ ++ return dest; ++free_out: ++ free_nri_linux_device(dest); ++ return NULL; + } + +-bool copy_nri_posix_rlimit(const nri_posix_rlimit *src, nri_posix_rlimit **dest) ++nri_key_value *copy_nri_key_value(const nri_key_value *src) + { +- if (src == NULL || dest == NULL) { ++ nri_key_value *dest = NULL; ++ if (src == NULL) { + ERROR("Invalid input arguments"); + return false; + } +- *dest = (nri_posix_rlimit *)util_common_calloc_s(sizeof(nri_posix_rlimit)); ++ dest = (nri_key_value *)util_common_calloc_s(sizeof(nri_key_value)); + if (dest == NULL) { + ERROR("Out of memory"); + return false; + } +- (*dest)->hard = src->hard; +- (*dest)->soft = src->soft; +- (*dest)->type = util_strdup_s(src->type); +- return true; ++ dest->key = util_strdup_s(src->key); ++ dest->value = util_strdup_s(src->value); ++ return dest; + } + +-bool copy_nri_linux_resources(const nri_linux_resources *src, nri_linux_resources **dest) ++nri_posix_rlimit *copy_nri_posix_rlimit(const nri_posix_rlimit *src) + { +- if (src == NULL || dest == NULL) { ++ nri_posix_rlimit *dest = NULL; ++ if (src == NULL) { + ERROR("Invalid input arguments"); + return false; + } +- +- *dest = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); +- if (*dest == NULL) { ++ dest = (nri_posix_rlimit *)util_common_calloc_s(sizeof(nri_posix_rlimit)); ++ if (dest == NULL) { + ERROR("Out of memory"); + return false; + } ++ dest->hard = src->hard; ++ dest->soft = src->soft; ++ dest->type = util_strdup_s(src->type); ++ return dest; ++} + +- if (!init_nri_linux_resources(dest)) { +- ERROR("Failed to init dest nri linux resources"); +- goto free_out; ++nri_linux_resources *copy_nri_linux_resources(const nri_linux_resources *src) ++{ ++ nri_linux_resources *dest = NULL; ++ if (src == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; + } + +- if (!copy_nri_linux_cpu(src->cpu, &(*dest)->cpu)) { +- ERROR("Failed to copy nri_linux_cpu"); +- goto free_out; ++ dest = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); ++ if (dest == NULL) { ++ ERROR("Out of memory"); ++ return false; + } + +- if (!copy_nri_linux_memory(src->memory, &(*dest)->memory)) { +- ERROR("Failed to copy nri_linux_memory"); +- goto free_out; ++ if (src->cpu != NULL) { ++ dest->cpu = copy_nri_linux_cpu(src->cpu); ++ if (dest->cpu == NULL) { ++ ERROR("Failed to copy nri_linux_cpu"); ++ goto free_out; ++ } + } + +- (*dest)->blockio_class = util_strdup_s(src->blockio_class); +- (*dest)->rdt_class = util_strdup_s(src->rdt_class); ++ if (src->memory != NULL) { ++ dest->memory = copy_nri_linux_memory(src->memory); ++ if (dest->memory == NULL) { ++ ERROR("Failed to copy nri_linux_memory"); ++ goto free_out; ++ } ++ } ++ ++ dest->blockio_class = util_strdup_s(src->blockio_class); ++ dest->rdt_class = util_strdup_s(src->rdt_class); + + if (src->hugepage_limits_len > 0) { +- (*dest)->hugepage_limits = (nri_hugepage_limit**)util_smart_calloc_s(sizeof(nri_hugepage_limit*), ++ dest->hugepage_limits = (nri_hugepage_limit**)util_smart_calloc_s(sizeof(nri_hugepage_limit*), + src->hugepage_limits_len); +- for (size_t i = 0; i < src->hugepage_limits_len; ++i) { +- if (!copy_nri_hugepage_limit(src->hugepage_limits[i], &((*dest)->hugepage_limits[i]))) { ++ for (size_t i = 0; i < src->hugepage_limits_len; i++) { ++ dest->hugepage_limits[i] = copy_nri_hugepage_limit(src->hugepage_limits[i]); ++ if (dest->hugepage_limits[i] == NULL) { + ERROR("Failed to copy nri_hugepage_limit"); + goto free_out; + } +@@ -349,25 +425,26 @@ bool copy_nri_linux_resources(const nri_linux_resources *src, nri_linux_resource + } + + if (src->devices_len > 0) { +- (*dest)->devices = (nri_linux_device_cgroup**)util_smart_calloc_s(sizeof(nri_linux_device_cgroup*), src->devices_len); +- for (size_t i = 0; i < src->devices_len; ++i) { +- if (!copy_nri_linux_device_cgroup(src->devices[i], &((*dest)->devices[i]))) { ++ dest->devices = (nri_linux_device_cgroup**)util_smart_calloc_s(sizeof(nri_linux_device_cgroup*), src->devices_len); ++ for (size_t i = 0; i < src->devices_len; i++) { ++ dest->devices[i] = copy_nri_linux_device_cgroup(src->devices[i]); ++ if (dest->devices[i] == NULL) { + ERROR("Failed to copy nri_linux_device_cgroup"); + goto free_out; + } + } + } + +- if (dup_json_map_string_string(src->unified, (*dest)->unified)) { ++ if (dup_json_map_string_string(src->unified, dest->unified)) { + ERROR("Failed to copy json_map_string_string"); + goto free_out; + } + +- return true; ++ return dest; + + free_out: +- free_nri_linux_resources(*dest); +- return false; ++ free_nri_linux_resources(dest); ++ return NULL; + } + + bool merge_nri_hooks(nri_hook **targetHooks, size_t targetSize, const nri_hook **sourceHooks, +@@ -381,140 +458,67 @@ bool merge_nri_hooks(nri_hook **targetHooks, size_t targetSize, const nri_hook * + return false; + } + +- if (util_mem_realloc((void**)&targetHooks, newSize, (void**)&targetHooks, oldSize) != 0) { ++ if (util_mem_realloc((void**)&targetHooks, newSize, targetHooks, oldSize) != 0) { + ERROR("Failed to realloc and assign hook array"); + return false; + } + + for (size_t i = 0; i < sourceLen; i++) { +- if (!copy_nri_hook(sourceHooks[i], &targetHooks[targetSize++])) { ++ targetHooks[targetSize] = copy_nri_hook(sourceHooks[i]); ++ if (targetHooks[targetSize] == NULL) { + ERROR("Failed to copy hook"); + return false; + } ++ targetSize++; + } + + return true; + } + +-bool init_nri_container_adjust(nri_container_adjustment **adjust) ++nri_container_update *init_nri_container_update(const char *id, const uint8_t ignore_failure) + { +- if (adjust == NULL) { ++ nri_container_update *update = NULL; ++ if (id == NULL) { + ERROR("Invalid input arguments"); + return false; + } + +- *adjust = (nri_container_adjustment *)util_common_calloc_s(sizeof(nri_container_adjustment)); +- if (*adjust == NULL) { ++ update = (nri_container_update *)util_common_calloc_s(sizeof(nri_container_update)); ++ if (update == NULL) { + ERROR("Out of memory"); + return false; + } + +- (*adjust)->annotations = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); +- if ((*adjust)->annotations == NULL) { +- goto free_out; +- } +- +- (*adjust)->env = (nri_key_value **)util_common_calloc_s(sizeof(nri_key_value *)); +- if ((*adjust)->env == NULL) { +- goto free_out; +- } +- (*adjust)->env_len = 0; +- +- (*adjust)->hooks = (nri_hooks *)util_common_calloc_s(sizeof(nri_hooks)); +- if ((*adjust)->hooks == NULL) { +- goto free_out; +- } +- +- (*adjust)->linux = (nri_linux_container_adjustment *)util_common_calloc_s(sizeof(nri_linux_container_adjustment)); +- if ((*adjust)->linux == NULL) { +- goto free_out; +- } +- +- (*adjust)->linux->resources = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); +- if ((*adjust)->linux->resources == NULL) { +- ERROR("Out of memory"); +- return false; +- } +- +- (*adjust)->mounts = (nri_mount **)util_common_calloc_s(sizeof(nri_mount *)); +- if ((*adjust)->mounts == NULL) { +- goto free_out; +- } +- (*adjust)->mounts_len = 0; +- +- (*adjust)->rlimits = (nri_posix_rlimit **)util_common_calloc_s(sizeof(nri_posix_rlimit *)); +- if ((*adjust)->rlimits == NULL) { +- goto free_out; +- } +- (*adjust)->rlimits_len = 0; ++ update->container_id = util_strdup_s(id); + +- return true; +- +-free_out: +- ERROR("Out of memory"); +- free_nri_container_adjustment(*adjust); +- return false; ++ update->ignore_failure = ignore_failure; ++ return update; + } + +-bool init_nri_container_update(nri_container_update **update, const char *id, uint8_t ignore_failure) ++nri_linux_resources *init_nri_linux_resources() + { +- if (update == NULL || id == NULL) { +- ERROR("Invalid input arguments"); +- return false; +- } +- +- *update = (nri_container_update *)util_common_calloc_s(sizeof(nri_container_update)); +- if (*update == NULL) { +- ERROR("Out of memory"); +- return false; +- } +- +- (*update)->container_id = util_strdup_s(id); +- (*update)->linux = (nri_linux_container_update *)util_common_calloc_s(sizeof(nri_linux_container_update)); +- if ((*update)->linux == NULL) { +- goto free_out; +- } ++ nri_linux_resources *resources = NULL; + +- (*update)->ignore_failure = ignore_failure; +- return true; +- +-free_out: +- ERROR("Out of memory"); +- free_nri_container_update(*update); +- return false; +-} +- +-bool init_nri_linux_resources(nri_linux_resources **resources) +-{ ++ resources = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); + if (resources == NULL) { +- ERROR("Invalid input arguments"); +- return false; +- } +- +- *resources = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); +- if (*resources == NULL) { + ERROR("Out of memory"); + return false; + } + +- (*resources)->cpu = (nri_linux_cpu *)util_common_calloc_s(sizeof(nri_linux_cpu)); +- if ((*resources)->cpu == NULL) { ++ resources->cpu = (nri_linux_cpu *)util_common_calloc_s(sizeof(nri_linux_cpu)); ++ if (resources->cpu == NULL) { + goto free_out; + } + +- (*resources)->memory = (nri_linux_memory *)util_common_calloc_s(sizeof(nri_linux_memory)); +- if ((*resources)->memory == NULL) { ++ resources->memory = (nri_linux_memory *)util_common_calloc_s(sizeof(nri_linux_memory)); ++ if (resources->memory == NULL) { + goto free_out; + } + +- (*resources)->unified = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); +- if ((*resources)->unified == NULL) { +- goto free_out; +- } +- return true; ++ return resources; + + free_out: + ERROR("Out of memory"); +- free_nri_linux_resources(*resources); +- return false; ++ free_nri_linux_resources(resources); ++ return NULL; + } +\ No newline at end of file +diff --git a/src/daemon/common/nri/nri_utils.h b/src/daemon/common/nri/nri_utils.h +index 7bf54a71..171055df 100644 +--- a/src/daemon/common/nri/nri_utils.h ++++ b/src/daemon/common/nri/nri_utils.h +@@ -49,19 +49,19 @@ typedef enum { + LAST = 12, + } NRI_Event; + +-bool copy_nri_mount(const nri_mount *src, nri_mount **dest); +-bool copy_nri_key_value(const nri_key_value *src, nri_key_value **dest); +-bool copy_nri_posix_rlimit(const nri_posix_rlimit *src, nri_posix_rlimit **dest); +-bool copy_nri_linux_resources(const nri_linux_resources *src, nri_linux_resources **dest); ++nri_mount *copy_nri_mount(const nri_mount *src); ++nri_posix_rlimit *copy_nri_posix_rlimit(const nri_posix_rlimit *src); ++nri_linux_resources *copy_nri_linux_resources(const nri_linux_resources *src); ++nri_key_value *copy_nri_key_value(const nri_key_value *src); ++nri_linux_device *copy_nri_device(const nri_linux_device *src); + + bool is_marked_for_removal(const char* key, char **out); + + bool merge_nri_hooks(nri_hook **targetHooks, size_t targetSize, const nri_hook **sourceHooks, + size_t sourceLen); + +-bool init_nri_container_adjust(nri_container_adjustment **adjust); +-bool init_nri_container_update(nri_container_update **update, const char *id, uint8_t ignore_failure); +-bool init_nri_linux_resources(nri_linux_resources **resources); ++nri_container_update *init_nri_container_update(const char *id, const uint8_t ignore_failure); ++nri_linux_resources *init_nri_linux_resources(void); + + #ifdef __cplusplus + } +diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c +index 9ba1c8a0..020c9f32 100644 +--- a/src/daemon/config/isulad_config.c ++++ b/src/daemon/config/isulad_config.c +@@ -1938,7 +1938,7 @@ int merge_json_confs_into_global(struct service_arguments *args) + override_string_value(&args->json_confs->plugin_path, &tmp_json_confs->plugin_path); + args->json_confs->plugin_registration_timeout = tmp_json_confs->plugin_registration_timeout; + args->json_confs->plugin_requst_timeout = tmp_json_confs->plugin_requst_timeout; +- override_string_value(&args->json_confs->nri_socket_path, &tmp_json_confs->nri_socket_path); ++ // Setting socket plugin path is not supported now + #endif + args->json_confs->enable_pod_events = tmp_json_confs->enable_pod_events; + #endif +diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +index 1cee68ec..d3fdd76a 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +@@ -27,6 +27,11 @@ + #include "stream_server.h" + #include "sandbox_manager.h" + ++#ifdef ENABLE_NRI ++#include "nri_adaption.h" ++#include "nri_helpers.h" ++#endif ++ + namespace CRIV1 { + auto ContainerManagerService::GetContainerOrSandboxRuntime(const std::string &realID, Errors &error) -> std::string + { +@@ -505,11 +510,24 @@ std::string ContainerManagerService::CreateContainer(const std::string &podSandb + return response_id; + } + ++#ifdef ENABLE_NRI ++ Errors nriErr; ++ nri_container_adjustment *adjust = NULL; ++ if (!NRIAdaptation::GetInstance()->CreateContainer(sandbox, response_id, containerConfig, &adjust, nriErr)) { ++ ERROR("Failed to get NRI adjustment for container: %s", nriErr.GetCMessage()); ++ NRIAdaptation::GetInstance()->UndoCreateContainer(sandbox, response_id, nriErr); ++ return response_id; ++ } ++#endif ++ + request = GenerateCreateContainerRequest(*sandbox, containerConfig, podSandboxConfig, error); + if (error.NotEmpty()) { + error.SetError("Failed to generate create container request"); + goto cleanup; + } ++#ifdef ENABLE_NRI ++ request->adjust = adjust; ++#endif + + if (m_cb->container.create(request, &response) != 0) { + if (response != nullptr && (response->errmsg != nullptr)) { +@@ -522,6 +540,12 @@ std::string ContainerManagerService::CreateContainer(const std::string &podSandb + + response_id = response->id; + ++#ifdef ENABLE_NRI ++ if (!NRIAdaptation::GetInstance()->PostCreateContainer(response_id, nriErr)) { ++ ERROR("NRI post-create notification failed: %s", nriErr.GetCMessage()); ++ } ++#endif ++ + cleanup: + free_container_create_request(request); + free_container_create_response(response); +@@ -530,6 +554,9 @@ cleanup: + + void ContainerManagerService::StartContainer(const std::string &containerID, Errors &error) + { ++#ifdef ENABLE_NRI ++ Errors nriErr; ++#endif + if (containerID.empty()) { + error.SetError("Invalid empty container id."); + return; +@@ -563,6 +590,14 @@ void ContainerManagerService::StartContainer(const std::string &containerID, Err + goto cleanup; + } + ++#ifdef ENABLE_NRI ++ if (!NRIAdaptation::GetInstance()->StartContainer(containerID, nriErr)) { ++ ERROR("NRI container start failed: %s", nriErr.GetCMessage()); ++ NRIAdaptation::GetInstance()->StopContainer(containerID, nriErr); ++ goto cleanup; ++ } ++#endif ++ + if (ret != 0) { + if (response != nullptr && response->errmsg != nullptr) { + error.SetError(response->errmsg); +@@ -571,6 +606,11 @@ void ContainerManagerService::StartContainer(const std::string &containerID, Err + } + goto cleanup; + } ++#ifdef ENABLE_NRI ++ if (!NRIAdaptation::GetInstance()->PostStartContainer(containerID, nriErr)) { ++ ERROR("NRI PostStartContainer notification failed: %s", nriErr.GetCMessage()); ++ } ++#endif + cleanup: + free_container_start_request(request); + free_container_start_response(response); +@@ -578,11 +618,25 @@ cleanup: + + void ContainerManagerService::StopContainer(const std::string &containerID, int64_t timeout, Errors &error) + { ++#ifdef ENABLE_NRI ++ Errors nriErr; ++#endif + CRIHelpers::StopContainer(m_cb, containerID, timeout, error); ++#ifdef ENABLE_NRI ++ if (!NRIAdaptation::GetInstance()->StopContainer(containerID, nriErr)) { ++ ERROR("NRI StopContainer notification failed: %s", nriErr.GetCMessage()); ++ } ++#endif + } + + void ContainerManagerService::RemoveContainer(const std::string &containerID, Errors &error) + { ++#ifdef ENABLE_NRI ++ Errors nriErr; ++ if (!NRIAdaptation::GetInstance()->RemoveContainer(containerID, nriErr)) { ++ ERROR("NRI RemoveContainer notification failed: %s", nriErr.GetCMessage()); ++ } ++#endif + CRIHelpers::RemoveContainer(m_cb, containerID, error); + if (error.NotEmpty()) { + WARN("Failed to remove container %s", containerID.c_str()); +@@ -1048,6 +1102,18 @@ void ContainerManagerService::UpdateContainerResources(const std::string &contai + struct parser_context ctx { + OPT_GEN_SIMPLIFY, 0 + }; ++ ++ runtime::v1::LinuxContainerResources updateRes = resources; ++#ifdef ENABLE_NRI ++ Errors nriErr; ++ runtime::v1::LinuxContainerResources adjust; ++ if (!NRIAdaptation::GetInstance()->UpdateContainer(containerID, resources, adjust, nriErr)) { ++ ERROR("NRI UpdateContainer notification failed: %s", nriErr.GetCMessage()); ++ goto cleanup; ++ } ++ updateRes = adjust; ++#endif ++ + request = (container_update_request *)util_common_calloc_s(sizeof(container_update_request)); + if (request == nullptr) { + error.SetError("Out of memory"); +@@ -1061,17 +1127,17 @@ void ContainerManagerService::UpdateContainerResources(const std::string &contai + goto cleanup; + } + +- hostconfig->cpu_period = resources.cpu_period(); +- hostconfig->cpu_quota = resources.cpu_quota(); +- hostconfig->cpu_shares = resources.cpu_shares(); ++ hostconfig->cpu_period = updateRes.cpu_period(); ++ hostconfig->cpu_quota = updateRes.cpu_quota(); ++ hostconfig->cpu_shares = updateRes.cpu_shares(); + +- if (!resources.unified().empty()) { ++ if (!updateRes.unified().empty()) { + hostconfig->unified = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); + if (hostconfig->unified == nullptr) { + error.SetError("Out of memory"); + goto cleanup; + } +- for (auto &iter : resources.unified()) { ++ for (auto &iter : updateRes.unified()) { + if (append_json_map_string_string(hostconfig->unified, iter.first.c_str(), iter.second.c_str()) != 0) { + error.SetError("Failed to append string"); + goto cleanup; +@@ -1079,30 +1145,30 @@ void ContainerManagerService::UpdateContainerResources(const std::string &contai + } + } + +- hostconfig->memory = resources.memory_limit_in_bytes(); +- hostconfig->memory_swap = resources.memory_swap_limit_in_bytes(); +- if (!resources.cpuset_cpus().empty()) { +- hostconfig->cpuset_cpus = util_strdup_s(resources.cpuset_cpus().c_str()); ++ hostconfig->memory = updateRes.memory_limit_in_bytes(); ++ hostconfig->memory_swap = updateRes.memory_swap_limit_in_bytes(); ++ if (!updateRes.cpuset_cpus().empty()) { ++ hostconfig->cpuset_cpus = util_strdup_s(updateRes.cpuset_cpus().c_str()); + } +- if (!resources.cpuset_mems().empty()) { +- hostconfig->cpuset_mems = util_strdup_s(resources.cpuset_mems().c_str()); ++ if (!updateRes.cpuset_mems().empty()) { ++ hostconfig->cpuset_mems = util_strdup_s(updateRes.cpuset_mems().c_str()); + } +- if (resources.hugepage_limits_size() != 0) { ++ if (updateRes.hugepage_limits_size() != 0) { + hostconfig->hugetlbs = (host_config_hugetlbs_element **)util_smart_calloc_s( +- sizeof(host_config_hugetlbs_element *), resources.hugepage_limits_size()); ++ sizeof(host_config_hugetlbs_element *), updateRes.hugepage_limits_size()); + if (hostconfig->hugetlbs == nullptr) { + error.SetError("Out of memory"); + goto cleanup; + } +- for (int i = 0; i < resources.hugepage_limits_size(); i++) { ++ for (int i = 0; i < updateRes.hugepage_limits_size(); i++) { + hostconfig->hugetlbs[i] = + (host_config_hugetlbs_element *)util_common_calloc_s(sizeof(host_config_hugetlbs_element)); + if (hostconfig->hugetlbs[i] == nullptr) { + error.SetError("Out of memory"); + goto cleanup; + } +- hostconfig->hugetlbs[i]->page_size = util_strdup_s(resources.hugepage_limits(i).page_size().c_str()); +- hostconfig->hugetlbs[i]->limit = resources.hugepage_limits(i).limit(); ++ hostconfig->hugetlbs[i]->page_size = util_strdup_s(updateRes.hugepage_limits(i).page_size().c_str()); ++ hostconfig->hugetlbs[i]->limit = updateRes.hugepage_limits(i).limit(); + hostconfig->hugetlbs_len++; + } + } +@@ -1121,6 +1187,12 @@ void ContainerManagerService::UpdateContainerResources(const std::string &contai + error.SetError("Failed to call update container callback"); + } + } ++#ifdef ENABLE_NRI ++ if (!NRIAdaptation::GetInstance()->PostUpdateContainer(containerID, nriErr)) { ++ ERROR("NRI PostUpdateContainer notification failed: %s", nriErr.GetCMessage()); ++ goto cleanup; ++ } ++#endif + cleanup: + free_container_update_request(request); + free_container_update_response(response); +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +index 77faf48a..0140eb99 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +@@ -37,6 +37,9 @@ + #include "transform.h" + #include "isulad_config.h" + #include "mailbox.h" ++#ifdef ENABLE_NRI ++#include "nri_adaption.h" ++#endif + + namespace CRIV1 { + void PodSandboxManagerService::PrepareSandboxData(const runtime::v1::PodSandboxConfig &config, +@@ -304,6 +307,9 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1::PodSandboxConfig + std::string network_setting_json; + runtime::v1::PodSandboxConfig copyConfig = config; + cri_container_message_t msg = { 0 }; ++#ifdef ENABLE_NRI ++ Errors nriErr; ++#endif + + // Step 1: Parepare sandbox name, runtime and networkMode + PrepareSandboxData(config, runtimeHandler, sandboxName, runtimeInfo, networkMode, error); +@@ -401,6 +407,14 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1::PodSandboxConfig + msg.type = CRI_CONTAINER_MESSAGE_TYPE_STARTED; + mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &msg); + ++#ifdef ENABLE_NRI ++ if (!NRIAdaptation::GetInstance()->RunPodSandbox(sandbox, nriErr)) { ++ ERROR("NRI RunPodSandbox failed: %s", nriErr.GetCMessage()); ++ error.Errorf("NRI RunPodSandbox failed: %s", nriErr.GetCMessage()); ++ return response_id; ++ } ++#endif ++ + return sandbox->GetId(); + + cleanup_network: +@@ -418,6 +432,11 @@ cleanup_sandbox: + if (error.NotEmpty()) { + ERROR("Failed to delete sandbox: %s", sandbox->GetId().c_str()); + } ++#ifdef ENABLE_NRI ++ if (!NRIAdaptation::GetInstance()->RemovePodSandbox(sandbox, nriErr)) { ++ DEBUG("NRI RemovePodSandbox failed: %s", nriErr.GetCMessage()); ++ } ++#endif + + return response_id; + } +@@ -618,7 +637,15 @@ void PodSandboxManagerService::StopPodSandbox(const std::string &podSandboxID, E + return; + } + +- sandbox->Stop(sandbox::DEFAULT_STOP_TIMEOUT, error); ++ if (!sandbox->Stop(sandbox::DEFAULT_STOP_TIMEOUT, error)) { ++ return; ++ } ++#ifdef ENABLE_NRI ++ Errors nriStopPodErr; ++ if (!NRIAdaptation::GetInstance()->StopPodSandbox(sandbox, nriStopPodErr)) { ++ ERROR("NRI sandbox stop notification failed %s", nriStopPodErr.GetCMessage()); ++ } ++#endif + } + + void PodSandboxManagerService::RemoveAllContainersInSandbox(const std::string &readSandboxID, +@@ -656,6 +683,9 @@ void PodSandboxManagerService::RemovePodSandbox(const std::string &podSandboxID, + { + std::vector errors; + std::string realSandboxID; ++#ifdef ENABLE_NRI ++ Errors nriErr; ++#endif + + if (m_cb == nullptr || m_cb->container.remove == nullptr) { + ERROR("Unimplemented callback"); +@@ -724,6 +754,11 @@ void PodSandboxManagerService::RemovePodSandbox(const std::string &podSandboxID, + msg.type = CRI_CONTAINER_MESSAGE_TYPE_DELETED; + mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &msg); + } ++#ifdef ENABLE_NRI ++ if (!NRIAdaptation::GetInstance()->RemovePodSandbox(sandbox, nriErr)) { ++ ERROR("NRI RemovePodSandbox failed: %s", nriErr.GetCMessage()); ++ } ++#endif + } + + auto PodSandboxManagerService::SharesHostNetwork(const container_inspect *inspect) -> runtime::v1::NamespaceMode +diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c +index 2b6b2a15..dcbdd1d3 100644 +--- a/src/daemon/executor/container_cb/execution_create.c ++++ b/src/daemon/executor/container_cb/execution_create.c +@@ -65,6 +65,10 @@ + #include "mailbox.h" + #include "specs_mount.h" + ++#ifdef ENABLE_NRI ++#include "nri_spec.h" ++#endif ++ + #ifdef ENABLE_CRI_API_V1 + static bool validate_sandbox_info(const container_sandbox_info *sandbox) + { +@@ -1465,6 +1469,14 @@ int container_create_cb(const container_create_request *request, container_creat + skip_sandbox_key_manage = (is_sandbox_container(request->sandbox) && namespace_is_cni(host_spec->network_mode)); + #endif + ++#ifdef ENABLE_NRI ++ if (request->adjust != NULL && nri_adjust_host_spec(request->adjust, host_spec) != 0) { ++ ERROR("Failed to adjust host spec"); ++ cc = ISULAD_ERR_INPUT; ++ goto clean_container_root_dir; ++ } ++#endif ++ + if (save_container_config_before_create(id, runtime_root, host_spec, v2_spec) != 0) { + ERROR("Failed to malloc container_config_v2_common_config"); + cc = ISULAD_ERR_INPUT; +@@ -1553,6 +1565,15 @@ int container_create_cb(const container_create_request *request, container_creat + } + #endif + ++#ifdef ENABLE_NRI ++ // modify oci spec by nri plugin ++ if (request->adjust != NULL && nri_adjust_oci_spec(request->adjust, oci_spec) != 0) { ++ ERROR("Failed to adjust oci spec"); ++ cc = ISULAD_ERR_EXEC; ++ goto clean_netns; ++ } ++#endif ++ + host_channel = dup_host_channel(host_spec->host_channel); + if (prepare_host_channel(host_channel, host_spec->user_remap)) { + ERROR("Failed to prepare host channel"); +diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c +index 0b95cdad..4157c631 100644 +--- a/src/daemon/modules/service/service_container.c ++++ b/src/daemon/modules/service/service_container.c +@@ -76,6 +76,9 @@ + #include "sandbox_ops.h" + #include "vsock_io_handler.h" + #endif ++#ifdef ENABLE_NRI ++#include "nri_spec.h" ++#endif + + #define KATA_RUNTIME "kata-runtime" + +@@ -726,6 +729,15 @@ static int do_oci_spec_update(const char *id, oci_runtime_spec *oci_spec, contai + return -1; + } + ++#ifdef ENABLE_NRI ++ // update oci spec with nri ++ ret = update_oci_nri(oci_spec, hostconfig); ++ if (ret != 0) { ++ ERROR("Failed to update oci spec with nri"); ++ return -1; ++ } ++#endif ++ + // renew_oci_config() will update process->user and share namespace after. + + return 0; +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index 002431d8..36e89343 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -2883,7 +2883,12 @@ int spec_add_linux_resources_rlimit(oci_runtime_spec *oci_spec, const char *type + return -1; + } + +- ret = make_sure_oci_spec_linux_resources(oci_spec); ++ ret = make_sure_oci_spec_process(oci_spec); ++ if (ret < 0) { ++ return -1; ++ } ++ ++ ret = merge_ulimits_pre(oci_spec, 1); + if (ret < 0) { + return -1; + } +diff --git a/src/daemon/nri/nri_adaption.cc b/src/daemon/nri/nri_adaption.cc +new file mode 100644 +index 00000000..3190767f +--- /dev/null ++++ b/src/daemon/nri/nri_adaption.cc +@@ -0,0 +1,1165 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-03-15 ++ * Description: provide plugin manager(NRI adaption) class definition ++ *********************************************************************************/ ++ ++#include "nri_adaption.h" ++ ++#include ++ ++#include ++#include ++ ++#include "isulad_config.h" ++#include "utils_file.h" ++#include "utils_string.h" ++#include "utils.h" ++#include "nri_convert.h" ++#include "nri_plugin.h" ++#include "nri_result.h" ++#include "sandbox_manager.h" ++ ++std::atomic NRIAdaptation::m_instance; ++ ++NRIAdaptation *NRIAdaptation::GetInstance() noexcept ++{ ++ static std::once_flag flag; ++ ++ std::call_once(flag, [] { m_instance = new NRIAdaptation; }); ++ ++ return m_instance; ++} ++ ++NRIAdaptation::~NRIAdaptation() ++{ ++ for (const auto &pair : m_storeMap) { ++ auto plugin = pair.second; ++ plugin->shutdown(); ++ } ++} ++ ++auto NRIAdaptation::Init(Errors &error) -> bool ++{ ++ std::map> tmp_storeMap; ++ ++ m_support = conf_get_nri_support(); ++ if (!m_support) { ++ return true; ++ } ++ ++ m_external_support = conf_get_nri_external_support(); ++ service_executor_t *cb = get_service_executor(); ++ if (cb == nullptr) { ++ ERROR("Init isulad service executor failure."); ++ return false; ++ } ++ ++ m_containerManager = std::make_unique(cb); ++ ++ m_pluginConfigPath = GetNRIPluginConfigPath(); ++ m_pluginPath = GetNRIPluginPath(); ++ m_plugin_registration_timeout = conf_get_nri_plugin_registration_timeout(); ++ m_plugin_requst_timeout = conf_get_nri_plugin_requst_timeout(); ++ m_sock_path = GetNRISockPath(); ++ ++ if (!StartPlugin()) { ++ ERROR("Failed to do StartPlugin"); ++ return false; ++ } ++ ++ if (!SortPlugins()) { ++ ERROR("Failed to do SortPlugins"); ++ return false; ++ } ++ ++ return true; ++} ++ ++void NRIAdaptation::RemoveClosedPlugins() ++{ ++ std::vector closedPlugin; ++ ++ GetClosedPlugins(closedPlugin); ++ ++ for (const auto &key : closedPlugin) { ++ RemovePluginByIndex(key); ++ } ++} ++ ++void NRIAdaptation::GetClosedPlugins(std::vector &closedPlugin) ++{ ++ ReadGuard lock(m_mutex); ++ for (const auto &pair : m_storeMap) { ++ auto plugin = pair.second; ++ if (plugin->IsClose()) { ++ closedPlugin.push_back(pair.first); ++ } ++ } ++} ++ ++auto NRIAdaptation::IsSupport() -> bool ++{ ++ return m_support; ++} ++ ++auto NRIAdaptation::GetPluginByIndex(const std::string &index) -> std::shared_ptr ++{ ++ ReadGuard lock(m_mutex); ++ return m_storeMap[index]; ++} ++ ++void NRIAdaptation::RemovePluginByIndex(const std::string &index) ++{ ++ WriteGuard lock(m_mutex); ++ m_storeMap.erase(index); ++} ++ ++void NRIAdaptation::AddPluginByIndex(const std::string &index, std::shared_ptr plugin) ++{ ++ WriteGuard lock(m_mutex); ++ m_storeMap[index] = plugin; ++} ++ ++auto NRIAdaptation::ApplyUpdates(const std::vector &update, ++ std::vector &failed, bool getFailed, Errors &error) -> bool ++{ ++ for (auto &u : update) { ++ runtime::v1::LinuxContainerResources resources; ++ if (!LinuxResourcesFromNRI(u->linux->resources, resources)) { ++ ERROR("Failed to convert Linux resources from NRI"); ++ error.Errorf("Failed to convert Linux resources from NRI"); ++ return false; ++ } ++ ++ m_containerManager->UpdateContainerResources(u->container_id, resources, error); ++ ++ if (error.NotEmpty()) { ++ ERROR("Failed to update container: %s resources: %s", u->container_id, error.GetCMessage()); ++ if (!u->ignore_failure && getFailed) { ++ failed.push_back(u); ++ } ++ continue; ++ } ++ ++ TRACE("NRI update of container %s successful", u->container_id); ++ } ++ ++ if (failed.size() != 0) { ++ error.Errorf("NRI update of some containers failed"); ++ return false; ++ } ++ return true; ++} ++ ++auto NRIAdaptation::NewExternalPlugin(int fd) -> bool ++{ ++ if (!m_external_support) { ++ ERROR("External plugin support is disabled"); ++ return false; ++ } ++ if (fd < 0) { ++ ERROR("Invalid fd"); ++ return false; ++ } ++ ++ std::string plugin_name; ++ NRIHelpers::GenerateRandomExternalName(plugin_name); ++ if (plugin_name.empty()) { ++ ERROR("Failed to generate random external name"); ++ return false; ++ } ++ ++ auto plugin = std::make_shared(fd, plugin_name); ++ ++ AddPluginByIndex(plugin_name, plugin); ++ ++ if (!plugin->Start(m_plugin_registration_timeout, m_plugin_requst_timeout)) { ++ ERROR("Failed to start plugin ready for conn fd %d", fd); ++ RemovePluginByIndex(plugin_name); ++ return false; ++ } ++ ++ std::vector pods; ++ std::vector cons; ++ nri_container_update **updateRes; ++ size_t update_len = 0; ++ ++ std::vector> sandboxes; ++ runtime::v1::PodSandboxFilter podFilter; ++ std::vector> containers; ++ Errors tmpError; ++ ++ std::vector updates; ++ std::vector failed; ++ ++ sandbox::SandboxManager::GetInstance()->ListAllSandboxes(podFilter, sandboxes); ++ ++ if (!PodSandboxesToNRI(sandboxes, pods)) { ++ ERROR("Failed to convert podsandbox to nri"); ++ NRIHelpers::FreeNriPodVector(pods); ++ return false; ++ } ++ ++ m_containerManager->ListContainers(nullptr, containers, tmpError); ++ ++ if (!ContainersToNRI(containers, cons)) { ++ ERROR("Failed to convert container to nri"); ++ NRIHelpers::FreeNriPodVector(pods); ++ NRIHelpers::FreeNriContainerVector(cons); ++ return false; ++ } ++ ++ // pods and cons's memory transfer to nri_synchronize_request, ++ // and is automatically freed when nri_synchronize_request is freed ++ if (!plugin->Synchronize(pods, cons, &updateRes, update_len, tmpError)) { ++ ERROR("Failed to synchronize plugin"); ++ return false; ++ } ++ ++ for (size_t i = 0; i < update_len; i++) { ++ updates.push_back(updateRes[i]); ++ } ++ ++ if (!ApplyUpdates(updates, failed, false, tmpError)) { ++ ERROR("Failed to update post-sync"); ++ } ++ ++ NRIHelpers::FreeNriContainerUpdateVector(updates); ++ NRIHelpers::FreeNriContainerUpdateVector(failed); ++ INFO("plugin %s connected", plugin_name.c_str()); ++ return true; ++} ++ ++auto NRIAdaptation::RunPodSandbox(std::shared_ptr sandbox, Errors &error) -> bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto runPodEvent = makeUniquePtrCStructWrapper(free_nri_state_change_event); ++ if (runPodEvent == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ runPodEvent->get()->pod = pod->move(); ++ runPodEvent->get()->event = RUN_POD_SANDBOX; ++ return StateChange(runPodEvent->get(), error); ++} ++ ++auto NRIAdaptation::StopPodSandbox(std::shared_ptr sandbox, Errors &error) -> bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto stopPodEvent = makeUniquePtrCStructWrapper(free_nri_state_change_event); ++ if (stopPodEvent == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ stopPodEvent->get()->pod = pod->move(); ++ stopPodEvent->get()->event = STOP_POD_SANDBOX; ++ return StateChange(stopPodEvent->get(), error); ++} ++ ++auto NRIAdaptation::RemovePodSandbox(std::shared_ptr sandbox, Errors &error) -> bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto removePodEvent = makeUniquePtrCStructWrapper(free_nri_state_change_event); ++ if (removePodEvent == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ removePodEvent->get()->pod = pod->move(); ++ removePodEvent->get()->event = REMOVE_POD_SANDBOX; ++ return StateChange(removePodEvent->get(), error); ++} ++ ++auto NRIAdaptation::CreateContainer(std::shared_ptr sandbox, const std::string &conId, ++ const runtime::v1::ContainerConfig &containerConfig, nri_container_adjustment **adjust, Errors &error) -> bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto con = NRIContainerByConConfig(sandbox, containerConfig, error); ++ if (con == nullptr) { ++ ERROR("Failed to covert container to nri: %s", conId.c_str()); ++ return false; ++ } ++ ++ auto req = makeUniquePtrCStructWrapper(free_nri_create_container_request); ++ if (req == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ req->get()->container = con->move(); ++ req->get()->pod = pod->move(); ++ ++ pluginResult result; ++ result.InitByConId(conId); ++ ++ if (!PluginsCreateContainer(req->get(), conId, result)) { ++ ERROR("Failed to call create container to all plugins"); ++ return false; ++ } ++ ++ RemoveClosedPlugins(); ++ ++ // TODO:evict container do not aply ++ ++ // TODO:how can i rollback on failure ++ std::vector failed; ++ if (!ApplyUpdates(result.GetReplyUpdate(), failed, false, error)) { ++ ERROR("Failed to apply updates"); ++ NRIHelpers::FreeNriContainerUpdateVector(failed); ++ return false; ++ } ++ ++ *adjust = result.MoveReplyAdjust(); ++ NRIHelpers::FreeNriContainerUpdateVector(failed); ++ return true; ++} ++ ++bool NRIAdaptation::PluginsCreateContainer(nri_create_container_request *req, const std::string &conId, ++ pluginResult &result) ++{ ++ ReadGuard lock(m_mutex); ++ ++ for (const auto &pair : m_storeMap) { ++ auto plugin = pair.second; ++ Errors tmpError; ++ nri_create_container_response *resp = nullptr; ++ ++ if (!plugin->CreateContainer(req, &resp, tmpError)) { ++ ERROR("Failed to call create container: %s to pliugin: %s", conId.c_str(), plugin->GetName().c_str()); ++ (void)plugin->shutdown(); ++ continue; ++ } ++ ++ if (resp == nullptr) { ++ ERROR("Empty CreateContainer resp : %s", plugin->GetName().c_str()); ++ continue; ++ } ++ ++ auto resp_wrapper = ++ makeUniquePtrCStructWrapper(resp, free_nri_create_container_response); ++ if (resp_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ result.Apply(CREATE_CONTAINER, resp_wrapper->get()->adjust, resp_wrapper->get()->update, ++ resp_wrapper->get()->update_len, plugin->GetName()); ++ } ++ return true; ++} ++ ++auto NRIAdaptation::PostCreateContainer(const std::string &conId, Errors &error) -> bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto con = NRIContainerByID(conId, error); ++ if (con == nullptr) { ++ ERROR("Failed to covert container to nri: %s", conId.c_str()); ++ return false; ++ } ++ ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(con->get()->pod_sandbox_id); ++ if (sandbox == nullptr) { ++ ERROR("Failed to get sandbox info for nri"); ++ return false; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto postCreateConEvent = makeUniquePtrCStructWrapper(free_nri_state_change_event); ++ if (postCreateConEvent == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ postCreateConEvent->get()->container = con->move(); ++ postCreateConEvent->get()->pod = pod->move(); ++ postCreateConEvent->get()->event = POST_CREATE_CONTAINER; ++ return StateChange(postCreateConEvent->get(), error); ++} ++ ++auto NRIAdaptation::StartContainer(const std::string &conId, Errors &error) -> bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto con = NRIContainerByID(conId, error); ++ if (con == nullptr) { ++ ERROR("Failed to covert container to nri: %s", conId.c_str()); ++ return false; ++ } ++ ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(con->get()->pod_sandbox_id); ++ if (sandbox == nullptr) { ++ ERROR("Failed to get sandbox info for nri"); ++ return false; ++ } ++ ++ if (sandbox == nullptr) { ++ ERROR("Failed to get sandbox for container: %s, sandbox id: %s", conId.c_str(), con->get()->pod_sandbox_id); ++ return false; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto startConEvent = makeUniquePtrCStructWrapper(free_nri_state_change_event); ++ if (startConEvent == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ startConEvent->get()->container = con->move(); ++ startConEvent->get()->pod = pod->move(); ++ startConEvent->get()->event = START_CONTAINER; ++ return StateChange(startConEvent->get(), error); ++} ++ ++auto NRIAdaptation::PostStartContainer(const std::string &conId, Errors &error) -> bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto con = NRIContainerByID(conId, error); ++ if (con == nullptr) { ++ ERROR("Failed to covert container to nri: %s", conId.c_str()); ++ return false; ++ } ++ ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(con->get()->pod_sandbox_id); ++ if (sandbox == nullptr) { ++ ERROR("Failed to get sandbox info for nri"); ++ return false; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto postStartConEvent = makeUniquePtrCStructWrapper(free_nri_state_change_event); ++ if (postStartConEvent == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ postStartConEvent->get()->container = con->move(); ++ postStartConEvent->get()->pod = pod->move(); ++ postStartConEvent->get()->event = POST_START_CONTAINER; ++ return StateChange(postStartConEvent->get(), error); ++} ++ ++auto NRIAdaptation::UndoCreateContainer(std::shared_ptr sandbox, const std::string &conId, ++ Errors &error) -> bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ if (!StopContainer(conId, error)) { ++ ERROR("container creation undo (stop) failed: %s", conId.c_str()); ++ } ++ ++ if (!RemoveContainer(conId, error)) { ++ ERROR("container creation undo (remove) failed: %s", conId.c_str()); ++ } ++ ++ return true; ++} ++ ++auto NRIAdaptation::UpdateContainer(const std::string &conId, const runtime::v1::LinuxContainerResources &resources, ++ runtime::v1::LinuxContainerResources &adjust, Errors &error) -> bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto con = NRIContainerByID(conId, error); ++ if (con == nullptr) { ++ ERROR("Failed to covert container to nri: %s", conId.c_str()); ++ return false; ++ } ++ ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(con->get()->pod_sandbox_id); ++ if (sandbox == nullptr) { ++ ERROR("Failed to get sandbox info for nri"); ++ return false; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto req = makeUniquePtrCStructWrapper(free_nri_update_container_request); ++ if (req == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ auto reqRes = LinuxResourcesToNRI(resources); ++ ++ req->get()->container = con->move(); ++ req->get()->pod = pod->move(); ++ req->get()->linux_resources = reqRes; ++ ++ pluginResult result; ++ result.InitByUpdateReq(req->get()); ++ ++ if (!PluginsUpdateContainer(req->get(), conId, result)) { ++ ERROR("Failed to call update container to all plugins"); ++ return false; ++ } ++ ++ RemoveClosedPlugins(); ++ ++ // TODO:evict container do not aply ++ ++ // TODO:how can i rollback on failure ++ std::vector failed; ++ if (!ApplyUpdates(result.GetReplyUpdate(), failed, false, error)) { ++ ERROR("Failed to apply updates"); ++ NRIHelpers::FreeNriContainerUpdateVector(failed); ++ return false; ++ } ++ ++ NRIHelpers::FreeNriContainerUpdateVector(failed); ++ ++ if (!LinuxResourcesFromNRI(result.GetReplyResources(conId), adjust)) { ++ ERROR("Failed to convert Linux resources from NRI"); ++ return false; ++ } ++ ++ return true; ++} ++ ++bool NRIAdaptation::PluginsUpdateContainer(nri_update_container_request *req, const std::string &conId, ++ pluginResult &result) ++{ ++ ReadGuard lock(m_mutex); ++ ++ for (const auto &pair : m_storeMap) { ++ auto plugin = pair.second; ++ Errors tmpError; ++ nri_update_container_response *resp = nullptr; ++ ++ if (!plugin->UpdateContainer(req, &resp, tmpError)) { ++ ERROR("Failed to call create container: %s to pliugin: %s", conId.c_str(), plugin->GetName().c_str()); ++ plugin->shutdown(); ++ continue; ++ } ++ ++ if (resp == nullptr) { ++ ERROR("Empty UpdateContainer resp : %s", plugin->GetName().c_str()); ++ continue; ++ } ++ ++ auto resp_wrapper = ++ makeUniquePtrCStructWrapper(resp, free_nri_update_container_response); ++ if (resp_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ result.Apply(UPDATE_CONTAINER, nullptr, resp_wrapper->get()->update, resp_wrapper->get()->update_len, ++ plugin->GetName()); ++ } ++ return true; ++} ++ ++bool NRIAdaptation::PluginsStopContainer(nri_stop_container_request *req, const std::string &conId, ++ pluginResult &result) ++{ ++ ReadGuard lock(m_mutex); ++ ++ for (const auto &pair : m_storeMap) { ++ auto plugin = pair.second; ++ Errors tmpError; ++ nri_stop_container_response *resp = nullptr; ++ ++ if (!plugin->StopContainer(req, &resp, tmpError)) { ++ ERROR("Failed to call create container: %s to pliugin: %s", conId.c_str(), plugin->GetName().c_str()); ++ plugin->shutdown(); ++ continue; ++ } ++ ++ if (resp == nullptr) { ++ ERROR("Empty StopContainer resp : %s", plugin->GetName().c_str()); ++ continue; ++ } ++ ++ auto resp_wrapper = makeUniquePtrCStructWrapper(resp, free_nri_stop_container_response); ++ if (resp_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ result.Apply(STOP_CONTAINER, nullptr, resp_wrapper->get()->update, resp_wrapper->get()->update_len, plugin->GetName()); ++ } ++ return true; ++} ++ ++auto NRIAdaptation::PostUpdateContainer(const std::string &conId, Errors &error) ->bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto con = NRIContainerByID(conId, error); ++ if (con == nullptr) { ++ ERROR("Failed to covert container to nri: %s", conId.c_str()); ++ return false; ++ } ++ ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(con->get()->pod_sandbox_id); ++ if (sandbox == nullptr) { ++ ERROR("Failed to get sandbox info for nri"); ++ return false; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto postUpdateConEvent = makeUniquePtrCStructWrapper(free_nri_state_change_event); ++ if (postUpdateConEvent == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ postUpdateConEvent->get()->container = con->move(); ++ postUpdateConEvent->get()->pod = pod->move(); ++ postUpdateConEvent->get()->event = POST_UPDATE_CONTAINER; ++ return StateChange(postUpdateConEvent->get(), error); ++} ++ ++auto NRIAdaptation::StopContainer(const std::string &conId, Errors &error) ->bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto con = NRIContainerByID(conId, error); ++ if (con == nullptr) { ++ ERROR("Failed to covert container to nri: %s", conId.c_str()); ++ return false; ++ } ++ ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(con->get()->pod_sandbox_id); ++ if (sandbox == nullptr) { ++ ERROR("Failed to get sandbox info for nri"); ++ return false; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto req = makeUniquePtrCStructWrapper(free_nri_stop_container_request); ++ if (req == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ req->get()->pod = pod->move(); ++ req->get()->container = con->move(); ++ ++ pluginResult result; ++ result.Init(); ++ ++ if (!PluginsStopContainer(req->get(), conId, result)) { ++ ERROR("Failed to call stop container to all plugins"); ++ return false; ++ } ++ ++ RemoveClosedPlugins(); ++ ++ // TODO:how can i rollback on failure ++ std::vector failed; ++ if (!ApplyUpdates(result.GetReplyUpdate(), failed, false, error)) { ++ ERROR("Failed to apply updates"); ++ NRIHelpers::FreeNriContainerUpdateVector(failed); ++ return false; ++ } ++ ++ NRIHelpers::FreeNriContainerUpdateVector(failed); ++ return true; ++} ++ ++auto NRIAdaptation::RemoveContainer(const std::string &conId, Errors &error) ->bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto con = NRIContainerByID(conId, error); ++ if (con == nullptr) { ++ ERROR("Failed to covert container to nri: %s", conId.c_str()); ++ return false; ++ } ++ ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(con->get()->pod_sandbox_id); ++ if (sandbox == nullptr) { ++ ERROR("Failed to get sandbox info for nri"); ++ return false; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto removeConEvent = makeUniquePtrCStructWrapper(free_nri_state_change_event); ++ if (removeConEvent == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ removeConEvent->get()->container = con->move(); ++ removeConEvent->get()->pod = pod->move(); ++ removeConEvent->get()->event = REMOVE_CONTAINER; ++ return StateChange(removeConEvent->get(), error); ++} ++ ++auto NRIAdaptation::StateChange(nri_state_change_event *evt, Errors &error) ->bool ++{ ++ if (evt->event == UNKNOWN) { ++ ERROR("invalid (unset) event in state change notification"); ++ error.SetError("invalid (unset) event in state change notification"); ++ return false; ++ } ++ ++ PluginsStateChange(evt); ++ ++ RemoveClosedPlugins(); ++ return true; ++} ++ ++void NRIAdaptation::PluginsStateChange(nri_state_change_event *evt) ++{ ++ ReadGuard lock(m_mutex); ++ ++ for (const auto &pair : m_storeMap) { ++ auto plugin = pair.second; ++ Errors tmpError; ++ if (!plugin->StateChange(evt, tmpError)) { ++ ERROR("invalid (unset) event in state change notification: %s", plugin->GetName().c_str()); ++ plugin->shutdown(); ++ continue; ++ } ++ } ++} ++ ++// Perform a set of unsolicited container updates requested by a plugin. ++auto NRIAdaptation::updateContainers(const nri_update_containers_request *req, ++ nri_update_containers_response **resp) ->bool ++{ ++ std::vector failed; ++ std::vector vec; ++ size_t i; ++ Errors error; ++ bool ret = false; ++ ++ if (req == nullptr) { ++ ERROR("Invalid request"); ++ return false; ++ } ++ ++ for (i = 0; i < req->update_len; i++) { ++ vec.push_back(req->update[i]); ++ } ++ ++ if (!ApplyUpdates(vec, failed, false, error)) { ++ ERROR("Failed to apply updates: %s", error.GetCMessage()); ++ goto free_out; ++ } ++ ++ if (failed.size() == 0) { ++ ret = true; ++ goto free_out; ++ } ++ ++ (*resp)->failed = (nri_container_update **)util_common_calloc_s(failed.size() * sizeof(nri_container_update *)); ++ if ((*resp)->failed == nullptr) { ++ ERROR("Out of memory"); ++ goto free_out; ++ } ++ ++ for (i = 0; i < failed.size(); i++) { ++ (*resp)->failed[i] = failed[i]; ++ failed[i] = nullptr; ++ (*resp)->failed_len++; ++ } ++ ret = true; ++ ++free_out: ++ NRIHelpers::FreeNriContainerUpdateVector(vec); ++ NRIHelpers::FreeNriContainerUpdateVector(failed); ++ return ret; ++} ++ ++auto NRIAdaptation::StartPlugin() -> bool ++{ ++ std::map> tmp_storeMap; ++ ++ if (!DiscoverPlugins(tmp_storeMap) != 0) { ++ ERROR("Failed to do DiscoverPlugins"); ++ return false; ++ } ++ ++ for (const auto &pair : tmp_storeMap) { ++ const std::string &index = pair.first; ++ auto plugin = pair.second; ++ ++ if (!NewLaunchedPlugin(plugin)) { ++ ERROR("Failed to do NewLaunchedPlugin for %s", plugin->GetName().c_str()); ++ continue; ++ } ++ ++ AddPluginByIndex(index, plugin); ++ ++ if (!plugin->Start(m_plugin_registration_timeout, m_plugin_requst_timeout)) { ++ ERROR("Failed to start plugin %s ready", plugin->GetName().c_str()); ++ RemovePluginByIndex(index); ++ (void)plugin->shutdown(); ++ continue; ++ } ++ } ++ ++ return true; ++} ++ ++// return true always, failure to obtain one plugin does not affect other plugin ++static auto walk_plugin_dir_cb(const char *path_name, const struct dirent *sub_dir, void *context) -> bool ++{ ++ std::string full_path = std::string(path_name) + "/" + sub_dir->d_name; ++ struct stat file_stat = { 0 }; ++ ++ if (stat(full_path.c_str(), &file_stat) != 0) { ++ WARN("Failed to get NRI plugin %s stat", sub_dir->d_name); ++ return true; ++ } ++ ++ if (S_ISDIR(file_stat.st_mode)) { ++ INFO("Skip dir in plugin path %s", sub_dir->d_name); ++ return true; ++ } ++ ++ // 1. Verify plugin permissions for exec ++ if (!(file_stat.st_mode & S_IXUSR)) { ++ ERROR("NRI plugin %s has no permission for exec", sub_dir->d_name); ++ return true; ++ } ++ ++ // 2. Parse plugin name ++ __isula_auto_array_t char **arr = util_string_split_n(sub_dir->d_name, '-', 2); ++ if (arr == nullptr) { ++ ERROR("Invalid plugin name %s, idx-pluginname expected", sub_dir->d_name); ++ return true; ++ } ++ ++ if (!NRIHelpers::CheckPluginIndex(arr[0])) { ++ ERROR("Invalid plugin name %s, invalid idx", sub_dir->d_name); ++ return true; ++ } ++ ++ // 3. init plugin ++ std::string index(arr[0]); ++ std::string pluginName(arr[1]); ++ std::string config; ++ ++ std::map> &map = ++ *static_cast>*>(context); ++ if (!NRIHelpers::GetPluginConfig(index, pluginName, config)) { ++ ERROR("Failed to get plugin %s config", pluginName.c_str()); ++ return true; ++ } ++ ++ auto plugin = std::make_shared(index, pluginName, config); ++ ++ // todo:use random id str ++ map[pluginName] = plugin; ++ return true; ++} ++ ++static void plugin_exec_func(nri_plugin_exec_args_t * plugin_args) ++{ ++ const char *params[PARAM_NUM] = {0}; ++ int i = 0; ++ std::string sock = std::to_string(plugin_args->sockFd); ++ ++ if (plugin_args == nullptr) { ++ ERROR("Missing plugin exec info"); ++ _exit(EXIT_FAILURE); ++ } ++ ++ if (chdir(plugin_args->workdir) < 0) { ++ ERROR("Failed to chdir to %s", plugin_args->workdir); ++ _exit(EXIT_FAILURE); ++ } ++ ++ if (setenv(PluginNameEnv.c_str(), plugin_args->name, 1) != 0) { ++ ERROR("%s: failed to set PluginNameEnv for process %d", plugin_args->name, getpid()); ++ exit(EXIT_FAILURE); ++ } ++ ++ if (setenv(PluginIdxEnv.c_str(), plugin_args->index, 1) != 0) { ++ ERROR("%s: failed to set PluginIdxEnv for process %d", plugin_args->name, getpid()); ++ exit(EXIT_FAILURE); ++ } ++ ++ if (setenv(PluginSocketEnv.c_str(), sock.c_str(), 1) != 0) { ++ ERROR("%s: failed to set PluginSocketEnv for process %d", plugin_args->name, getpid()); ++ exit(EXIT_FAILURE); ++ } ++ ++ if (util_check_inherited(true, plugin_args->sockFd) != 0) { ++ ERROR("Failed to close inherited fds"); ++ exit(EXIT_FAILURE); ++ } ++ ++ if (setsid() < 0) { ++ ERROR("Failed to setsid for nri plugin: %s", plugin_args->name); ++ exit(EXIT_FAILURE); ++ } ++ ++ params[i++] = plugin_args->name; ++ ++ execvp(plugin_args->cmd, (char * const *)params); ++ ERROR("Failed to exec %s", plugin_args->cmd); ++ _exit(EXIT_FAILURE); ++} ++ ++// create socket, and call plugin start ++auto NRIAdaptation::NewLaunchedPlugin(const std::shared_ptr &plugin) -> bool ++{ ++ // 1. create socket for plugin ++ if (!plugin->CreateSocketPair()) { ++ ERROR("Failed to create socket pair"); ++ return false; ++ } ++ ++ std::string name = plugin->GetQualifiedName(); ++ std::string cmd = m_pluginPath + "/" + name; ++ ++ DEBUG("Plugin %s start", cmd.c_str()); ++ ++ // 2. exec plugin ++ nri_plugin_exec_args_t p_args = { ++ .workdir = m_pluginPath.c_str(), ++ .cmd = cmd.c_str(), ++ .name = name.c_str(), ++ .index = plugin->GetIndex().c_str(), ++ .sockFd = plugin->GetPeerSockFd(), ++ }; ++ ++ int pid = fork(); ++ if (pid == (pid_t) -1) { ++ SYSERROR("Failed to fork"); ++ return false; ++ } ++ ++ if (pid == (pid_t)0) { ++ set_child_process_pdeathsig(); ++ ++ plugin_exec_func(&p_args); ++ } ++ ++ close(plugin->GetPeerSockFd()); ++ ++ plugin->SetPid(pid); ++ ++ return true; ++} ++ ++// find plugin and create plugin ++auto NRIAdaptation::DiscoverPlugins(std::map> &map) -> bool ++{ ++ int nret = 0; ++ ++ // 1. get all plugin ++ nret = util_scan_subdirs(m_pluginPath.c_str(), walk_plugin_dir_cb, static_cast(&map)); ++ if (nret != 0) { ++ ERROR("Failed to scan nri plugin subdirs"); ++ } ++ return true; ++} ++ ++auto NRIAdaptation::SortPlugins() -> bool ++{ ++ RemoveClosedPlugins(); ++ ++ std::vector>> sortedPlugins(m_storeMap.begin(), m_storeMap.end()); ++ ++ std::sort(sortedPlugins.begin(), sortedPlugins.end(), [](const auto & a, const auto & b) { ++ return a.first < b.first; ++ }); ++ ++ WriteGuard lock(m_mutex); ++ m_storeMap.clear(); ++ for (const auto &pair : sortedPlugins) { ++ m_storeMap.insert(pair); ++ } ++ ++ return true; ++} ++ ++auto NRIAdaptation::GetNRIPluginConfigPath(void) -> std::string ++{ ++ __isula_auto_free char *config_path = nullptr; ++ std::string ret; ++ ++ config_path = conf_get_nri_plugin_config_path(); ++ if (config_path == nullptr) { ++ return ret; ++ } ++ ret = std::string(config_path); ++ return ret; ++} ++ ++auto NRIAdaptation::GetNRIPluginPath(void) -> std::string ++{ ++ __isula_auto_free char *plugin_path = nullptr; ++ std::string ret; ++ ++ plugin_path = conf_get_nri_plugin_path(); ++ if (plugin_path == nullptr) { ++ return ret; ++ } ++ ret = std::string(plugin_path); ++ return ret; ++} ++ ++auto NRIAdaptation::GetNRISockPath(void) -> std::string ++{ ++ __isula_auto_free char *sock_path = nullptr; ++ std::string ret; ++ ++ sock_path = conf_get_socket_path(); ++ if (sock_path == nullptr) { ++ return ret; ++ } ++ ret = std::string(sock_path); ++ return ret; ++} ++ ++auto NRIAdaptation::NRIPodSandbox(const std::shared_ptr &sandbox, ++ Errors &error) -> std::unique_ptr> ++{ ++ auto pod = makeUniquePtrCStructWrapper(free_nri_pod_sandbox); ++ if (pod == nullptr) { ++ ERROR("Out of memory"); ++ return nullptr; ++ } ++ ++ if (!PodSandboxToNRI(sandbox, *pod->get())) { ++ error.Errorf("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return nullptr; ++ } ++ ++ return pod; ++} ++ ++auto NRIAdaptation::NRIContainerByID(const std::string &id, ++ Errors &error) -> std::unique_ptr> ++{ ++ auto con = makeUniquePtrCStructWrapper(free_nri_container); ++ if (con == nullptr) { ++ ERROR("Out of memory"); ++ return nullptr; ++ } ++ ++ if (!ContainerToNRIByID(id, *con->get())) { ++ error.Errorf("Failed to covert container to nri: %s", id.c_str()); ++ ERROR("Failed to covert container to nri: %s", id.c_str()); ++ return nullptr; ++ } ++ ++ return con; ++} ++ ++auto NRIAdaptation::NRIContainerByConConfig(const std::shared_ptr &sandbox, ++ const runtime::v1::ContainerConfig &containerConfig, Errors &error) -> std::unique_ptr> ++{ ++ auto con = makeUniquePtrCStructWrapper(free_nri_container); ++ if (con == nullptr) { ++ ERROR("Out of memory"); ++ return nullptr; ++ } ++ ++ if (!ContainerToNRIByConConfig(containerConfig, *con->get())) { ++ error.Errorf("Failed to covert container to nri: %s", con->get()->name); ++ ERROR("Failed to covert container to nri: %s", con->get()->name); ++ return nullptr; ++ } ++ con->get()->pod_sandbox_id = isula_strdup_s(sandbox->GetId().c_str()); ++ ++ return con; ++} +\ No newline at end of file +diff --git a/src/daemon/nri/nri_adaption.h b/src/daemon/nri/nri_adaption.h +index 874662cf..27a6d93e 100644 +--- a/src/daemon/nri/nri_adaption.h ++++ b/src/daemon/nri/nri_adaption.h +@@ -46,10 +46,6 @@ public: + + auto GetSockpath(std::vector &paths) -> bool; + +- auto StopPlugins() -> bool; +- +- void RemoveClosedPlugins(); +- + auto GetPluginByIndex(const std::string &index) -> std::shared_ptr; + void AddPluginByIndex(const std::string &index, std::shared_ptr plugin); + void RemovePluginByIndex(const std::string &index); +@@ -65,7 +61,8 @@ public: + Errors &error) -> bool; + auto StartContainer(const std::string &conId, Errors &error) -> bool; + auto PostStartContainer(const std::string &conId, Errors &error) -> bool; +- auto UpdateContainer(const std::string &conId, Errors &error) -> bool; ++ auto UpdateContainer(const std::string &conId, const runtime::v1::LinuxContainerResources &resources, ++ runtime::v1::LinuxContainerResources &adjust, Errors &error) -> bool; + auto PostUpdateContainer(const std::string &conId, Errors &error) -> bool; + auto StopContainer(const std::string &conId, Errors &error) -> bool; + auto RemoveContainer(const std::string &conId, Errors &error) -> bool; +@@ -87,6 +84,8 @@ private: + auto SyncPlugin() -> bool; + + auto SortPlugins() -> bool; ++ void RemoveClosedPlugins(); ++ + void GetClosedPlugins(std::vector &closedPlugin); + + auto ApplyUpdates(const std::vector &update, std::vector &failed, +@@ -107,18 +106,16 @@ private: + void PluginsStateChange(nri_state_change_event *evt); + bool PluginsCreateContainer(nri_create_container_request *req, const std::string &conId, pluginResult &result); + bool PluginsUpdateContainer(nri_update_container_request *req, const std::string &conId, pluginResult &result); ++ bool PluginsStopContainer(nri_stop_container_request *req, const std::string &conId, pluginResult &result); + + private: + RWMutex m_mutex; + static std::atomic m_instance; + bool m_support; + bool m_external_support; +- std::string m_version; + std::string m_sock_path; + std::string m_pluginConfigPath; + std::string m_pluginPath; +- std::vector m_socketPathArr; +- std::string m_disableConnections; + // id --> NRIPlugin map + std::map> m_storeMap; + // TODO:plugin monitor thread id?? +diff --git a/src/daemon/nri/nri_helpers.cc b/src/daemon/nri/nri_helpers.cc +index ff9d67c1..b660e7a7 100644 +--- a/src/daemon/nri/nri_helpers.cc ++++ b/src/daemon/nri/nri_helpers.cc +@@ -90,4 +90,25 @@ bool CheckPluginIndex(const std::string &idx) + + return true; + } ++ ++void FreeNriContainerUpdateVector(std::vector &vec) ++{ ++ for (auto ptr : vec) { ++ free_nri_container_update(ptr); ++ } ++} ++ ++void FreeNriContainerVector(std::vector &vec) ++{ ++ for (auto ptr : vec) { ++ free_nri_container(ptr); ++ } ++} ++ ++void FreeNriPodVector(std::vector &vec) ++{ ++ for (auto ptr : vec) { ++ free_nri_pod_sandbox(ptr); ++ } ++} + }// namespace NRIHelpers +\ No newline at end of file +diff --git a/src/daemon/nri/nri_helpers.h b/src/daemon/nri/nri_helpers.h +index 1a2f488e..e776987b 100644 +--- a/src/daemon/nri/nri_helpers.h ++++ b/src/daemon/nri/nri_helpers.h +@@ -41,6 +41,10 @@ void GenerateRandomExternalName(std::string &ret); + + bool CheckPluginIndex(const std::string &idx); + ++void FreeNriContainerUpdateVector(std::vector &vec); ++void FreeNriContainerVector(std::vector &vec); ++void FreeNriPodVector(std::vector &vec); ++ + template + void freeArray(T ** &arr, int size) + { +diff --git a/src/daemon/nri/nri_plugin_ops.cc b/src/daemon/nri/nri_plugin_ops.cc +new file mode 100644 +index 00000000..e2f88b63 +--- /dev/null ++++ b/src/daemon/nri/nri_plugin_ops.cc +@@ -0,0 +1,123 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-03-26 ++ * Description: provide nri plugin api definition ++ ******************************************************************************/ ++#include "nri_plugin_ops.h" ++ ++#include ++#include ++ ++#include "nri_adaption.h" ++#include "nri_plugin.h" ++#include "isulad_config.h" ++ ++static bool start_external_listener() ++{ ++ __isula_auto_free char *sock_path = NULL; ++ ++ sock_path = conf_get_socket_path(); ++ if (sock_path == NULL) { ++ ERROR("Failed to get socket path"); ++ return false; ++ } ++ ++ if (nri_external_service_start(sock_path, nri_external_plugin_connect) != 0) { ++ ERROR("Failed to lauch external service"); ++ return false; ++ } ++ return true; ++} ++ ++bool nri_adaption_init(void) ++{ ++ Errors error; ++ ++ if (!conf_get_nri_support()) { ++ return true; ++ } ++ ++ nri_runtime_callbacks callbacks; ++ callbacks.register_plugin = nri_registry_containers; ++ callbacks.update_containers = nri_update_containers; ++ if (nri_runtime_service_init(callbacks) != 0) { ++ ERROR("Failed to init runtime service\n"); ++ return false; ++ } ++ ++ if (conf_get_nri_external_support()) { ++ if (!start_external_listener()) { ++ ERROR("Failed to start external listener\n"); ++ goto clean_out; ++ } ++ } ++ ++ NRIAdaptation::GetInstance()->Init(error); ++ if (error.NotEmpty()) { ++ ERROR("Failed to init NRIAdaptation: %s", error.GetCMessage()); ++ goto clean_out; ++ } ++ return true; ++clean_out: ++ nri_runtime_service_destroy(); ++ return false; ++} ++ ++bool nri_adaption_shutdown(void) ++{ ++ nri_external_service_shutdown(); ++ nri_runtime_service_destroy(); ++ return true; ++} ++ ++int nri_update_containers(const char *plugin_id, const nri_update_containers_request *request, ++ nri_update_containers_response **response) ++{ ++ if (request == nullptr || response == nullptr || plugin_id == nullptr) { ++ ERROR("Invalid input arguments"); ++ return -1; ++ } ++ ++ if (!NRIAdaptation::GetInstance()->updateContainers(request, response)) { ++ ERROR("Failed to update containers by plugin %s", plugin_id); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++int nri_registry_containers(const char *plugin_id, const nri_register_plugin_request *request) ++{ ++ if (request == nullptr || plugin_id == nullptr) { ++ ERROR("Invalid input arguments"); ++ return -1; ++ } ++ ++ auto plugin = NRIAdaptation::GetInstance()->GetPluginByIndex(plugin_id); ++ if (plugin == nullptr) { ++ ERROR("Failed to get plugin by index %s", plugin_id); ++ return -1; ++ } ++ ++ plugin->SetReady(); ++ return 0; ++} ++ ++int nri_external_plugin_connect(int fd) ++{ ++ if (fd < 0) { ++ ERROR("Invalid input arguments"); ++ return -1; ++ } ++ ++ return NRIAdaptation::GetInstance()->NewExternalPlugin(fd) ? 0 : -1; ++} +\ No newline at end of file +diff --git a/src/daemon/nri/nri_plugin_ops.h b/src/daemon/nri/nri_plugin_ops.h +index 37d437d2..3a5393ba 100644 +--- a/src/daemon/nri/nri_plugin_ops.h ++++ b/src/daemon/nri/nri_plugin_ops.h +@@ -25,13 +25,15 @@ extern "C" { + #endif + + bool nri_adaption_init(void); ++bool nri_adaption_shutdown(void); + + #ifdef __cplusplus + } + #endif + +-int nri_update_containers(const nri_update_containers_request *request, nri_update_containers_response **response); +-int nri_registry_containers(const nri_register_plugin_request *request); ++int nri_update_containers(const char *plugin_id, const nri_update_containers_request *request, ++ nri_update_containers_response **response); ++int nri_registry_containers(const char *plugin_id, const nri_register_plugin_request *request); + + int nri_external_plugin_connect(int fd); + +diff --git a/src/daemon/nri/nri_result.cc b/src/daemon/nri/nri_result.cc +new file mode 100644 +index 00000000..7da4451d +--- /dev/null ++++ b/src/daemon/nri/nri_result.cc +@@ -0,0 +1,977 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-06-29 ++ * Description: provide nri result definition ++ *********************************************************************************/ ++ ++#include "nri_result.h" ++ ++#include ++#include ++ ++#include "cxxutils.h" ++#include "transform.h" ++#include "utils.h" ++ ++pluginResult::~pluginResult() ++{ ++ free_nri_linux_resources(m_update_req); ++ free_nri_container_adjustment(m_reply.adjust); ++ for (size_t i = 0; i < m_reply.update.size(); i++) { ++ free_nri_container_update(m_reply.update[i]); ++ } ++} ++ ++auto pluginResult::InitReply() -> bool ++{ ++ m_reply.adjust = (nri_container_adjustment *)util_common_calloc_s(sizeof(nri_container_adjustment)); ++ if (m_reply.adjust == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ return true; ++} ++ ++auto pluginResult::Init() -> bool ++{ ++ if (!InitReply()) { ++ ERROR("Failed to init reply"); ++ return false; ++ } ++ m_update_req = nullptr; ++ return true; ++} ++ ++auto pluginResult::InitByConId(std::string conId) -> bool ++{ ++ m_conId = conId; ++ ++ if (!InitReply()) { ++ ERROR("Failed to init reply"); ++ return false; ++ } ++ ++ m_update_req = nullptr; ++ return true; ++} ++ ++auto pluginResult::InitByUpdateReq(nri_update_container_request *req) -> bool ++{ ++ m_conId = req->container->id; ++ m_update_req = copy_nri_linux_resources(req->linux_resources); ++ if (m_update_req == nullptr) { ++ ERROR("Failed to copy nri linux resources"); ++ return false; ++ } ++ ++ if (!InitReply()) { ++ ERROR("Failed to init reply"); ++ return false; ++ } ++ m_update_req = nullptr; ++ return true; ++} ++ ++auto pluginResult::GetReplyUpdate() -> std::vector ++{ ++ return m_reply.update; ++} ++ ++auto pluginResult::MoveReplyAdjust() -> nri_container_adjustment * ++{ ++ nri_container_adjustment *ret = m_reply.adjust; ++ m_reply.adjust = nullptr; ++ return ret; ++} ++ ++auto pluginResult::GetReplyResources(const std::string &id) -> const nri_linux_resources * ++{ ++ nri_linux_resources *ret = NULL; ++ nri_container_update *update = m_updates[id]; ++ ret = update->linux->resources; ++ return ret; ++} ++ ++auto pluginResult::Apply(int32_t event, const nri_container_adjustment *adjust, nri_container_update **update, ++ size_t update_len, const std::string &plugin) -> bool ++{ ++ if (plugin.length() == 0) { ++ ERROR("Empty plugin name"); ++ return false; ++ } ++ if (event == CREATE_CONTAINER) { ++ if (!Adjust(adjust, plugin)) { ++ ERROR("Failed to do adjust to plugin: %s", plugin.c_str()); ++ return false; ++ } ++ ++ if (!Update(update, update_len, plugin)) { ++ ERROR("Failed to do update to plugin: %s", plugin.c_str()); ++ return false; ++ } ++ return true; ++ } else if (event == UPDATE_CONTAINER) { ++ if (!Update(update, update_len, plugin)) { ++ ERROR("Failed to do update to plugin: %s", plugin.c_str()); ++ return false; ++ } ++ return true; ++ } else if (event == STOP_CONTAINER) { ++ if (!Update(update, update_len, plugin)) { ++ ERROR("Failed to do update to plugin: %s", plugin.c_str()); ++ return false; ++ } ++ return true; ++ } else { ++ ERROR("Cannot apply response of invalid type %d", event); ++ return false; ++ } ++ return true; ++} ++ ++auto pluginResult::Adjust(const nri_container_adjustment *adjust, const std::string &plugin) -> bool ++{ ++ if (adjust == nullptr) { ++ return true; ++ } ++ ++ if (!AdjustAnnotations(adjust->annotations, plugin)) { ++ ERROR("Cannot adajust annotations by plugin %s", plugin.c_str()); ++ return false; ++ } ++ ++ if (!AdjustMounts(adjust->mounts, adjust->mounts_len, plugin)) { ++ ERROR("Cannot adajust mounts by plugin %s", plugin.c_str()); ++ return false; ++ } ++ ++ if (!AdjustEnv(adjust->env, adjust->env_len, plugin)) { ++ ERROR("Cannot adajust mounts by plugin %s", plugin.c_str()); ++ return false; ++ } ++ ++ if (!AdjustHooks(adjust->hooks, plugin)) { ++ ERROR("Cannot adajust hooks by plugin %s", plugin.c_str()); ++ return false; ++ } ++ ++ if (adjust->linux != nullptr) { ++ if (m_reply.adjust->linux == nullptr) { ++ m_reply.adjust->linux = (nri_linux_container_adjustment *)util_common_calloc_s(sizeof(nri_linux_container_adjustment)); ++ if (m_reply.adjust->linux == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ } ++ ++ if (!AdjustDevices(adjust->linux->devices, adjust->linux->devices_len, plugin)) { ++ ERROR("Cannot adajust devices by plugin %s", plugin.c_str()); ++ return false; ++ } ++ ++ if (!AdjustResources(adjust->linux->resources, plugin)) { ++ ERROR("Cannot adajust devices by plugin %s", plugin.c_str()); ++ return false; ++ } ++ ++ if (!AdjustCgroupsPath(adjust->linux->cgroups_path, plugin)) { ++ ERROR("Cannot adajust cgroups path by plugin %s", plugin.c_str()); ++ return false; ++ } ++ } ++ ++ if (!AdjustRlimits(adjust->rlimits, adjust->rlimits_len, plugin)) { ++ ERROR("Cannot adajust rlimits path by plugin %s", plugin.c_str()); ++ return false; ++ } ++ ++ return true; ++} ++ ++auto pluginResult::AdjustAnnotations(json_map_string_string *annos, const std::string &plugin) -> bool ++{ ++ if (annos == nullptr || annos->len == 0) { ++ return true; ++ } ++ ++ if (m_reply.adjust->annotations == nullptr) { ++ m_reply.adjust->annotations = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); ++ if (m_reply.adjust->annotations == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ } ++ ++ google::protobuf::Map del; ++ const char *id = m_conId.c_str(); ++ google::protobuf::Map mapAnno; ++ Transform::JsonMapToProtobufMapForString(annos, mapAnno); ++ ++ // if key is marked for remove, add pair to del, and delete key from annos ++ for (auto it = mapAnno.begin(); it != mapAnno.end();) { ++ const std::string &key = it->first; ++ char *out = NULL; ++ if (is_marked_for_removal(key.c_str(), &out)) { ++ del[out] = ""; ++ it = mapAnno.erase(it); ++ } else { ++ ++it; ++ } ++ } ++ ++ for (const auto &iter : mapAnno) { ++ std::string key = iter.first; ++ std::string value = iter.second; ++ auto it = del.find(key); ++ if (it != del.end()) { ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ m_owners[id].annotations.erase(key); ++ } ++ append_json_map_string_string(m_reply.adjust->annotations, NRIHelpers::MarkForRemoval(key).c_str(), ""); ++ } ++ ++ // set annotations's owner plugin ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ auto anno = m_owners[id].annotations.find(key); ++ if (anno != m_owners[id].annotations.end()) { ++ ERROR("plugins %s and %s both tried to set annotation: %s", plugin.c_str(), anno->second.c_str(), key.c_str()); ++ return false; ++ } ++ m_owners[id].annotations[key] = plugin; ++ } ++ ++ // add pair to m_reply.adjust ++ append_json_map_string_string(m_reply.adjust->annotations, key.c_str(), value.c_str()); ++ del.erase(key); ++ } ++ ++ // add del to m_reply.adjust ++ for (auto &pair : del) { ++ append_json_map_string_string(m_reply.adjust->annotations, NRIHelpers::MarkForRemoval(pair.first).c_str(), ""); ++ } ++ ++ return true; ++} ++ ++auto pluginResult::AdjustMounts(nri_mount **mounts, size_t mounts_size, const std::string &plugin) -> bool ++{ ++ if (mounts == nullptr || mounts_size == 0) { ++ return true; ++ } ++ ++ size_t i; ++ std::vector add; ++ std::map del; ++ std::string id = m_conId.c_str(); ++ ++ // first split removals from the rest of adjustments ++ for (i = 0; i < mounts_size; i++) { ++ char *out = NULL; ++ if (is_marked_for_removal(mounts[i]->destination, &out)) { ++ del[out] = mounts[i]; ++ } else { ++ add.push_back(mounts[i]); ++ } ++ } ++ ++ // next remove marked mounts from collected adjustments ++ nri_mount** cleard = nullptr; ++ size_t clearLen = 0; ++ ++ if (m_reply.adjust->mounts_len > 0) { ++ cleard = (nri_mount **)util_common_calloc_s(m_reply.adjust->mounts_len * sizeof(nri_mount *)); ++ if (cleard == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ for (i = 0; i < m_reply.adjust->mounts_len; i++) { ++ auto removed = del.find(m_reply.adjust->mounts[i]->destination); ++ if (removed != del.end()) { ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ m_owners[id].mounts.erase(m_reply.adjust->mounts[i]->destination); ++ } ++ continue; ++ } ++ cleard[clearLen] = copy_nri_mount(m_reply.adjust->mounts[i]); ++ if (cleard[clearLen] == nullptr) { ++ ERROR("Failed to copy nri mounts to cleard"); ++ return false; ++ } ++ clearLen++; ++ } ++ ++ NRIHelpers::freeArray(m_reply.adjust->mounts, m_reply.adjust->mounts_len); ++ m_reply.adjust->mounts = cleard; ++ m_reply.adjust->mounts_len = clearLen; ++ } ++ ++ // finally, apply additions to collected adjustments ++ size_t oldSize, newSize; ++ oldSize = m_reply.adjust->mounts_len * sizeof(nri_mount *); ++ newSize = oldSize + add.size() * sizeof(nri_mount *); ++ int ret = util_mem_realloc((void **)(&m_reply.adjust->mounts), newSize, (void *)m_reply.adjust->mounts, oldSize); ++ if (ret != 0) { ++ ERROR("Failed to realloc and assign nri mounts array"); ++ return false; ++ } ++ for (i = 0; i < add.size(); i++) { ++ // set mounts's owner plugin ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ auto mount = m_owners[id].mounts.find(add[i]->destination); ++ if (mount != m_owners[id].mounts.end()) { ++ ERROR("plugins %s and %s both tried to set mount: %s", plugin.c_str(), mount->second.c_str(), add[i]->destination); ++ return false; ++ } ++ m_owners[id].mounts[add[i]->destination] = plugin; ++ } ++ m_reply.adjust->mounts[m_reply.adjust->mounts_len] = copy_nri_mount(add[i]); ++ if (m_reply.adjust->mounts[m_reply.adjust->mounts_len] == nullptr) { ++ ERROR("Failed to copy add nri mounts to reply adjust"); ++ return false; ++ } ++ m_reply.adjust->mounts_len++; ++ } ++ ++ return true; ++} ++ ++auto pluginResult::AdjustEnv(nri_key_value **envs, size_t envs_size, const std::string &plugin) -> bool ++{ ++ if (envs == nullptr || envs_size == 0) { ++ return true; ++ } ++ ++ size_t i; ++ std::vector add; ++ std::map del; ++ std::string id = m_conId.c_str(); ++ ++ // first split removals from the rest of adjustments ++ for (i = 0; i < envs_size; i++) { ++ char *out = NULL; ++ if (is_marked_for_removal(envs[i]->key, &out)) { ++ del[out] = envs[i]; ++ } else { ++ add.push_back(envs[i]); ++ } ++ } ++ ++ // next remove marked mounts from collected adjustments ++ nri_key_value** cleard; ++ size_t clearLen = 0; ++ ++ if(m_reply.adjust->env_len > 0) { ++ cleard = (nri_key_value **)util_common_calloc_s(m_reply.adjust->env_len * sizeof(nri_key_value *)); ++ if (cleard == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ for (i = 0; i < m_reply.adjust->env_len; i++) { ++ auto removed = del.find(m_reply.adjust->env[i]->key); ++ if (removed != del.end()) { ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ m_owners[id].env.erase(m_reply.adjust->env[i]->key); ++ } ++ continue; ++ } ++ cleard[clearLen] = copy_nri_key_value(m_reply.adjust->env[i]); ++ if (cleard[clearLen] == nullptr) { ++ ERROR("Failed to copy nri env key value to cleard"); ++ return false; ++ } ++ clearLen++; ++ } ++ ++ NRIHelpers::freeArray(m_reply.adjust->env, m_reply.adjust->env_len); ++ m_reply.adjust->env = cleard; ++ m_reply.adjust->env_len = clearLen; ++ } ++ ++ // finally, apply additions to collected adjustments ++ size_t oldSize, newSize; ++ oldSize = m_reply.adjust->env_len * sizeof(nri_key_value *); ++ newSize = oldSize + add.size() * sizeof(nri_key_value *); ++ int ret = util_mem_realloc((void **)(&m_reply.adjust->env), newSize, m_reply.adjust->env, oldSize); ++ if (ret != 0) { ++ ERROR("Failed to realloc and assign nri env array"); ++ return false; ++ } ++ for (i = 0; i < add.size(); i++) { ++ // set env's owner plugin ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ auto env = m_owners[id].env.find(add[i]->key); ++ if (env != m_owners[id].env.end()) { ++ ERROR("plugins %s and %s both tried to set env: %s", plugin.c_str(), env->second.c_str(), add[i]->key); ++ return false; ++ } ++ m_owners[id].env[add[i]->key] = plugin; ++ } ++ m_reply.adjust->env[m_reply.adjust->env_len] = copy_nri_key_value(add[i]); ++ if (m_reply.adjust->env[m_reply.adjust->env_len] == nullptr) { ++ ERROR("Failed to copy add nri env to reply adjust"); ++ return false; ++ } ++ m_reply.adjust->env_len++; ++ } ++ ++ return true; ++} ++ ++auto pluginResult::AdjustHooks(const nri_hooks *hooks, const std::string &plugin) -> bool ++{ ++ if (hooks == nullptr) { ++ return true; ++ } ++ ++ if (m_reply.adjust->hooks == nullptr) { ++ m_reply.adjust->hooks = (nri_hooks *)util_common_calloc_s(sizeof(nri_hooks)); ++ if (m_reply.adjust->hooks == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ } ++ ++ nri_hooks * reply = m_reply.adjust->hooks; ++ ++ if (!merge_nri_hooks(reply->prestart, reply->prestart_len, (const nri_hook**)hooks->prestart, hooks->prestart_len)) { ++ ERROR("Failed to realloc and copy prestart hooks"); ++ return false; ++ } ++ ++ if (!merge_nri_hooks(reply->poststart, reply->poststart_len, (const nri_hook**)hooks->poststart, ++ hooks->poststart_len)) { ++ ERROR("Failed to realloc and copy poststart hooks"); ++ return false; ++ } ++ ++ if (!merge_nri_hooks(reply->poststop, reply->poststop_len, (const nri_hook**)hooks->poststop, hooks->poststop_len)) { ++ ERROR("Failed to realloc and copy poststop hooks"); ++ return false; ++ } ++ ++ /* TODO:zhongtao ++ * The OCI being used by the iSulad not supportes ++ * createRuntime/createContainer/startContainer currently. ++ */ ++ if (!merge_nri_hooks(reply->create_runtime, reply->create_runtime_len, (const nri_hook**)hooks->create_runtime, ++ hooks->create_runtime_len)) { ++ ERROR("Failed to realloc and copy create_runtime hooks"); ++ return false; ++ } ++ ++ if (!merge_nri_hooks(reply->create_container, reply->create_container_len, (const nri_hook**)hooks->create_container, ++ hooks->create_container_len)) { ++ ERROR("Failed to realloc and copy create_container hooks"); ++ return false; ++ } ++ ++ if (!merge_nri_hooks(reply->start_container, reply->start_container_len, (const nri_hook**)hooks->start_container, ++ hooks->start_container_len)) { ++ ERROR("Failed to realloc and copy start_container hooks"); ++ return false; ++ } ++ ++ return false; ++} ++ ++auto pluginResult::AdjustDevices(nri_linux_device **devices, size_t devices_size, const std::string &plugin) -> bool ++{ ++ if (devices_size == 0) { ++ return true; ++ } ++ ++ size_t i; ++ std::vector add; ++ std::map del; ++ std::string id = m_conId.c_str(); ++ ++ // first split removals from the rest of adjustments ++ for (i = 0; i < devices_size; i++) { ++ char *out = NULL; ++ if (is_marked_for_removal(devices[i]->path, &out)) { ++ del[out] = devices[i]; ++ } else { ++ add.push_back(devices[i]); ++ } ++ } ++ ++ // next remove marked mounts from collected adjustments ++ nri_linux_device** cleard; ++ size_t clearLen = 0; ++ ++ if (m_reply.adjust->linux->devices_len > 0) { ++ cleard = (nri_linux_device **)util_common_calloc_s(m_reply.adjust->linux->devices_len * sizeof(nri_linux_device *)); ++ if (cleard == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ for (i = 0; i < m_reply.adjust->linux->devices_len; i++) { ++ auto removed = del.find(m_reply.adjust->linux->devices[i]->path); ++ if (removed != del.end()) { ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ m_owners[id].devices.erase(m_reply.adjust->linux->devices[i]->path); ++ } ++ continue; ++ } ++ cleard[clearLen] = copy_nri_device(m_reply.adjust->linux->devices[i]); ++ if (cleard[clearLen] == nullptr) { ++ ERROR("Failed to copy nri linux device to cleard"); ++ return false; ++ } ++ clearLen++; ++ } ++ ++ NRIHelpers::freeArray(m_reply.adjust->linux->devices, m_reply.adjust->linux->devices_len); ++ m_reply.adjust->linux->devices = cleard; ++ m_reply.adjust->linux->devices_len = clearLen; ++ } ++ ++ // finally, apply additions to collected adjustments ++ size_t oldSize, newSize; ++ oldSize = m_reply.adjust->linux->devices_len * sizeof(nri_linux_device *); ++ newSize = oldSize + add.size() * sizeof(nri_linux_device *); ++ int ret = util_mem_realloc((void **)(&m_reply.adjust->linux->devices), newSize, m_reply.adjust->linux->devices, oldSize); ++ if (ret != 0) { ++ ERROR("Failed to realloc and assign nri devices array"); ++ return false; ++ } ++ for (i = 0; i < add.size(); i++) { ++ // set mounts's owner plugin ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ auto device = m_owners[id].devices.find(add[i]->path); ++ if (device != m_owners[id].devices.end()) { ++ ERROR("plugins %s and %s both tried to set devices: %s", plugin.c_str(), device->second.c_str(), add[i]->path); ++ return false; ++ } ++ m_owners[id].devices[add[i]->path] = plugin; ++ } ++ m_reply.adjust->linux->devices[m_reply.adjust->linux->devices_len] = copy_nri_device(add[i]); ++ if (m_reply.adjust->linux->devices[m_reply.adjust->linux->devices_len] == nullptr) { ++ ERROR("Failed to copy add nri devices to reply adjust"); ++ return false; ++ } ++ m_reply.adjust->linux->devices_len++; ++ } ++ ++ return true; ++} ++ ++auto pluginResult::AdjustResources(nri_linux_resources *resources, const std::string &plugin) -> bool ++{ ++ if (resources == nullptr) { ++ return true; ++ } ++ ++ if (m_reply.adjust->linux->resources == nullptr) { ++ m_reply.adjust->linux->resources = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); ++ if (m_reply.adjust->linux->resources == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ } ++ ++ std::string id = m_conId.c_str(); ++ nri_linux_resources *reply = m_reply.adjust->linux->resources; ++ ++ return ClaimAndCopyResources(resources, id, plugin, reply); ++} ++ ++bool pluginResult::ClaimAndCopyResources(nri_linux_resources *src, std::string &id, const std::string &plugin, ++ nri_linux_resources *dest) ++{ ++ size_t i; ++ if (src->memory != nullptr) { ++ if (src->memory->limit != nullptr) { ++ auto memLimit = m_owners[id].memLimit; ++ if (!memLimit.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's memory limit", plugin.c_str(), memLimit.c_str()); ++ return false; ++ } ++ m_owners[id].memLimit = plugin; ++ *dest->memory->limit = *src->memory->limit; ++ } ++ ++ if (src->memory->reservation != nullptr) { ++ auto memReservation = m_owners[id].memReservation; ++ if (!memReservation.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's memory reservation", plugin.c_str(), ++ memReservation.c_str()); ++ return false; ++ } ++ m_owners[id].memReservation = plugin; ++ *dest->memory->reservation = *src->memory->reservation; ++ } ++ ++ if (src->memory->swap != nullptr) { ++ auto memSwapLimit = m_owners[id].memSwapLimit; ++ if (!memSwapLimit.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's memory swap limit", plugin.c_str(), ++ memSwapLimit.c_str()); ++ return false; ++ } ++ m_owners[id].memSwapLimit = plugin; ++ *dest->memory->swap = *src->memory->swap; ++ } ++ ++ if (src->memory->kernel != nullptr) { ++ auto memKernelLimit = m_owners[id].memKernelLimit; ++ if (!memKernelLimit.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's memory kernel limit", plugin.c_str(), ++ memKernelLimit.c_str()); ++ return false; ++ } ++ m_owners[id].memKernelLimit = plugin; ++ *dest->memory->kernel = *src->memory->kernel; ++ } ++ ++ if (src->memory->kernel_tcp != nullptr) { ++ auto memTCPLimit = m_owners[id].memTCPLimit; ++ if (!memTCPLimit.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's memory tcp limit", plugin.c_str(), ++ memTCPLimit.c_str()); ++ return false; ++ } ++ m_owners[id].memTCPLimit = plugin; ++ *dest->memory->kernel_tcp = *src->memory->kernel_tcp; ++ } ++ ++ if (src->memory->swappiness != nullptr) { ++ auto memSwappiness = m_owners[id].memSwappiness; ++ if (!memSwappiness.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's memory swappiness", plugin.c_str(), ++ memSwappiness.c_str()); ++ return false; ++ } ++ m_owners[id].memSwappiness = plugin; ++ *dest->memory->swappiness = *src->memory->swappiness; ++ } ++ ++ if (src->memory->disable_oom_killer != nullptr) { ++ auto memDisableOomKiller = m_owners[id].memDisableOomKiller; ++ if (!memDisableOomKiller.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's memory disable_oom_killer", plugin.c_str(), ++ memDisableOomKiller.c_str()); ++ return false; ++ } ++ m_owners[id].memDisableOomKiller = plugin; ++ *dest->memory->disable_oom_killer = *src->memory->disable_oom_killer; ++ } ++ ++ if (src->memory->use_hierarchy != nullptr) { ++ auto memUseHierarchy = m_owners[id].memUseHierarchy; ++ if (!memUseHierarchy.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's memory use_hierarchy", plugin.c_str(), ++ memUseHierarchy.c_str()); ++ return false; ++ } ++ m_owners[id].memUseHierarchy = plugin; ++ *dest->memory->use_hierarchy = *src->memory->use_hierarchy; ++ } ++ } ++ ++ if (src->cpu != nullptr) { ++ if (src->cpu->shares != nullptr) { ++ auto cpuShares = m_owners[id].cpuShares; ++ if (!cpuShares.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's cpu shares", plugin.c_str(), cpuShares.c_str()); ++ return false; ++ } ++ m_owners[id].cpuShares = plugin; ++ *dest->cpu->shares = *src->cpu->shares; ++ } ++ ++ if (src->cpu->quota != nullptr) { ++ auto cpuQuota = m_owners[id].cpuQuota; ++ if (!cpuQuota.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's cpu quota", plugin.c_str(), cpuQuota.c_str()); ++ return false; ++ } ++ m_owners[id].cpuQuota = plugin; ++ *dest->cpu->quota = *src->cpu->quota; ++ } ++ ++ if (src->cpu->period != nullptr) { ++ auto cpuPeriod = m_owners[id].cpuPeriod; ++ if (!cpuPeriod.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's cpu period", plugin.c_str(), cpuPeriod.c_str()); ++ return false; ++ } ++ m_owners[id].cpuPeriod = plugin; ++ *dest->cpu->period = *src->cpu->period; ++ } ++ ++ if (src->cpu->realtime_runtime != nullptr) { ++ auto cpuRealtimePeriod = m_owners[id].cpuRealtimePeriod; ++ if (!cpuRealtimePeriod.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's cpu realtime_runtime", plugin.c_str(), ++ cpuRealtimePeriod.c_str()); ++ return false; ++ } ++ m_owners[id].cpuRealtimePeriod = plugin; ++ *dest->cpu->realtime_runtime = *src->cpu->realtime_runtime; ++ } ++ ++ if (src->cpu->realtime_period != nullptr) { ++ auto cpuRealtimePeriod = m_owners[id].cpuRealtimePeriod; ++ if (!cpuRealtimePeriod.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's cpu realtime_period", plugin.c_str(), ++ cpuRealtimePeriod.c_str()); ++ return false; ++ } ++ m_owners[id].cpuRealtimePeriod = plugin; ++ *dest->cpu->realtime_period = *src->cpu->realtime_period; ++ } ++ ++ if (src->cpu->cpus != nullptr) { ++ auto cpusetCpus = m_owners[id].cpusetCpus; ++ if (!cpusetCpus.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's cpu cpus", plugin.c_str(), cpusetCpus.c_str()); ++ return false; ++ } ++ m_owners[id].cpusetCpus = plugin; ++ *dest->cpu->cpus = *src->cpu->cpus; ++ } ++ ++ if (src->cpu->mems != nullptr) { ++ auto cpusetMems = m_owners[id].cpusetMems; ++ if (!cpusetMems.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's cpu mems", plugin.c_str(), cpusetMems.c_str()); ++ return false; ++ } ++ m_owners[id].cpusetMems = plugin; ++ *dest->cpu->mems = *src->cpu->mems; ++ } ++ } ++ ++ for (i = 0; i < src->hugepage_limits_len; i++) { ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ auto find = m_owners[id].hugepageLimits.find(src->hugepage_limits[i]->page_size); ++ if (find != m_owners[id].hugepageLimits.end()) { ++ ERROR("plugins %s and %s both tried to set hugepageLimits: %s", plugin.c_str(), find->second.c_str(), ++ src->hugepage_limits[i]->page_size); ++ return false; ++ } ++ m_owners[id].hugepageLimits[src->hugepage_limits[i]->page_size] = plugin; ++ } ++ } ++ ++ if (src->unified->len != 0) { ++ google::protobuf::Map mapAnno; ++ Transform::JsonMapToProtobufMapForString(src->unified, mapAnno); ++ for (const auto &iter : mapAnno) { ++ std::string key = iter.first; ++ std::string value = iter.second; ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ auto anno = m_owners[id].unified.find(key); ++ if (anno != m_owners[id].unified.end()) { ++ ERROR("plugins %s and %s both tried to set unified: %s", plugin.c_str(), anno->second.c_str(), ++ key.c_str()); ++ return false; ++ } ++ m_owners[id].unified[key] = plugin; ++ } ++ // add pair to m_reply.adjust ++ append_json_map_string_string(dest->unified, key.c_str(), value.c_str()); ++ } ++ } ++ ++ if (src->blockio_class != nullptr) { ++ auto blockioClass = m_owners[id].blockioClass; ++ if (!blockioClass.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's blockio_class", plugin.c_str(), blockioClass.c_str()); ++ return false; ++ } ++ m_owners[id].blockioClass = plugin; ++ dest->blockio_class = util_strdup_s(src->blockio_class); ++ } ++ ++ if (src->rdt_class != nullptr) { ++ auto rdtClass = m_owners[id].rdtClass; ++ if (!rdtClass.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's rdt_class", plugin.c_str(), rdtClass.c_str()); ++ return false; ++ } ++ m_owners[id].rdtClass = plugin; ++ dest->rdt_class = util_strdup_s(src->rdt_class); ++ } ++ return true; ++} ++ ++auto pluginResult::AdjustCgroupsPath(char *path, const std::string &plugin) -> bool ++{ ++ if (path == nullptr || strcmp(path, "") == 0) { ++ return true; ++ } ++ ++ std::string id = m_conId.c_str(); ++ ++ auto cgroupsPath = m_owners[id].cgroupsPath; ++ if (!cgroupsPath.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's cgroups path", plugin.c_str(), cgroupsPath.c_str()); ++ return false; ++ } ++ m_owners[id].cgroupsPath = plugin; ++ m_reply.adjust->linux->cgroups_path = util_strdup_s(path); ++ ++ return true; ++} ++ ++auto pluginResult::AdjustRlimits(nri_posix_rlimit **rlimits, size_t rlimits_len, const std::string &plugin) -> bool ++{ ++ if (rlimits_len == 0) { ++ return true; ++ } ++ ++ size_t i; ++ std::string id = m_conId.c_str(); ++ ++ size_t oldSize, newSize; ++ oldSize = m_reply.adjust->rlimits_len * sizeof(nri_posix_rlimit *); ++ newSize = oldSize + rlimits_len * sizeof(nri_posix_rlimit *); ++ int ret = util_mem_realloc((void **)(&m_reply.adjust->rlimits), newSize, m_reply.adjust->rlimits, oldSize); ++ if (ret != 0) { ++ ERROR("Failed to realloc and assign nri rlimits array"); ++ return false; ++ } ++ ++ for (i = 0; i < rlimits_len; i++) { ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ auto find = m_owners[id].rlimits.find(rlimits[i]->type); ++ if (find != m_owners[id].rlimits.end()) { ++ ERROR("plugins %s and %s both tried to set rlimits type: %s", plugin.c_str(), find->second.c_str(), rlimits[i]->type); ++ return false; ++ } ++ m_owners[id].rlimits[rlimits[i]->type] = plugin; ++ } ++ m_reply.adjust->rlimits[m_reply.adjust->rlimits_len] = copy_nri_posix_rlimit(rlimits[i]); ++ if (m_reply.adjust->rlimits[m_reply.adjust->rlimits_len] == nullptr) { ++ ERROR("Failed to copy add nri rlimits to reply adjust"); ++ return false; ++ } ++ m_reply.adjust->rlimits_len++; ++ } ++ return true; ++} ++ ++auto pluginResult::Update(nri_container_update **updates, size_t update_len, const std::string &plugin) -> bool ++{ ++ if (update_len == 0) { ++ return true; ++ } ++ ++ size_t i; ++ ++ for (i = 0; i < update_len; i++) { ++ nri_container_update *reply; ++ if (!GetContainerUpdate(updates[i], plugin, &reply)) { ++ ERROR("Failed to get container update in plugin result"); ++ return false; ++ } ++ ++ if (!UpdateResources(reply, updates[i], plugin)) { ++ ERROR("Failed to update container resources in plugin result"); ++ return false; ++ } ++ } ++ ++ return true; ++} ++ ++auto pluginResult::GetContainerUpdate(nri_container_update *update, const std::string &plugin, ++ nri_container_update **out) -> bool ++{ ++ if (update == nullptr || out == nullptr || plugin.empty()) { ++ ERROR("Empyt input args"); ++ return false; ++ } ++ ++ auto id = update->container_id; ++ ++ if (std::string(id) == m_conId) { ++ ERROR("Plugin %s asked update of %s during creation", plugin.c_str(), id); ++ return false; ++ } ++ ++ auto find = m_updates.find(id); ++ if (find != m_updates.end()) { ++ *out = m_updates[id]; ++ (*out)->ignore_failure = (*out)->ignore_failure && update->ignore_failure; ++ return true; ++ } ++ ++ *out = init_nri_container_update(id, update->ignore_failure); ++ if (*out == nullptr) { ++ ERROR("Failed to init nri container update"); ++ return false; ++ } ++ ++ m_updates[id] = *out; ++ ++ // for update requests delay appending the requested container (in the response getter) ++ if (m_conId != id) { ++ m_reply.update.push_back(*out); ++ } ++ ++ return true; ++} ++ ++auto pluginResult::UpdateResources(nri_container_update *reply, nri_container_update *u, ++ const std::string &plugin) -> bool ++{ ++ if (u->linux == nullptr || u->linux->resources == nullptr) { ++ return true; ++ } ++ ++ std::string id = u->container_id; ++ nri_linux_resources *resources; ++ ++ if (m_conId == id) { ++ resources = copy_nri_linux_resources(m_update_req); ++ if (resources == nullptr) { ++ ERROR("Failed to copy request's nri linux resources"); ++ return false; ++ } ++ } else { ++ resources = copy_nri_linux_resources(reply->linux->resources); ++ if (resources == nullptr) { ++ ERROR("Failed to copy reply's nri linux resources"); ++ return false; ++ } ++ } ++ ++ if (!ClaimAndCopyResources(u->linux->resources, id, plugin, resources)) { ++ ERROR("Failed to claim and copy resources in plugin result"); ++ return false; ++ } ++ ++ // update reply from copy on success ++ reply->linux->resources = copy_nri_linux_resources(resources); ++ if (reply->linux->resources == nullptr) { ++ ERROR("Failed to copy resources's nri linux resources to reply"); ++ return false; ++ } ++ ++ return true; ++} +\ No newline at end of file +diff --git a/src/daemon/nri/nri_result.h b/src/daemon/nri/nri_result.h +index f2896ea0..4f26385a 100644 +--- a/src/daemon/nri/nri_result.h ++++ b/src/daemon/nri/nri_result.h +@@ -63,7 +63,7 @@ struct owners { + + struct resultReply { + nri_container_adjustment* adjust; +- std::vector update; ++ std::vector update; + }; + + using resultOwners = std::map; +@@ -71,16 +71,19 @@ using resultOwners = std::map; + class pluginResult { + public: + pluginResult() = default; ++ pluginResult(std::string conId); + +- ~pluginResult() = default; ++ virtual ~pluginResult(); + ++ auto Init() -> bool; + auto InitByConId(std::string conId) -> bool; + auto InitByUpdateReq(nri_update_container_request *req) -> bool; + + auto GetReplyUpdate() -> std::vector; +- auto GetReplyAdjust() -> nri_container_adjustment *; ++ auto MoveReplyAdjust() -> nri_container_adjustment *; ++ auto GetReplyResources(const std::string &id) -> const nri_linux_resources *; + +- auto Apply(int32_t event, nri_container_adjustment *adjust, nri_container_update **update, size_t update_len, ++ auto Apply(int32_t event, const nri_container_adjustment *adjust, nri_container_update **update, size_t update_len, + const std::string &plugin) -> bool; + auto Update(nri_container_update **updates, size_t update_len, const std::string &plugin) -> bool; + +@@ -90,12 +93,12 @@ private: + + auto InitReply(void) -> bool; + +- auto Adjust(nri_container_adjustment *adjust, const std::string &plugin) -> bool; ++ auto Adjust(const nri_container_adjustment *adjust, const std::string &plugin) -> bool; + + auto AdjustAnnotations(json_map_string_string *annos, const std::string &plugin) -> bool; + auto AdjustMounts(nri_mount **mounts, size_t mounts_size, const std::string &plugin) -> bool; + auto AdjustEnv(nri_key_value **envs, size_t envs_size, const std::string &plugin) -> bool; +- auto AdjustHooks(nri_hooks *hooks, const std::string &plugin) -> bool; ++ auto AdjustHooks(const nri_hooks *hooks, const std::string &plugin) -> bool; + auto AdjustDevices(nri_linux_device **devices, size_t devices_size, const std::string &plugin) -> bool; + auto AdjustResources(nri_linux_resources *resources, const std::string &plugin) -> bool; + bool ClaimAndCopyResources(nri_linux_resources *src, std::string &id, const std::string &plugin, +@@ -107,7 +110,9 @@ private: + std::string m_conId; + nri_linux_resources *m_update_req; + resultReply m_reply; ++ // update plugin -> update context + std::map m_updates; ++ // adjust plugin -> adjust context + resultOwners m_owners; + }; + +diff --git a/src/daemon/nri/plugin.cc b/src/daemon/nri/plugin.cc +new file mode 100644 +index 00000000..904677c4 +--- /dev/null ++++ b/src/daemon/nri/plugin.cc +@@ -0,0 +1,417 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-03-15 ++ * Description: provide plugin class definition ++ *********************************************************************************/ ++ ++#include "plugin.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "utils.h" ++#include "cstruct_wrapper.h" ++ ++// same as containerd ++std::string DefaultNRIVersion = "2.0.0-beta.2+unknown"; ++// same as containerd ++std::string DefaultNRIRuntimeName = "v2"; ++// defualt timeout for wait: 2s ++const int64_t DefaultWaitTimeout = 2000; ++const uint64_t SECOND_TO_NANOS = 1000000000; ++ ++// init client conn ++NRIPlugin::NRIPlugin(std::string &idx, std::string &name, std::string &config) ++{ ++ m_idx = idx; ++ m_name = name; ++ m_config = config; ++ m_closed = false; ++ m_external = false; ++ m_pid = -1; ++} ++ ++NRIPlugin::NRIPlugin(int fd, std::string &name) ++{ ++ m_sockFds.push_back(fd); ++ m_name = name; ++ m_closed = false; ++ m_external = true; ++ m_pid = -1; ++} ++ ++// wait for plugin to register, then configure it. ++auto NRIPlugin::Start(int64_t registry_timeout, int64_t request_timeout) -> bool ++{ ++ Errors error; ++ ++ // todo: what if timeout is 0 or other invalid value? ++ ++ if (!Connect(request_timeout)) { ++ ERROR("Failed to connect nri plugin %s", m_name.c_str()); ++ return false; ++ } ++ ++ if (!WaitForReady(registry_timeout)) { ++ ERROR("Failed to wait plugin %s ready with timeout %ld", m_name.c_str(), registry_timeout); ++ return false; ++ } ++ ++ if (!Configure(error)) { ++ ERROR("Failed to configure nri plugin %s", m_name.c_str()); ++ return false; ++ } ++ ++ return true; ++} ++ ++auto NRIPlugin::shutdown() -> void ++{ ++ if (!Close()) { ++ ERROR("Failed to close plugin %s", m_name.c_str()); ++ } ++ ++ if (!Stop()) { ++ ERROR("Failed to stop plugin %s", m_name.c_str()); ++ } ++} ++ ++// create client connect ++auto NRIPlugin::Connect(int64_t timeout) -> bool ++{ ++ if (m_name.empty()) { ++ ERROR("Empty nri plugin name"); ++ return false; ++ } ++ ++ if (nri_plugin_connect(m_name.c_str(), m_sockFds[0], timeout * SECOND_TO_NANOS) != 0) { ++ ERROR("Failed to create a new client for plugin %s", m_name.c_str()); ++ return false; ++ } ++ ++ return true; ++} ++ ++// close a plugin shutting down its multiplexed ttrpc connections. ++auto NRIPlugin::Close() -> bool ++{ ++ if (IsClose()) { ++ return true; ++ } ++ ++ if (nri_plugin_disconnect(m_name.c_str()) != 0) { ++ ERROR("Failed to close plugin %s", m_name.c_str()); ++ return false; ++ } ++ ++ SetClose(); ++ return true; ++} ++ ++// stop a plugin (if it was launched by us) ++auto NRIPlugin::Stop() -> bool ++{ ++ if (m_external) { ++ return true; ++ } ++ ++ if (m_pid <= 0) { ++ WARN("Invalid pid %d", m_pid); ++ return false; ++ } ++ ++ int nret = kill(m_pid, SIGKILL); ++ if (nret < 0 && errno != ESRCH) { ++ SYSWARN("Can not kill process (pid=%d) with SIGKILL", m_pid); ++ return false; ++ } ++ ++ if (util_waitpid_with_timeout(m_pid, DefaultWaitTimeout, NULL) != 0) { ++ WARN("Failed to wait for plugin %s to exit", m_name.c_str()); ++ return false; ++ } ++ return true; ++} ++ ++// Name returns a string indentication for the plugin. ++auto NRIPlugin::GetName() -> const std::string & ++{ ++ return m_name; ++} ++ ++auto NRIPlugin::GetIndex() -> const std::string & ++{ ++ return m_idx; ++} ++ ++auto NRIPlugin::GetPeerSockFd() -> uint32_t ++{ ++ return m_sockFds[1]; ++} ++ ++auto NRIPlugin::GetQualifiedName() -> std::string ++{ ++ return m_idx + "-" + m_name; ++} ++ ++void NRIPlugin::SetReady(void) ++{ ++ std::unique_lock lock(m_readyMutex); ++ m_ready = true; ++ m_condition.notify_one(); ++} ++ ++void NRIPlugin::SetPid(int pid) ++{ ++ m_pid = pid; ++} ++ ++auto NRIPlugin::CreateSocketPair() -> bool ++{ ++ int fds[2]; ++ ++ if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) == -1) { ++ ERROR("Failed to create socketpair"); ++ return false; ++ } ++ ++ m_sockFds.push_back(fds[0]); ++ m_sockFds.push_back(fds[1]); ++ return true; ++} ++ ++auto NRIPlugin::Configure(Errors &error) -> bool ++{ ++ auto req = makeUniquePtrCStructWrapper(free_nri_configure_request); ++ if (req == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ req->get()->config = isula_strdup_s(m_config.c_str()); ++ req->get()->runtime_name = isula_strdup_s(NRIRruntime.c_str()); ++ req->get()->runtime_version = isula_strdup_s(NRIVersion.c_str()); ++ ++ nri_configure_response *resp = nullptr; ++ if (nri_plugin_configure(m_name.c_str(), req->get(), &resp) != 0) { ++ ERROR("Failed to configure plugin %s", m_name.c_str()); ++ return false; ++ } ++ ++ auto resp_wrapper = makeUniquePtrCStructWrapper(resp, free_nri_configure_response); ++ if (resp_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ EventMask events = resp_wrapper->get()->events; ++ if (events != 0) { ++ EventMask extra = events & ~ValidEvents; ++ if (extra != 0) { ++ ERROR("Invalid plugin events: %d", extra); ++ return false; ++ } ++ } else { ++ events = ValidEvents; ++ } ++ ++ m_events = events; ++ return true; ++} ++ ++auto NRIPlugin::Synchronize(std::vector &pods, std::vector &containers, ++ nri_container_update ***update, size_t update_len, Errors &error) -> bool ++{ ++ size_t i; ++ ++ auto req = makeUniquePtrCStructWrapper(free_nri_synchronize_request); ++ if (req == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ if (pods.size() != 0) { ++ req->get()->pods = (nri_pod_sandbox **)util_common_calloc_s(pods.size() * sizeof(nri_pod_sandbox *)); ++ if (req->get()->pods == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ for (i = 0; i < pods.size(); i++) { ++ req->get()->pods[i] = pods[i]; ++ req->get()->pods_len++; ++ } ++ } ++ ++ if (containers.size() != 0) { ++ req->get()->containers = (nri_container **)util_common_calloc_s(containers.size() * sizeof(nri_container *)); ++ if (req->get()->containers == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ for (i = 0; i < containers.size(); i++) { ++ req->get()->containers[i] = containers[i]; ++ req->get()->containers_len++; ++ } ++ } ++ ++ nri_synchronize_response *resp = nullptr; ++ if (nri_plugin_synchronize(m_name.c_str(), req->get(), &resp) != 0) { ++ ERROR("Failed to synchronize plugin %s", m_name.c_str()); ++ return false; ++ } ++ ++ auto resp_wrapper = makeUniquePtrCStructWrapper(resp, free_nri_synchronize_response); ++ if (resp_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ *update = resp->update; ++ resp->update = nullptr; ++ update_len = resp->update_len; ++ resp->update_len = 0; ++ return true; ++} ++ ++auto NRIPlugin::CreateContainer(nri_create_container_request *req, nri_create_container_response **resp, ++ Errors &error) -> bool ++{ ++ if (req == nullptr) { ++ ERROR("Invalid input"); ++ return false; ++ } ++ ++ if (IsSetEvent(CREATE_CONTAINER) == false) { ++ return true; ++ } ++ ++ if (nri_plugin_create_container(m_name.c_str(), req, resp) != 0) { ++ ERROR("Failed to create container by plugin %s", m_name.c_str()); ++ return false; ++ } ++ ++ return true; ++} ++ ++auto NRIPlugin::UpdateContainer(nri_update_container_request *req, nri_update_container_response **resp, ++ Errors &error) -> bool ++{ ++ if (req == nullptr) { ++ ERROR("Invalid input"); ++ return false; ++ } ++ ++ if (!IsSetEvent(UPDATE_CONTAINER)) { ++ return true; ++ } ++ ++ if (nri_plugin_update_container(m_name.c_str(), req, resp) != 0) { ++ ERROR("Failed to update container by plugin %s", m_name.c_str()); ++ return false; ++ } ++ return true; ++} ++ ++auto NRIPlugin::StopContainer(nri_stop_container_request *req, nri_stop_container_response **resp, ++ Errors &error) -> bool ++{ ++ if (req == nullptr) { ++ ERROR("Invalid input"); ++ return false; ++ } ++ ++ if (!IsSetEvent(STOP_CONTAINER)) { ++ return true; ++ } ++ ++ if (nri_plugin_stop_container(m_name.c_str(), req, resp) != 0) { ++ ERROR("Failed to stop container by plugin %s", m_name.c_str()); ++ return false; ++ } ++ return true; ++} ++ ++// do nothing with event ++auto NRIPlugin::StateChange(nri_state_change_event *evt, Errors &error) -> bool ++{ ++ if (evt == nullptr) { ++ ERROR("Invalid input"); ++ return false; ++ } ++ ++ if (!IsSetEvent(evt->event)) { ++ return true; ++ } ++ ++ if (nri_plugin_state_change(m_name.c_str(), evt) != 0) { ++ ERROR("Failed to state change by plugin %s", m_name.c_str()); ++ return false; ++ } ++ return true; ++} ++ ++auto NRIPlugin::WaitForReady(int64_t timeout) -> bool ++{ ++ auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(timeout * 1000); ++ std::unique_lock readyMutex(m_readyMutex); ++ ++ if (timeout == 0) { ++ m_condition.wait(readyMutex); ++ return true; ++ } ++ ++ if (m_condition.wait_until(readyMutex, deadline) == std::cv_status::timeout) { ++ return false; ++ } ++ ++ return true; ++} ++ ++auto NRIPlugin::IsSetEvent(EventMask e) -> bool ++{ ++ return (m_events & (1 << (e - 1))) != 0; ++} ++ ++auto NRIPlugin::IsClose() -> bool ++{ ++ ReadGuard lock(m_mutex); ++ return m_closed; ++} ++ ++void NRIPlugin::SetClose() ++{ ++ WriteGuard lock(m_mutex); ++ m_closed = true; ++} +\ No newline at end of file +diff --git a/src/daemon/nri/plugin.h b/src/daemon/nri/plugin.h +index f60a9b3d..ed298be6 100644 +--- a/src/daemon/nri/plugin.h ++++ b/src/daemon/nri/plugin.h +@@ -38,17 +38,14 @@ public: + virtual ~NRIPlugin() = default; + // wait for plugin to register, then configure it. + auto Start(int64_t registry_timeout, int64_t request_timeout) -> bool; +- // close a plugin shutting down its multiplexed ttrpc connections. +- auto Close(void) -> bool; +- // stop a plugin (if it was launched by us) +- auto Stop(void) -> bool; ++ ++ auto shutdown() -> void; + + // Name returns a string indentication for the plugin. + auto GetName(void) -> const std::string &; + auto GetIndex(void) -> const std::string &; + auto GetPeerSockFd(void) -> uint32_t; + auto GetQualifiedName(void) -> std::string; +- + void SetReady(void); + void SetPid(int pid); + +@@ -58,8 +55,8 @@ public: + + auto Configure(Errors &error) -> bool; + // Only called in external plugin scenario +- auto Synchronize(std::vector> pods, +- std::vector> &containers, nri_container_update ***update, size_t update_len, ++ auto Synchronize(std::vector &pods, ++ std::vector &containers, nri_container_update ***update, size_t update_len, + Errors &error) -> bool; + auto CreateContainer(nri_create_container_request *req, nri_create_container_response **resp, Errors &error) -> bool; + auto UpdateContainer(nri_update_container_request *req, nri_update_container_response **resp, Errors &error) -> bool; +@@ -72,6 +69,13 @@ private: + auto WaitForReady(int64_t timeout) -> bool; + auto IsSetEvent(EventMask e) -> bool; + ++ // close a plugin shutting down its multiplexed ttrpc connections. ++ auto Close(void) -> bool; ++ // stop a plugin (if it was launched by us) ++ auto Stop(void) -> bool; ++ ++ void SetClose(void); ++ + private: + RWMutex m_mutex; + bool m_external; +@@ -83,7 +87,7 @@ private: + std::vector m_sockFds; + std::string m_localFileName; + std::string m_peerFileName; +- // TODO:zhontao monitor? ++ // TODO: plugin monitor? + bool m_closed; + std::mutex m_readyMutex; + bool m_ready; +diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c +index f81a9141..69f6dbf0 100644 +--- a/src/utils/cutils/utils.c ++++ b/src/utils/cutils/utils.c +@@ -1705,4 +1705,4 @@ void set_child_process_pdeathsig(void) + if (prctl(PR_SET_PDEATHSIG, SIGKILL) < 0) { + SYSERROR("Failed to set child process pdeathsig"); + } +-} +\ No newline at end of file ++} +-- +2.25.1 + diff --git a/iSulad.spec b/iSulad.spec new file mode 100644 index 0000000..6db1f73 --- /dev/null +++ b/iSulad.spec @@ -0,0 +1,1118 @@ +%global _version 2.1.5 +%global _release 11 +%global is_systemd 1 +%global enable_criv1 1 +%global enable_cdi 1 +%global enable_shimv2 1 +%global is_embedded 1 +%global cpp_std 17 + +Name: iSulad +Version: %{_version} +Release: %{_release} +Summary: Lightweight Container Runtime Daemon +License: Mulan PSL v2 +URL: https://gitee.com/openeuler/iSulad +Source: https://gitee.com/openeuler/iSulad/repository/archive/v%{version}.tar.gz +BuildRoot: {_tmppath}/iSulad-%{version} + +Patch0001: 0001-code-improve-for-sandbox.cc.patch +Patch0002: 0002-fix-compile-error-with-protobuf-25.1-and-grpc-1.60.x.patch +Patch0003: 0003-bugfix-for-mount-point-remains-under-special-circums.patch +Patch0004: 0004-do-not-cleanup-if-the-directory-does-not-exist.patch +Patch0005: 0005-module-only-deletes-the-temporary-files-it-creates.patch +Patch0006: 0006-skip-devmapper-ut.patch +Patch0007: 0007-update-annotations-and-add-ci-cases.patch +Patch0008: 0008-bug-fix-for-device-cgroup-ulimt-oci-update.patch +Patch0009: 0009-improve-dt-for-oci-spec-update.patch +Patch0010: 0010-open-run-container-with-dev-volume-testcase.patch +Patch0011: 0011-add-cpu-usage-nano-cores-for-sandbox.patch +Patch0012: 0012-sleep-some-time-in-ServiceWorkThread-to-prevent-the-.patch +Patch0013: 0013-restore-name-for-rename-failed.patch +Patch0014: 0014-2371-Allow-iSulad-to-pull-load-image-with-symlink.patch +Patch0015: 0015-Replace-http-parser-dependency-with-lcr.patch +Patch0016: 0016-add-more-detailed-log-information-for-load-sandbox.patch +Patch0017: 0017-bugfix-for-the-concurrency-competition-between-the-r.patch +Patch0018: 0018-add-concurrent-load-test.patch +Patch0019: 0019-get-the-realpath-of-the-host-path-for-archive-when-c.patch +Patch0020: 0020-bugfix-for-wrong-goto-branch.patch +Patch0021: 0021-bugfix-for-wrong-dynamic-allocation-object-type.patch +Patch0022: 0022-add-swap-usage-in-cri.patch +Patch0023: 0023-add-benchmark-result-of-perf-test-in-cri.patch +Patch0024: 0024-add-support-for-systemd-cgroup-driver.patch +Patch0025: 0025-add-ci-cases-for-systemd-cgroup-driver.patch +Patch0026: 0026-move-systemd_cgroup-CI-test-to-manual-cases.patch +Patch0027: 0027-feature-add-support-for-cgroup-v2-metrics.patch +Patch0028: 0028-use-supervisor-to-notify-sandbox-exit-event.patch +Patch0029: 0029-refactor-cgroup-module.patch +Patch0030: 0030-adaptor-unit-test-for-cgroup-module.patch +Patch0031: 0031-cgroup-v2-does-not-support-isulad-setting-cpu_rt-opt.patch +Patch0032: 0032-add-test-that-isulad-cannot-set-cpu_rt-parameters-wh.patch +Patch0033: 0033-fix-sandbox-container-bool-value-uninitialized.patch +Patch0034: 0034-bugfix-for-cpurt.sh.patch +Patch0035: 0035-monitor-cgroup-oom-killed-event-and-update-to-cri-of.patch +Patch0036: 0036-add-ci-cases-for-oomkilled-monitor.patch +Patch0037: 0037-add-cgroup-v2-doc.patch +Patch0038: 0038-add-modify-for-cgroup-v2-ci-test.patch +Patch0039: 0039-fix-run-ubuntu-container-bug-in-inspect.sh.patch +Patch0040: 0040-add-support-for-GetContainerEvents.patch +Patch0041: 0041-fix-cpurt-init-bug-for-systemd-cgroup.patch +Patch0042: 0042-fix-message-queue-concurrent-bug.patch +Patch0043: 0043-specify-runtime-as-runc-for-oom-test-CI.patch +Patch0044: 0044-set-oomkilled-in-cri.patch +Patch0045: 0045-add-cri-1.29-update-design-doc.patch +Patch0046: 0046-oom-monitor-in-manual-cases.patch +Patch0047: 0047-add-usage-restrictions-for-CRI-1.29-update.patch +Patch0048: 0048-CDI-interface-definition.patch +Patch0049: 0049-distinguish-between-runtime-and-runtime_cmd-in-isula.patch +Patch0050: 0050-Use-user-defined-shm-for-CRI-request.patch +Patch0051: 0051-Fix-memory-leak-in-set_connected_container_shm_path.patch +Patch0052: 0052-init-enable_pod_events-as-false.patch +Patch0053: 0053-remove-container-root-path-in-rt_lcr_rm-if-lcr-runti.patch +Patch0054: 0054-ensure-sandbox-can-be-removed-if-sandbox-container-r.patch +Patch0055: 0055-bugfix-for-shim-timeout-exit-error-log-changes.patch +Patch0056: 0056-bugfix-for-the-pre-created-pipe-was-not-closed-when-.patch +Patch0057: 0057-add-debug-msg-info-in-image_load.sh.patch +Patch0058: 0058-empty-pointer-check-in-lcr_rt_ops.patch +Patch0059: 0059-modify-some-grpc-status-codes-of-cri-in-case-of-erro.patch +Patch0060: 0060-cdi-return-int-instead-of-error-string.patch +Patch0061: 0061-cdi-support-modules-operate-registry-annotations.patch +Patch0062: 0062-do-not-umount-shmpath-for-sandbox-container.patch +Patch0063: 0063-remove-default-systemd-cgroup-and-enable-cri-v1-valu.patch +Patch0064: 0064-cdi-support-module-cache.patch +Patch0065: 0065-change-default-subscribe-timeout-to-5min.patch +Patch0066: 0066-cdi-support-modules-version-spec-spec_dirs-device.patch +Patch0067: 0067-cdi-support-modules-container_edits-parser.patch +Patch0068: 0068-cdi-invoke-cdi-operate-when-init-isulad-and-create-c.patch +Patch0069: 0069-bugfix-fix-cni_operate_ut-ut.patch +Patch0070: 0070-isolate-sandboxer-code-by-using-macro.patch +Patch0071: 0071-Remove-sandboxer-ut-if-sandboxer-is-not-enabled.patch +Patch0072: 0072-cdi-design-doc.patch +Patch0073: 0073-bugfix-cdi-version-check.patch +Patch0074: 0074-bugfix-of-background-execution-exec-error-command.patch +Patch0075: 0075-bugfix-for-setting-cpu-rt-to-a-negative-value-when-e.patch +Patch0076: 0076-cdi-add-UT.patch +Patch0077: 0077-remove-extra-s-in-CreateContainerLogSymlink.patch +Patch0078: 0078-allow-env-variable-has-an-empty-value.patch +Patch0079: 0079-Fix-Failed-to-execute-image-pull-on-name-tag-digest-.patch +Patch0080: 0080-bugfix-for-hostname-env-set-only-once.patch +Patch0081: 0081-set-the-sandbox-status-to-not-ready-under-abnormal-c.patch +Patch0082: 0082-fix-shim-controller-set-incorrect-sandbox-status-sta.patch +Patch0083: 0083-fix-bug-for-invalid-env-write.patch +Patch0084: 0084-trim-key-value-for-env.patch +Patch0085: 0085-cdi-allow-env-variable-has-an-empty-value.patch +Patch0086: 0086-cdi-test-case-and-gateway.patch +Patch0087: 0087-code-improve.patch +Patch0088: 0088-testcase-close-cdi-testcase.patch +Patch0089: 0089-docs-update-cni-doc.patch +Patch0090: 0090-modify-the-user-error-log-to-be-the-same-as-before.patch +Patch0091: 0091-add-enable-cri-v1-in-k8s-integration.patch +Patch0092: 0092-isolate-oom-monitor-codes.patch +Patch0093: 0093-change-fork-process-exit-mode.patch +Patch0094: 0094-fix-error-log-for-verify_cpu_realtime.patch +Patch0095: 0095-bugfix-change-max-network-name-len.patch +Patch0096: 0096-del-useless-info.patch +Patch0097: 0097-code-improve.patch +Patch0098: 0098-cdi-add-debug-info.patch +Patch0099: 0099-bugfix-cni-network-name-UT.patch +Patch0100: 0100-bugfix-malloc-right-type-size.patch +Patch0101: 0101-use-isula_clean_path-rather-than-realpath.patch +Patch0102: 0102-fix-false-engine-rootpath-reference.patch +Patch0103: 0103-bugfix-add-note.patch +Patch0104: 0104-bugfix-adapt-network-name-max-len.patch +Patch0105: 0105-start-sandbox-before-setup-network-by-default.patch +Patch0106: 0106-Revert-use-isula_clean_path-rather-than-realpath.patch +Patch0107: 0107-bugfix-for-start-sandbox-before-setup-network-by-def.patch +Patch0108: 0108-skip-test-rely-on-docker.io.patch +Patch0109: 0109-modify-default-registry-mirrors-in-ci-test.patch +Patch0110: 0110-add-timestamp-in-PodSandboxStatu-response.patch +Patch0111: 0111-bugfix-for-file-param-verify.patch +Patch0112: 0112-bugfix-change-cni-log-info.patch +Patch0113: 0113-move-shutdown-handle-after-init-module.patch +Patch0114: 0114-bugfix-for-null-pointer-reference.patch +Patch0115: 0115-bugfix-for-m_criService-shutdown.patch +Patch0116: 0116-fix-bug-in-ci-test.patch +Patch0117: 0117-NRI-add-nri-head-file-and-common-func.patch +Patch0118: 0118-skip-calling-cni-plugin-cleanup-when-network-namespa.patch +Patch0119: 0119-nri-add-convert-and-utils-impl-for-nri.patch +Patch0120: 0120-get-realpath-before-ns-mountpoint-verification.patch +Patch0121: 0121-nri-impl-for-nri-plugin-and-adaption.patch + +%ifarch x86_64 aarch64 +Provides: libhttpclient.so()(64bit) +Provides: libisula_client.so()(64bit) +Provides: libisulad_tools.so()(64bit) +%endif + +%if 0%{?is_systemd} +# Systemd 230 and up no longer have libsystemd-journal +BuildRequires: pkgconfig(systemd) +Requires: systemd-units +%else +Requires(post): chkconfig +Requires(preun): chkconfig +# This is for /sbin/service +Requires(preun): initscripts +%endif + +%if 0%{?is_embedded} +BuildRequires: sqlite-devel +Requires: sqlite +%endif + +%if %{defined openeuler} +BuildRequires: gtest-devel gmock-devel +%endif + +%define lcrver_lower 2.1.4-0 +%define lcrver_upper 2.1.5-0 + +BuildRequires: libisula-devel > %{lcrver_lower} libisula-devel < %{lcrver_upper} +BuildRequires: cmake gcc-c++ yajl-devel +BuildRequires: grpc grpc-plugins grpc-devel protobuf-devel ncurses-devel +BuildRequires: libcurl libcurl-devel libarchive-devel device-mapper-devel +BuildRequires: libseccomp-devel libcap-devel libselinux-devel libwebsockets libwebsockets-devel +BuildRequires: systemd-devel git +BuildRequires: libevhtp-devel libevent-devel +%if 0%{?enable_shimv2} +BuildRequires: lib-shim-v2 lib-shim-v2-devel +%endif + + +Requires: libisula > %{lcrver_lower} libisula < %{lcrver_upper} +Requires: grpc protobuf +Requires: libcurl +Requires: libseccomp +Requires: libcap libselinux libwebsockets libarchive device-mapper +Requires: systemd +Requires: (docker-runc or runc) +BuildRequires: libevhtp libevent +%if 0%{?enable_shimv2} +Requires: lib-shim-v2 +%endif + +%description +This is a umbrella project for gRPC-services based Lightweight Container +Runtime Daemon, written by C. + +%prep +%autosetup -n iSulad-v%{_version} -Sgit -p1 + +%build +mkdir -p build +cd build +%cmake \ + -DDEBUG=ON \ + -DCMAKE_SKIP_RPATH=TRUE \ + -DLIB_INSTALL_DIR=%{_libdir} \ + -DCMAKE_INSTALL_PREFIX=/usr \ +%if 0%{?enable_criv1} + -DENABLE_CRI_API_V1=ON \ +%if 0%{?enable_cdi} + -DENABLE_CDI=ON \ +%endif +%endif +%if 0%{?enable_shimv2} + -DENABLE_SHIM_V2=ON \ +%endif +%if %{defined openeuler} + -DENABLE_UT=OFF \ +%endif + -DENABLE_GRPC_REMOTE_CONNECT=OFF \ + -DENABLE_GRPC=ON \ + -DCMAKE_CXX_STANDARD=%{cpp_std} \ + ../ + +sed -i "10 a\# undef linux" grpc/src/api/services/cri/v1alpha/api.pb.h +%if 0%{?enable_criv1} +sed -i "10 a\# undef linux" grpc/src/api/services/cri/v1/api_v1.pb.h +%endif + +%make_build + +%check +%if %{defined openeuler} +cd build +# registry_images_ut and volume_ut must run with root user +ctest -E "registry_images_ut|volume_ut" +%endif + +%install +rm -rf %{buildroot} +cd build +install -d $RPM_BUILD_ROOT/%{_libdir} +install -m 0755 ./src/libisula_client.so %{buildroot}/%{_libdir}/libisula_client.so +install -m 0755 ./src/utils/http/libhttpclient.so %{buildroot}/%{_libdir}/libhttpclient.so +install -m 0755 ./src/libisulad_tools.so %{buildroot}/%{_libdir}/libisulad_tools.so + +install -d $RPM_BUILD_ROOT/%{_libdir}/pkgconfig +install -m 0640 ./conf/isulad.pc %{buildroot}/%{_libdir}/pkgconfig/isulad.pc + +install -d $RPM_BUILD_ROOT/%{_bindir} + +install -m 0755 ./src/isula %{buildroot}/%{_bindir}/isula +install -m 0755 ./src/isulad-shim %{buildroot}/%{_bindir}/isulad-shim + +install -m 0755 ./src/isulad %{buildroot}/%{_bindir}/isulad + +install -d $RPM_BUILD_ROOT/%{_includedir}/isulad + +install -d $RPM_BUILD_ROOT/%{_sysconfdir}/isulad +install -m 0640 ../src/contrib/config/daemon.json %{buildroot}/%{_sysconfdir}/isulad/daemon.json +install -m 0440 ../src/contrib/config/daemon_constants.json %{buildroot}/%{_sysconfdir}/isulad/daemon_constants.json +install -m 0640 ../src/contrib/config/seccomp_default.json %{buildroot}/%{_sysconfdir}/isulad/seccomp_default.json + +install -d $RPM_BUILD_ROOT/%{_sysconfdir}/default/isulad +install -m 0640 ../src/contrib/config/config.json %{buildroot}/%{_sysconfdir}/default/isulad/config.json +install -m 0640 ../src/contrib/config/systemcontainer_config.json %{buildroot}/%{_sysconfdir}/default/isulad/systemcontainer_config.json +install -m 0550 ../src/contrib/sysmonitor/isulad-check.sh %{buildroot}/%{_sysconfdir}/default/isulad/isulad-check.sh + +mkdir -p $RPM_BUILD_ROOT/%{_sysconfdir}/sysmonitor/process +cp ../src/contrib/sysmonitor/isulad-monit $RPM_BUILD_ROOT/etc/sysmonitor/process + +install -d $RPM_BUILD_ROOT/%{_sysconfdir}/default/isulad/hooks +install -m 0640 ../src/contrib/config/hooks/default.json %{buildroot}/%{_sysconfdir}/default/isulad/hooks/default.json + +install -d $RPM_BUILD_ROOT/%{_sysconfdir}/sysconfig +install -p -m 0640 ../src/contrib/config/iSulad.sysconfig $RPM_BUILD_ROOT/%{_sysconfdir}/sysconfig/iSulad + +%if 0%{?is_systemd} +install -d $RPM_BUILD_ROOT/%{_unitdir} +install -p -m 0640 ../src/contrib/init/isulad.service $RPM_BUILD_ROOT/%{_unitdir}/isulad.service +%else +install -d $RPM_BUILD_ROOT/%{_initddir} +install -p -m 0640 ../src/contrib/init/isulad.init $RPM_BUILD_ROOT/%{_initddir}/isulad.init +%endif + +%clean +rm -rf %{buildroot} + +%pre +# support update from lcrd to isulad, will remove in next version +if [ "$1" = "2" ]; then +%if 0%{?is_systemd} +systemctl stop lcrd &>/dev/null +systemctl disable lcrd &>/dev/null +if [ -e %{_sysconfdir}/isulad/daemon.json ];then + sed -i 's#/etc/default/lcrd/hooks#/etc/default/isulad/hooks#g' %{_sysconfdir}/isulad/daemon.json +fi +%else +/sbin/chkconfig --del lcrd &>/dev/null +%endif +fi + +%post +if ! getent group isula > /dev/null; then + groupadd --system isula +fi + +if [ "$1" = "1" ]; then +%if 0%{?is_systemd} +systemctl enable isulad +systemctl start isulad +%else +/sbin/chkconfig --add isulad +%endif +elif [ "$1" = "2" ]; then +%if 0%{?is_systemd} +# support update from lcrd to isulad, will remove in next version +if [ -e %{_unitdir}/lcrd.service.rpmsave ]; then + mv %{_unitdir}/lcrd.service.rpmsave %{_unitdir}/isulad.service + sed -i 's/lcrd/isulad/g' %{_unitdir}/isulad.service +fi +# During the isulad upgrade process, the isulad service may still be running, but the service may be unavailable +# due to configuration updates and other reasons. +# it may fail if the X package is upgraded synchronously with isulad and depends on the isulad command, +# For example syscontianer-tools and lxcfs-tools. +# Therefore, after upgrading isulad, if the original status of isulad is running, +# we need to restart isulad to ensure that the service is available during the upgrade process. +systemctl status isulad | grep 'Active:' | grep 'running' +if [ $? -eq 0 ]; then + systemctl restart isulad +fi +%else +/sbin/service isulad status | grep 'Active:' | grep 'running' +if [ $? -eq 0 ]; then + /sbin/service isulad restart +fi +%endif +fi + +if ! getent group isula > /dev/null; then + groupadd --system isula +fi + +%preun +%if 0%{?is_systemd} +%systemd_preun isulad +%else +if [ $1 -eq 0 ] ; then + /sbin/service isulad stop >/dev/null 2>&1 + /sbin/chkconfig --del isulad +fi +%endif + +%postun +%if 0%{?is_systemd} +%systemd_postun_with_restart isulad +%else +if [ "$1" -ge "1" ] ; then + /sbin/service isulad condrestart >/dev/null 2>&1 || : +fi +%endif + +%files +%attr(0600,root,root) %{_sysconfdir}/sysmonitor/process/isulad-monit +%attr(0550,root,root) %{_sysconfdir}/default/isulad/isulad-check.sh +%defattr(0640,root,root,0750) +%{_sysconfdir}/isulad +%{_sysconfdir}/isulad/* +%{_sysconfdir}/default/* +%defattr(-,root,root,-) +%if 0%{?is_systemd} +%{_unitdir}/isulad.service +%attr(0640,root,root) %{_unitdir}/isulad.service +%else +%{_initddir}/isulad.init +%attr(0640,root,root) %{_initddir}/isulad.init +%endif +%attr(0755,root,root) %{_libdir}/pkgconfig +%attr(0640,root,root) %{_libdir}/pkgconfig/isulad.pc +%defattr(0755,root,root,0755) +%{_bindir}/* +%{_libdir}/* +%attr(0640,root,root) %{_sysconfdir}/sysconfig/iSulad +%attr(0640,root,root) %{_sysconfdir}/isulad/daemon.json + +%config(noreplace,missingok) %{_sysconfdir}/sysconfig/iSulad +%config(noreplace,missingok) %{_sysconfdir}/isulad/daemon.json +%if 0%{?is_systemd} +%config(noreplace,missingok) %{_unitdir}/isulad.service +%else +%config(noreplace,missingok) %{_initddir}/isulad.init +%endif + +%changelog +* Mon Aug 19 2024 zhongtao - 2.1.5-11 +- Type: update +- ID: NA +- SUG: NA +- DESC: add impl for nri and bugfix + +* Tue Jun 11 2024 zhongtao - 2.1.5-10 +- Type: update +- ID: NA +- SUG: NA +- DESC: code improve and bugfix + +* Sat May 11 2024 liuxu - 2.1.5-9 +- Type: update +- ID: NA +- SUG: NA +- DESC: upgrade from upstream + +* Mon Apr 29 2024 zhongtao - 2.1.5-8 +- Type: update +- ID: NA +- SUG: NA +- DESC: remove extra %s in CreateContainerLogSymlink and allow env variable has an empty value + +* Thu Apr 25 2024 zhongtao - 2.1.5-7 +- Type: update +- ID: NA +- SUG: NA +- DESC: bugfix of background execution exec error command and setting negative cpu-rt issue + +* Sat Apr 20 2024 liuxu - 2.1.5-6 +- Type: update +- ID: NA +- SUG: NA +- DESC: upgrade from upstream + +* Sat Apr 20 2024 liuxu - 2.1.5-5 +- Type: update +- ID: NA +- SUG: NA +- DESC: upgrade from upstream + +* Tue Apr 02 2024 jikai - 2.1.5-4 +- Type: update +- ID: NA +- SUG: NA +- DESC: upgrade from upstream + +* Sat Mar 30 2024 zhongtao - 2.1.5-3 +- Type: update +- ID: NA +- SUG: NA +- DESC: add support for cgroup v2 + +* Tue Mar 19 2024 zhongtao - 2.1.5-2 +- Type: update +- ID: NA +- SUG: NA +- DESC: upgrade from upstream + +* Tue Jan 30 2024 zhongtao - 2.1.5-1 +- Type: update +- ID: NA +- SUG: NA +- DESC: update to v2.1.5 + +* Fri Dec 29 2023 yangjiaqi - 2.1.4-5 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: keep the service status unchanged after iSulad service upgrade + +* Thu Dec 28 2023 leizhongkai - 2.1.4-4 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: upgrade from upstream + +* Wed Dec 20 2023 zhongtao - 2.1.4-3 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: upgrade from upstream + +* Wed Nov 15 2023 zhongtao - 2.1.4-2 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: delete libisulad_img.so and compilation dependency: lxc, and add running dependency:runc + +* Tue Nov 14 2023 zhongtao - 2.1.4-1 +- Type: update +- ID: NA +- SUG: NA +- DESC: update to v2.1.4 + +* Wed Sep 13 2023 xuxuepeng - 2.1.3-2 +- Type: update +- ID: NA +- SUG: NA +- DESC: Add vsock support for exec + +* Tue Aug 29 2023 xuxuepeng - 2.1.3-1 +- Type: update +- ID: NA +- SUG: NA +- DESC: update to v2.1.3 + +* Mon Aug 28 2023 zhangxiaoyu - 2.1.2-8 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: modify lxc dependence to docker-runc dependence + +* Mon Aug 21 2023 zhangxiaoyu - 2.1.2-7 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: enable grpc and fix compile failed + +* Wed Aug 09 2023 zhangxiaoyu - 2.1.2-6 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: modify daemon json default runtime to runc + +* Thu Jul 20 2023 zhangxiaoyu - 2.1.2-5 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: compile using c++17 + +* Mon May 29 2023 zhangxiaoyu - 2.1.2-4 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: bugfix for memleak and malloc + +* Thu May 25 2023 zhongtao - 2.1.2-3 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: upgrade from upstream + +* Fri May 12 2023 zhangxiaoyu - 2.1.2-2 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix remote grpc macro + +* Thu May 11 2023 zhangxiaoyu - 2.1.2-1 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update from upstream to update cri + +* Fri Mar 24 2023 wangrunze - 2.1.1-6 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update from upstream to include remote feature + +* Thu Mar 16 2023 zhangxiaoyu - 2.1.1-5 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: use CURLOPT_XFERINFOFUNCTION instead of deprecated CURLOPT_PROGRESSFUNCTION since curl 7.32.0 + +* Wed Feb 22 2023 zhangxiaoyu - 2.1.1-4 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: upgrade from upstream + +* Thu Feb 16 2023 zhangxiaoyu - 2.1.1-3 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: add check + +* Mon Feb 06 2023 zhangxiaoyu - 2.1.1-2 +- Type: update +- ID: NA +- SUG: NA +- DESC: modify dependence from lcr to libisula + +* Mon Feb 06 2023 zhangxiaoyu - 2.1.1-1 +- Type: update +- ID: NA +- SUG: NA +- DESC: update to v2.1.1 + +* Tue Jan 03 2023 zhangxiaoyu - 2.0.18-1 +- Type: update +- ID: NA +- SUG: NA +- DESC: update to v2.0.18 + +* Thu Dec 22 2022 zhangxiaoyu - 2.0.17-14 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: remove clean_module_fill_ctx for libisulad_img.so + +* Mon Dec 19 2022 zhangxiaoyu - 2.0.17-13 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update lcr dependence version + +* Fri Dec 16 2022 zhangxiaoyu - 2.0.17-12 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update from upstream + +* Tue Dec 06 2022 zhangxiaoyu - 2.0.17-11 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update from upstream + +* Mon Nov 28 2022 yangjiaqi - 2.0.17-10 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: remove chmod 751 permission for dirs by engine when user-remap enabled + +* Fri Nov 25 2022 zhangxiaoyu - 2.0.17-9 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: add primary group to additional groups + +* Mon Nov 21 2022 zhangxiaoyu - 2.0.17-8 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix cannot install isulad and unknown option + +* Wed Nov 16 2022 zhangxiaoyu - 2.0.17-7 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update tar package from source + +* Wed Nov 02 2022 wangrunze - 2.0.17-6 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix cleanup module memory leak + +* Tue Nov 01 2022 zhangxiaoyu - 2.0.17-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: add ut and bugfix for device mapper and websocket + +* Mon Oct 31 2022 wujing - 2.0.17-4 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from openEuler + +* Wed Oct 19 2022 zhangxiaoyu - 2.0.17-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: add required package lcr clibcni lower and upper version + +* Mon Oct 10 2022 zhangxiaoyu - 2.0.17-2 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: use epoll instead of select for wait_exit_fifo + +* Sun Oct 09 2022 zhangxiaoyu - 2.0.17-1 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: update iSulad version to 2.0.17-1 + +* Thu Sep 29 2022 haozi007 - 2.0.16-8 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from openEuler + +* Tue Sep 20 2022 zhangxiaoyu - 2.0.16-7 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: change libisulad_tools.so mode + +* Thu Sep 15 2022 zhangxiaoyu - 2.0.16-6 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: shield upgrade error if lcrd not exist + +* Tue Sep 13 2022 zhangxiaoyu - 2.0.16-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: calculate timezone by tm_gmtoff + +* Thu Sep 08 2022 zhangxiaoyu - 2.0.16-4 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: invalid free default-runtime and cri-runtime after free json-confs + +* Wed Sep 07 2022 zhangxiaoyu - 2.0.16-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sycn patches from openeuler/iSulad + +* Tue Aug 30 2022 leizhongkai - 2.0.16-2 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream openeuler/iSulad + +* Tue Aug 23 2022 wangfengtu - 2.0.16-1 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: update iSulad version to 2.0.16-1 + +* Mon Aug 22 2022 zhongtao - 2.0.15-6 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: remove rpath by cmake + +* Wed Aug 17 2022 haozi007 - 2.0.15-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sycn patches from openeuler + +* Mon Aug 15 2022 wangfengtu - 2.0.15-4 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: change default umask to 0022 + +* Tue Aug 9 2022 haozi007 - 2.0.15-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sycn patches from openeuler + +* Mon Aug 1 2022 chengzeruizhi - 2.0.15-2 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sycn patches from openeuler branch + +* Fri Jul 8 2022 haozi007 - 2.0.15-1 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: update version to v2.0.15 + +* Fri Jul 8 2022 haozi007 - 2.0.14-11 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: add limit to lcr version + +* Wed Jun 22 2022 yangjiaqi - 2.0.14-10 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream openeuler/iSulad + +* Tue Jun 21 2022 zhangxiaoyu - 2.0.14-9 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream openeuler/iSulad + +* Wed Jun 15 2022 chengzeruizhi - 2.0.14-8 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream openeuler/iSulad + +* Tue May 31 2022 zhangxiaoyu - 2.0.14-7 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: fix type convert, add null pointer check, remove unuse macro + +* Tue May 31 2022 zhangxiaoyu - 2.0.14-6 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: fix different type convert and add check to arguments + +* Mon May 30 2022 chengzrz - 2.0.14-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: seccomp optimization + +* Fri May 27 2022 haozi007 - 2.0.14-4 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: improve fuzz test for pw and gr parser + +* Tue May 24 2022 wangfengtu - 2.0.14-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: fix install error when android + +* Tue May 24 2022 wangfengtu - 2.0.14-2 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: do not mkdir of isulad if no controller found + +* Mon May 23 2022 haozi007 - 2.0.14-1 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: update version to v2.0.14 + +* Mon May 16 2022 haozi007 - 2.0.13-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream openEuler/iSulad + +* Tue May 10 2022 hejunjie - 2.0.13-4 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: bionic adaptation, increase lcov coverage + +* Thu May 5 2022 hejunjie - 2.0.13-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: bionic adaptation for pwgr obj parser + +* Mon Apr 25 2022 zhangxiaoyu - 2.0.13-2 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: sync from upstream + +* Mon Apr 18 2022 wangfengtu - 2.0.13-1 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update version to v2.0.13 + +* Fri Mar 25 2022 wangfengtu - 2.0.12-1 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update version to v2.0.12 + +* Thu Mar 17 2022 haozi007 - 2.0.11-6 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: remove unnecessary error message + +* Thu Mar 17 2022 wangfengtu - 2.0.11-5 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix uid/gid error when load image + +* Wed Mar 09 2022 wangfengtu - 2.0.11-4 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: add provides of libisulad_tools.so + +* Thu Mar 03 2022 wangfengtu - 2.0.11-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: Add the function of isolating the user namespaces + +* Thu Mar 03 2022 wangfengtu - 2.0.11-2 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: add libisulad_tools.so + +* Thu Feb 24 2022 wangfengtu - 2.0.11-1 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: update version to v2.0.11 + +* Wed Jan 12 2022 wangfengtu - 2.0.10-15 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix compile error of isula-transform + +* Wed Jan 12 2022 zhangxiaoyu - 2.0.10-14 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix compile error with grpc 1.41.x + +* Tue Jan 4 2022 wangfengtu - 2.0.10-13 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix compile error when building embedded image + +* Mon Dec 27 2021 wangfengtu - 2.0.10-12 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: sync patches from upstream + +* Thu Dec 09 2021 chengzeruizhi - 2.0.10-11 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fixed a bug that occurs when starting a container in host mode + +* Thu Dec 09 2021 wangfengtu - 2.0.10-10 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: remove dependance of sqlite + +* Mon Dec 06 2021 gaohuatao - 2.0.10-9 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: specify version + +* Fri Dec 03 2021 wangfengtu - 2.0.10-8 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix undefined reference to `service_arguments_free' in libisulad_img.so + +* Thu Dec 02 2021 wangfengtu - 2.0.10-7 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: sync patches from upstream + +* Tue Nov 23 2021 chengzeruizhi - 2.0.10-6 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: modify the procedure of running a pod + +* Fri Nov 19 2021 gaohuatao - 2.0.10-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream + +* Fri Nov 19 2021 wangfengtu - 2.0.10-4 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix memleak when use multiple --volumes-from + +* Tue Nov 16 2021 wujing - 2.0.10-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: add shimv2 build switch + +* Tue Nov 16 2021 wujing - 2.0.10-2 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: remove build platform restrictions + +* Tue Nov 09 2021 gaohuatao - 2.0.10-1 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update from openeuler + +* Tue Oct 19 2021 wangfengtu - 2.0.9-20211019.121837.gitf067b3ce +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: strip sha256 prefix when decrease hold references + +* Fri Jun 25 2021 wujing - 2.0.9-20210625.165022.git5a088d9c +- Type: update to v2.0.9 +- ID: NA +- SUG: NA +- DESC: update from master + +* Tue May 18 2021 wangfengtu - 2.0.8-20210518.144540.git5288ed92 +- Type: sync from upstream +- ID: NA +- SUG: NA +- DESC: update from master + +* Fri Mar 26 2021 zhangxiaoyu - 2.0.8-20210326.094027.gitac974aa6 +- Type: sync from upstream +- ID: NA +- SUG: NA +- DESC: update from master + +* Tue Mar 23 2021 haozi007 - 20210323.094917.git7e6aa593 +- Type: sync from upstream +- ID: NA +- SUG: NA +- DESC: update from master + +* Tue Feb 2 2021 lifeng - 2.0.8-20210202.153251.gite082dcf3 +- Type: sync from upstream +- ID: NA +- SUG: NA +- DESC: update from master + +* Mon Jan 18 2021 lifeng - 2.0.8-20210118.195254.git077e10f2 +- Type: sync from upstream +- ID: NA +- SUG: NA +- DESC: update from master + +* Wed Dec 30 2020 lifeng - 2.0.8-20201230.155843.git6557a6eb +- Type: update to v2.0.8 +- ID: NA +- SUG: NA +- DESC: update from master + +* Mon Dec 7 2020 zhangxiaoyu - 2.0.7-20201207.151847.gita1fce123 +- Type: update +- ID: NA +- SUG: NA +- DESC: update from master + +* Sat Dec 5 2020 lifeng - 2.0.7-20201205.145752.gita461cc51 +- Type: bugfix +- ID:NA +- SUG:NA +- DESC: ignore list containers errors + +* Thu Dec 3 2020 haozi007 - 2.0.7-20201203.190902.git48f598fd +- Type:update from master +- ID:NA +- SUG:NA +- DESC: update from master + +* Sat Nov 28 2020 lifeng - 2.0.7-20201128.095506.git1e1623a5 +- Type: bugfix +- ID:NA +- SUG:NA +- DESC: Mounts: only qsort the configed mounts and make possible to bind mount /proc and /sys/fs. +- related lxc PR fixed: +- 1.add check whether have /proc mounts entry, if has, skip the auto +- 2.mount cgroup before do mount entrys +- 3.pass if the mount on top of /proc and the source of the mount is a proc filesystem + +* Wed Nov 25 2020 wangfengtu - 2.0.7-20201125.165149.git7d150c3c +- Type: bugfix +- ID:NA +- SUG:NA +- DESC: update from openeuler + +* Wed Nov 25 2020 wangfengtu - 2.0.6-20201125.160534.git9fb5e75d +- Type: bugfix +- ID:NA +- SUG:NA +- DESC: fix rpath not work + +* Thu Nov 12 2020 gaohuatao - 2.0.6-20201112.193005.git8a6b73c8 +- Type: update from openeuler +- ID:NA +- SUG:NA +- DESC: update from openeuler + +* Wed Oct 14 2020 lifeng68 - 2.0.6-20201014.152749.gitc8a43925 +- Type: upgrade to v2.0.6 +- ID:NA +- SUG:NA +- DESC: upgrade to v2.0.6 + +* Fri Sep 18 2020 - 2.0.5-20200918.112827.git9aea9b75 +- Type:bugfix +- ID:NA +- SUG:NA +- DESC: modify log level to warn + +* Mon Sep 14 2020 - 2.0.5-20200914.172527.gitae86920a +- Type:bugfix +- ID:NA +- SUG:NA +- DESC: remove unused config + +* Thu Sep 10 2020 - 2.0.5-20200910.144345.git71b1055b +- Type:enhancement +- ID:NA +- SUG:NA +- DESC: add chrpath + +* Fri Sep 04 2020 zhangxiaoyu - 2.0.5-20200904.114315.gitff1761c3 +- Type:enhancement +- ID:NA +- SUG:NA +- DESC: upgrade from v2.0.3 to v2.0.5 + +* Wed Sep 02 2020 YoungJQ - 2.0.3-20200902.114727.git6d945f26 +- Type:enhancement +- ID:NA +- SUG:NA +- DESC: modify source0 address diff --git a/sources b/sources new file mode 100644 index 0000000..33e8c41 --- /dev/null +++ b/sources @@ -0,0 +1 @@ +40038ec1745dd1df6dae47782e645f96 v2.1.5.tar.gz -- cgit v1.2.3