diff options
Diffstat (limited to '0190-modified-the-procedure-of-start-pod-and-set-up-netwo.patch')
| -rw-r--r-- | 0190-modified-the-procedure-of-start-pod-and-set-up-netwo.patch | 399 |
1 files changed, 399 insertions, 0 deletions
diff --git a/0190-modified-the-procedure-of-start-pod-and-set-up-netwo.patch b/0190-modified-the-procedure-of-start-pod-and-set-up-netwo.patch new file mode 100644 index 0000000..084b8b2 --- /dev/null +++ b/0190-modified-the-procedure-of-start-pod-and-set-up-netwo.patch @@ -0,0 +1,399 @@ +From 1d42d850b71e3a0e269f268411999c265a1a8f5b Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Mon, 10 Feb 2025 10:44:01 +0800 +Subject: [PATCH 190/198] modified the procedure of start pod and set up + network in cri v1 + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + .../v1/v1_cri_pod_sandbox_manager_service.cc | 276 +++++++++++++----- + .../v1/v1_cri_pod_sandbox_manager_service.h | 3 + + 2 files changed, 206 insertions(+), 73 deletions(-) + +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +index 43705853..62464acd 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +@@ -19,6 +19,7 @@ + #include <isula_libutils/host_config.h> + #include <isula_libutils/container_config.h> + #include <isula_libutils/auto_cleanup.h> ++#include <isula_libutils/container_network_settings.h> + #include <algorithm> + + #include "checkpoint_handler.h" +@@ -273,12 +274,6 @@ void PodSandboxManagerService::SetupSandboxNetwork(const std::shared_ptr<sandbox + std::map<std::string, std::string> networkOptions; + networkOptions["UID"] = config.metadata().uid(); + +- if (prepare_network_namespace(sandboxKey.c_str(), false, 0) != 0) { +- error.Errorf("Failed to prepare network namespace: %s", sandboxKey.c_str()); +- ERROR("Failed to prepare network namespace: %s", sandboxKey.c_str()); +- return; +- } +- + // Setup networking for the sandbox. + m_pluginManager->SetUpPod(config.metadata().namespace_(), config.metadata().name(), + Network::DEFAULT_NETWORK_INTERFACE_NAME, sandbox->GetId(), stdAnnos, networkOptions, +@@ -295,85 +290,137 @@ void PodSandboxManagerService::SetupSandboxNetwork(const std::shared_ptr<sandbox + DEBUG("set %s ready", sandbox->GetId().c_str()); + } + +-auto PodSandboxManagerService::RunPodSandbox(const runtime::v1::PodSandboxConfig &config, +- const std::string &runtimeHandler, Errors &error) -> std::string ++void PodSandboxManagerService::GenerateNetworkSetting(std::string &sandboxKey, std::string &network_setting_json, Errors &error) ++{ ++ container_network_settings *settings = NULL; ++ __isula_auto_free char *jerr = NULL; ++ __isula_auto_free char *setting_json { nullptr }; ++ ++ settings = (container_network_settings *)util_common_calloc_s(sizeof(container_network_settings)); ++ if (settings == NULL) { ++ ERROR("Out of memory"); ++ error.Errorf("Out of memory"); ++ return; ++ } ++ ++ auto settingsWarpper = std::unique_ptr<CStructWrapper<container_network_settings>>(new CStructWrapper<container_network_settings>(settings, free_container_network_settings)); ++ ++ settings->sandbox_key = util_strdup_s(sandboxKey.c_str()); ++ if (settings->sandbox_key == NULL) { ++ ERROR("Failed to set sandbox key for network setting"); ++ error.Errorf("Failed to set sandbox key for network setting"); ++ return; ++ } ++ ++ setting_json = container_network_settings_generate_json(settings, nullptr, &jerr); ++ if (setting_json == nullptr) { ++ error.Errorf("Get network settings json err:%s", jerr); ++ } ++ ++ network_setting_json = std::string(setting_json); ++} ++ ++void PodSandboxManagerService::StartPodSandboxAndSetupNetowrk(std::shared_ptr<sandbox::Sandbox> sandbox, std::string &sandboxKey, std::string &sandboxName, std::string &networkMode, Errors &error) + { +- std::string response_id; +- std::string sandboxName; +- sandbox::RuntimeInfo runtimeInfo; +- std::string networkMode; +- std::string sandboxKey; +- std::string jsonCheckpoint; +- std::string network_setting_json; +- runtime::v1::PodSandboxConfig copyConfig = config; + cri_container_message_t msg = { 0 }; +-#ifdef ENABLE_NRI +- Errors nriErr; +-#endif ++ std::string network_setting_json; + +- // Step 1: Parepare sandbox name, runtime and networkMode +- PrepareSandboxData(config, runtimeHandler, sandboxName, runtimeInfo, networkMode, error); ++ // Step 7.2.1: Call sandbox create. ++ sandbox->Create(error); + if (error.NotEmpty()) { +- return response_id; ++ ERROR("Failed to create sandbox: %s", sandboxName.c_str()); ++ return; + } + +- // Step 2: Pull the image for the sandbox. +- // Maybe we should pull image in shim controller ? +- // But pull image interface is only in CRI image service, and it can't be called in shim controller, +- // so we pull image in CRI pod service. +- const std::string &image = m_podSandboxImage; +- if (!EnsureSandboxImageExists(image, runtimeInfo.sandboxer, error)) { +- ERROR("Failed to pull sandbox image %s: %s", image.c_str(), error.NotEmpty() ? error.GetCMessage() : ""); +- error.Errorf("Failed to pull sandbox image %s: %s", image.c_str(), error.NotEmpty() ? error.GetCMessage() : ""); +- return response_id; ++ msg.container_id = sandbox->GetId().c_str(); ++ msg.sandbox_id = sandbox->GetId().c_str(); ++ msg.type = CRI_CONTAINER_MESSAGE_TYPE_CREATED; ++ mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &msg); ++ ++ // Step 7.2.2: Save network settings json to disk ++ // Update network settings before start sandbox since sandbox container will use the sandbox key ++ if (namespace_is_cni(networkMode.c_str())) { ++ GenerateNetworkSetting(sandboxKey, network_setting_json, error); ++ // If saving network settings failed, ignore error ++ if (error.NotEmpty()) { ++ ERROR("Failed to generate networksetting :%s", error.GetCMessage()); ++ return; ++ } ++ ++ sandbox->UpdateNetworkSettings(network_setting_json, error); ++ // If saving network settings failed, ignore error ++ if (error.NotEmpty()) { ++ ERROR("%s", error.GetCMessage()); ++ return; ++ } + } + +- // Step 3: Prepare sandbox checkpoint +- PrepareSandboxCheckpoint(config, jsonCheckpoint, error); ++ // Step 7.2.3: Call sandbox start. ++ sandbox->Start(error); + if (error.NotEmpty()) { +- return response_id; ++ ERROR("Failed to start sandbox: %s", sandboxName.c_str()); ++ return; + } + +- // Step 4: Update sandbox instance config +- UpdateSandboxConfig(copyConfig, jsonCheckpoint, error); ++ // Step 7.2.4:Setup networking for the sandbox. ++ SetupSandboxNetwork(sandbox, network_setting_json, error); + if (error.NotEmpty()) { +- return response_id; ++ goto stop_sandbox; + } + +- // Step 5: Prepare sandboxKey ++ // Step 7.2.5:update Network settings after setup network to update ip info. + if (namespace_is_cni(networkMode.c_str())) { +- // cleanup sandboxKey file in DeleteSandbox +- PrepareSandboxKey(sandboxKey, error); +- if (error.NotEmpty()) { +- return response_id; ++ Errors tmpErr; ++ sandbox->UpdateNetworkSettings(network_setting_json, tmpErr); ++ // If saving network settings failed, ignore error ++ if (tmpErr.NotEmpty()) { ++ WARN("%s", tmpErr.GetCMessage()); + } + } + +- // Step 6: Create sandbox instance +- auto sandbox = sandbox::SandboxManager::GetInstance()->CreateSandbox(sandboxName, runtimeInfo, sandboxKey, +- networkMode, copyConfig, image, error); ++ // Step 7.2.6: Save sandbox to disk ++ sandbox->Save(error); + if (error.NotEmpty()) { +- if (namespace_is_cni(networkMode.c_str())) { +- (void)remove_network_namespace_file(sandboxKey.c_str()); ++ ERROR("Failed to save sandbox, %s", sandboxName.c_str()); ++ goto cleanup_network; ++ } ++ return; ++ ++cleanup_network: ++ if (namespace_is_cni(sandbox->GetNetMode().c_str())) { ++ Errors clearErr; ++ ClearCniNetwork(sandbox, clearErr); ++ if (clearErr.NotEmpty()) { ++ ERROR("Failed to clean cni network: %s", clearErr.GetCMessage()); + } +- return response_id; + } + +- // Step 7: Setup networking for the sandbox. ++stop_sandbox: ++ Errors stopError; ++ CRIHelpers::StopContainerHelper(m_cb, sandbox->GetId(), 0, stopError); ++ WARN("Error stop container: %s: %s", sandbox->GetId().c_str(), stopError.GetCMessage()); ++} ++ ++void PodSandboxManagerService::SetupNetowrkAndStartPodSandbox(std::shared_ptr<sandbox::Sandbox> sandbox, std::string &sandboxName, std::string &networkMode, Errors &error) ++{ ++ cri_container_message_t msg = { 0 }; ++ std::string network_setting_json; ++ ++ // Step 7.1.1: Setup networking for the sandbox. + // Setup sandbox network before create sandbox since the remote create might fail for sandbox + SetupSandboxNetwork(sandbox, network_setting_json, error); + if (error.NotEmpty()) { +- goto cleanup_sandbox; ++ return; + } + +- // Step 8: Save sandbox to disk ++ // Step 7.1.2: Save sandbox to disk + sandbox->Save(error); + if (error.NotEmpty()) { + ERROR("Failed to save sandbox, %s", sandboxName.c_str()); + goto cleanup_network; + } + +- // Step 9: Call sandbox create. ++ // Step 7.1.3: Call sandbox create. + sandbox->Create(error); + if (error.NotEmpty()) { + ERROR("Failed to create sandbox: %s", sandboxName.c_str()); +@@ -385,7 +432,7 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1::PodSandboxConfig + msg.type = CRI_CONTAINER_MESSAGE_TYPE_CREATED; + mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &msg); + +- // Step 10: Save network settings json to disk ++ // Step 7.1.4: Save network settings json to disk + // Update network settings before start sandbox since sandbox container will use the sandbox key + if (namespace_is_cni(networkMode.c_str())) { + Errors tmpErr; +@@ -396,26 +443,15 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1::PodSandboxConfig + } + } + +- // Step 11: Call sandbox start. ++ // Step 7.1.5: Call sandbox start. + sandbox->Start(error); + if (error.NotEmpty()) { + ERROR("Failed to start sandbox: %s", sandboxName.c_str()); +- // If start failed, sandbox should be NotReady, we cleanup network and delete sandbox in remove +- return response_id; +- } +- +- msg.type = CRI_CONTAINER_MESSAGE_TYPE_STARTED; +- mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &msg); +- +-#ifdef ENABLE_NRI +- if (!NRIAdaptation::GetInstance()->RunPodSandbox(sandbox, nriErr)) { +- ERROR("NRI RunPodSandbox failed: %s", nriErr.GetCMessage()); +- error.Errorf("NRI RunPodSandbox failed: %s", nriErr.GetCMessage()); +- return response_id; ++ // If start failed, sandbox should be NotReady, we cleanup network, but delete sandbox in remove ++ goto cleanup_network; + } +-#endif + +- return sandbox->GetId(); ++ return; + + cleanup_network: + if (namespace_is_cni(sandbox->GetNetMode().c_str())) { +@@ -423,21 +459,115 @@ cleanup_network: + ClearCniNetwork(sandbox, clearErr); + if (clearErr.NotEmpty()) { + ERROR("Failed to clean cni network: %s", clearErr.GetCMessage()); ++ return; ++ } ++ } ++} ++ ++auto PodSandboxManagerService::RunPodSandbox(const runtime::v1::PodSandboxConfig &config, ++ const std::string &runtimeHandler, Errors &error) -> std::string ++{ ++ std::string response_id; ++ std::string sandboxName; ++ sandbox::RuntimeInfo runtimeInfo; ++ std::string networkMode; ++ std::string sandboxKey; ++ std::string jsonCheckpoint; ++ runtime::v1::PodSandboxConfig copyConfig = config; ++ std::map<std::string, std::string> stdAnnos; ++ cri_container_message_t msg = { 0 }; ++ std::shared_ptr<sandbox::Sandbox> sandbox; ++#ifdef ENABLE_NRI ++ Errors nriErr; ++#endif ++ ++ // Step 1: Parepare sandbox name, runtime and networkMode ++ PrepareSandboxData(config, runtimeHandler, sandboxName, runtimeInfo, networkMode, error); ++ if (error.NotEmpty()) { ++ return response_id; ++ } ++ ++ // Step 2: Pull the image for the sandbox. ++ // Maybe we should pull image in shim controller ? ++ // But pull image interface is only in CRI image service, and it can't be called in shim controller, ++ // so we pull image in CRI pod service. ++ const std::string &image = m_podSandboxImage; ++ if (!EnsureSandboxImageExists(image, runtimeInfo.sandboxer, error)) { ++ ERROR("Failed to pull sandbox image %s: %s", image.c_str(), error.NotEmpty() ? error.GetCMessage() : ""); ++ error.Errorf("Failed to pull sandbox image %s: %s", image.c_str(), error.NotEmpty() ? error.GetCMessage() : ""); ++ return response_id; ++ } ++ ++ // Step 3: Prepare sandbox checkpoint ++ PrepareSandboxCheckpoint(config, jsonCheckpoint, error); ++ if (error.NotEmpty()) { ++ return response_id; ++ } ++ ++ // Step 4: Update sandbox instance config ++ UpdateSandboxConfig(copyConfig, jsonCheckpoint, error); ++ if (error.NotEmpty()) { ++ return response_id; ++ } ++ ++ // Step 5: Prepare sandboxKey and mount ns namespace ++ if (namespace_is_cni(networkMode.c_str())) { ++ // cleanup sandboxKey file in DeleteSandbox ++ PrepareSandboxKey(sandboxKey, error); ++ if (error.NotEmpty()) { + return response_id; + } ++ if (prepare_network_namespace(sandboxKey.c_str(), false, 0) != 0) { ++ error.Errorf("Failed to prepare network namespace: %s", sandboxKey.c_str()); ++ ERROR("Failed to prepare network namespace: %s", sandboxKey.c_str()); ++ goto clean_ns; ++ } + } + +-cleanup_sandbox: +- sandbox::SandboxManager::GetInstance()->DeleteSandbox(sandbox->GetId(), error); ++ // Step 6: Create sandbox instance ++ sandbox = sandbox::SandboxManager::GetInstance()->CreateSandbox(sandboxName, runtimeInfo, sandboxKey, ++ networkMode, copyConfig, image, error); ++ if (error.NotEmpty()) { ++ goto clean_ns; ++ } ++ ++ CRIHelpers::ProtobufAnnoMapToStd(sandbox->GetSandboxConfig().annotations(), stdAnnos); ++ // Step 7: According to the annotation and network namespace mode, ++ // determine the order of start sandbox and setup network. ++ // tips: clean sandbox and network in sub function. ++ if (CRIHelpers::SetupNetworkFirst(stdAnnos)) { ++ // Step 7.1: Setup networking for the sandbox, and then start the sandbox container. ++ SetupNetowrkAndStartPodSandbox(sandbox, sandboxName, networkMode, error); ++ } else { ++ // Step 7.2: (Default)Start the sandbox container, and then setup networking for the sandbox. ++ // why: Some kata multi-network plane plugins (such as configuring vfio device pass-through) ++ // need to be called after the pod is already running. ++ StartPodSandboxAndSetupNetowrk(sandbox, sandboxKey, sandboxName, networkMode, error); ++ } + if (error.NotEmpty()) { +- ERROR("Failed to delete sandbox: %s", sandbox->GetId().c_str()); ++ goto clean_ns; + } ++ ++ msg.type = CRI_CONTAINER_MESSAGE_TYPE_STARTED; ++ mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &msg); ++ + #ifdef ENABLE_NRI +- if (!NRIAdaptation::GetInstance()->RemovePodSandbox(sandbox, nriErr)) { +- DEBUG("NRI RemovePodSandbox failed: %s", nriErr.GetCMessage()); ++ if (!NRIAdaptation::GetInstance()->RunPodSandbox(sandbox, nriErr)) { ++ ERROR("NRI RunPodSandbox failed: %s", nriErr.GetCMessage()); ++ error.Errorf("NRI RunPodSandbox failed: %s", nriErr.GetCMessage()); ++ return response_id; + } + #endif + ++ return sandbox->GetId(); ++clean_ns: ++ if (namespace_is_cni(networkMode.c_str())) { ++ // umount netns when prepare runp failed ++ if (remove_network_namespace(sandboxKey.c_str()) != 0) { ++ SYSERROR("Failed to umount directory %s", sandboxKey.c_str()); ++ } ++ (void)remove_network_namespace_file(sandboxKey.c_str()); ++ } + return response_id; + } + +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h +index b02216c0..01d06624 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h +@@ -86,6 +86,9 @@ private: + void UpdateSandboxConfig(runtime::v1::PodSandboxConfig &config, std::string &jsonCheckpoint, Errors &error); + void SetupSandboxFiles(const std::string &resolvPath, const runtime::v1::PodSandboxConfig &config, + Errors &error); ++ void GenerateNetworkSetting(std::string &sandboxKey, std::string &network_setting_json, Errors &error); ++ void StartPodSandboxAndSetupNetowrk(std::shared_ptr<sandbox::Sandbox> sandbox, std::string &sandboxKey, std::string &sandboxName, std::string &networkMode, Errors &error); ++ void SetupNetowrkAndStartPodSandbox(std::shared_ptr<sandbox::Sandbox> sandbox, std::string &sandboxName, std::string &networkMode, Errors &error); + void SetupSandboxNetwork(const std::shared_ptr<sandbox::Sandbox> sandbox, std::string &network_settings_json, + Errors &error); + void ClearCniNetwork(const std::shared_ptr<sandbox::Sandbox> sandbox, Errors &error); +-- +2.34.1 + |