diff options
74 files changed, 18571 insertions, 0 deletions
@@ -0,0 +1 @@ +/v2.1.4.tar.gz diff --git a/0001-sandbox-del-m_containers-and-m_containersMutex.patch b/0001-sandbox-del-m_containers-and-m_containersMutex.patch new file mode 100644 index 0000000..84c3455 --- /dev/null +++ b/0001-sandbox-del-m_containers-and-m_containersMutex.patch @@ -0,0 +1,347 @@ +From d1aa4166d8ce7f3db83ff1ffbd54b796943233b3 Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Tue, 24 Oct 2023 16:19:15 +0800 +Subject: [PATCH 01/14] sandbox:del m_containers and m_containersMutex + +--- + .../v1/v1_cri_container_manager_service.cc | 30 ----- + .../cri/v1/v1_cri_container_manager_service.h | 1 - + .../v1/v1_cri_pod_sandbox_manager_service.cc | 103 +++++++++++++++--- + .../v1/v1_cri_pod_sandbox_manager_service.h | 7 +- + src/daemon/sandbox/sandbox.cc | 31 ------ + src/daemon/sandbox/sandbox.h | 7 -- + 6 files changed, 95 insertions(+), 84 deletions(-) + +diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +index eb19cac6..70629591 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +@@ -499,7 +499,6 @@ std::string ContainerManagerService::CreateContainer(const std::string &podSandb + } + + response_id = response->id; +- sandbox->AddContainer(response_id); + + cleanup: + free_container_create_request(request); +@@ -591,37 +590,8 @@ void ContainerManagerService::StopContainer(const std::string &containerID, int6 + CRIHelpers::StopContainer(m_cb, containerID, timeout, error); + } + +-// TODO: Consider to refactor the way we handle container list in sandbox. +-// This function might be removed after that. +-void ContainerManagerService::RemoveContainerIDFromSandbox(const std::string &containerID) +-{ +- std::string realContainerID; +- std::string podSandboxID; +- Errors error; +- +- CRIHelpersV1::GetContainerSandboxID(containerID, realContainerID, podSandboxID, error); +- if (error.NotEmpty()) { +- WARN("Failed to get sandbox id for container %s: %s", containerID.c_str(), error.GetCMessage()); +- return; +- } +- +- std::shared_ptr<sandbox::Sandbox> sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(podSandboxID); +- if (sandbox == nullptr) { +- ERROR("Failed to get sandbox instance: %s for creating container", podSandboxID.c_str()); +- error.Errorf("Failed to get sandbox instance: %s for creating container", podSandboxID.c_str()); +- return; +- } +- +- sandbox->RemoveContainer(realContainerID); +-} +- + void ContainerManagerService::RemoveContainer(const std::string &containerID, Errors &error) + { +- // TODO: Refactor after adding the ability to use sandbox manager for sandboxid query +- // This will remove container id from sandbox container_list first, +- // if the following operation failed, it could cause inconsistency. +- RemoveContainerIDFromSandbox(containerID); +- + CRIHelpers::RemoveContainer(m_cb, containerID, error); + if (error.NotEmpty()) { + WARN("Failed to remove container %s", containerID.c_str()); +diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.h b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.h +index 31449170..1d210416 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.h ++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.h +@@ -97,7 +97,6 @@ private: + void MakeContainerConfig(const runtime::v1::ContainerConfig &config, container_config *cConfig, + Errors &error); + void CreateContainerLogSymlink(const std::string &containerID, Errors &error); +- void RemoveContainerIDFromSandbox(const std::string &containerID); + void ListContainersFromGRPC(const runtime::v1::ContainerFilter *filter, container_list_request **request, + Errors &error); + void ListContainersToGRPC(container_list_response *response, +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +index 901ef231..2c802900 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +@@ -452,20 +452,90 @@ auto PodSandboxManagerService::GetSandboxKey(const container_inspect *inspect_da + return std::string(inspect_data->network_settings->sandbox_key); + } + +-auto PodSandboxManagerService::StopAllContainersInSandbox(const std::vector<std::string> &containers, +- Errors &error) -> bool ++auto PodSandboxManagerService::GetContainerListResponse(const std::string &readSandboxID, ++ std::vector<std::string> &errors) -> std::unique_ptr<CStructWrapper<container_list_response>> + { ++ int ret = 0; ++ container_list_request *list_request { nullptr }; ++ container_list_response *list_response { nullptr }; ++ ++ if (m_cb == nullptr || m_cb->container.list == nullptr) { ++ ERROR("Unimplemented callback"); ++ errors.push_back("Unimplemented callback"); ++ return nullptr; ++ } ++ ++ // list all containers to stop ++ auto list_request_wrapper = makeUniquePtrCStructWrapper<container_list_request>(free_container_list_request); ++ if (list_request_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ errors.push_back("Out of memory"); ++ return nullptr; ++ } ++ list_request = list_request_wrapper->get(); ++ list_request->all = true; ++ ++ list_request->filters = (defs_filters *)util_common_calloc_s(sizeof(defs_filters)); ++ if (list_request->filters == nullptr) { ++ ERROR("Out of memory"); ++ errors.push_back("Out of memory"); ++ return nullptr; ++ } ++ ++ // Add sandbox label ++ if (CRIHelpers::FiltersAddLabel(list_request->filters, CRIHelpers::Constants::SANDBOX_ID_LABEL_KEY, ++ readSandboxID) != 0) { ++ std::string tmp_errmsg = "Failed to add label in sandbox" + readSandboxID; ++ ERROR(tmp_errmsg.c_str()); ++ errors.push_back(tmp_errmsg); ++ return nullptr; ++ } ++ ++ ret = m_cb->container.list(list_request, &list_response); ++ auto list_response_wrapper = makeUniquePtrCStructWrapper<container_list_response>(list_response, free_container_list_response); ++ if (list_response_wrapper == nullptr) { ++ ERROR("Failed to call list container callback"); ++ errors.push_back("Failed to call list container callback"); ++ return nullptr; ++ } ++ if (ret != 0) { ++ if (list_response != nullptr && list_response->errmsg != nullptr) { ++ ERROR(list_response->errmsg); ++ errors.push_back(list_response->errmsg); ++ } else { ++ ERROR("Failed to call list container callback"); ++ errors.push_back("Failed to call list container callback"); ++ } ++ return nullptr; ++ } ++ ++ return list_response_wrapper; ++} ++ ++auto PodSandboxManagerService::StopAllContainersInSandbox(const std::string &readSandboxID, ++ Errors &error) -> int ++{ ++ int ret = 0; ++ std::vector<std::string> errors; ++ auto list_response_wrapper = GetContainerListResponse(readSandboxID, errors); ++ if (list_response_wrapper == nullptr) { ++ error.SetAggregate(errors); ++ return -1; ++ } ++ auto list_response = list_response_wrapper->get(); ++ + // Stop all containers in the sandbox. +- for (const auto &con : containers) { ++ for (size_t i = 0; i < list_response->containers_len; i++) { + Errors stopError; +- CRIHelpers::StopContainerHelper(m_cb, con, 0, stopError); ++ CRIHelpers::StopContainerHelper(m_cb, list_response->containers[i]->id, 0, stopError); + if (stopError.NotEmpty() && !CRIHelpers::IsContainerNotFoundError(stopError.GetMessage())) { +- ERROR("Error stop container: %s: %s", con.c_str(), stopError.GetCMessage()); ++ ERROR("Error stop container: %s: %s", list_response->containers[i]->id, stopError.GetCMessage()); + error.SetError(stopError.GetMessage()); +- return false; ++ return -1; + } + } +- return true; ++ ++ return ret; + } + + auto PodSandboxManagerService::GetNetworkReady(const std::string &podSandboxID, Errors &error) -> bool +@@ -508,7 +578,7 @@ void PodSandboxManagerService::StopPodSandbox(const std::string &podSandboxID, E + // Stop all containers inside the sandbox. This terminates the container forcibly, + // and container may still be created, so production should not rely on this behavior. + // TODO: according to the state(stopping and removal) in sandbox to avoid future container creation. +- if (!StopAllContainersInSandbox(sandbox->GetContainers(), error)) { ++ if (StopAllContainersInSandbox(sandbox->GetId(), error) != 0) { + return; + } + +@@ -524,15 +594,22 @@ void PodSandboxManagerService::StopPodSandbox(const std::string &podSandboxID, E + sandbox->Stop(sandbox::DEFAULT_STOP_TIMEOUT, error); + } + +-void PodSandboxManagerService::RemoveAllContainersInSandbox(const std::vector<std::string> &containers, ++void PodSandboxManagerService::RemoveAllContainersInSandbox(const std::string &readSandboxID, + std::vector<std::string> &errors) + { ++ auto list_response_wrapper = GetContainerListResponse(readSandboxID, errors); ++ if (list_response_wrapper == nullptr) { ++ return; ++ } ++ ++ auto list_response = list_response_wrapper->get(); ++ + // Remove all containers in the sandbox. +- for (const auto &con : containers) { ++ for (size_t i = 0; i < list_response->containers_len; i++) { + Errors rmError; +- CRIHelpers::RemoveContainerHelper(m_cb, con, rmError); ++ CRIHelpers::RemoveContainerHelper(m_cb, list_response->containers[i]->id, rmError); + if (rmError.NotEmpty() && !CRIHelpers::IsContainerNotFoundError(rmError.GetMessage())) { +- ERROR("Error remove container: %s: %s", con.c_str(), rmError.GetCMessage()); ++ ERROR("Error remove container: %s: %s", list_response->containers[i]->id, rmError.GetCMessage()); + errors.push_back(rmError.GetMessage()); + } + } +@@ -598,7 +675,7 @@ void PodSandboxManagerService::RemovePodSandbox(const std::string &podSandboxID, + // Remove all containers inside the sandbox. + // container may still be created, so production should not rely on this behavior. + // TODO: according to the state(stopping and removal) in sandbox to avoid future container creation. +- RemoveAllContainersInSandbox(sandbox->GetContainers(), errors); ++ RemoveAllContainersInSandbox(sandbox->GetId(), errors); + if (errors.size() != 0) { + error.SetAggregate(errors); + return; +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h +index 48a7cf7f..2bd28007 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h +@@ -32,6 +32,7 @@ + #include "cgroup.h" + #include "sandbox.h" + #include "v1_cri_container_manager_service.h" ++#include "cstruct_wrapper.h" + + namespace CRIV1 { + class PodSandboxManagerService { +@@ -89,9 +90,11 @@ private: + Errors &error); + void ClearCniNetwork(const std::shared_ptr<sandbox::Sandbox> sandbox, Errors &error); + void StopContainerHelper(const std::string &containerID, Errors &error); +- auto StopAllContainersInSandbox(const std::vector<std::string> &containers, Errors &error) -> bool; ++ auto GetContainerListResponse(const std::string &readSandboxID, ++ std::vector<std::string> &errors) -> std::unique_ptr<CStructWrapper<container_list_response>>; ++ auto StopAllContainersInSandbox(const std::string &readSandboxID, Errors &error) -> int; + auto GetNetworkReady(const std::string &podSandboxID, Errors &error) -> bool; +- void RemoveAllContainersInSandbox(const std::vector<std::string> &containers, std::vector<std::string> &errors); ++ void RemoveAllContainersInSandbox(const std::string &readSandboxID, std::vector<std::string> &errors); + void ClearNetworkReady(const std::string &podSandboxID); + auto SharesHostNetwork(const container_inspect *inspect) -> runtime::v1::NamespaceMode; + auto SharesHostPid(const container_inspect *inspect) -> runtime::v1::NamespaceMode; +diff --git a/src/daemon/sandbox/sandbox.cc b/src/daemon/sandbox/sandbox.cc +index ece28f4d..c8fd30be 100644 +--- a/src/daemon/sandbox/sandbox.cc ++++ b/src/daemon/sandbox/sandbox.cc +@@ -135,12 +135,6 @@ auto Sandbox::GetRuntimeHandle() const -> const std::string & + return m_runtimeInfo.runtimeHandler; + } + +-auto Sandbox::GetContainers() -> std::vector<std::string> +-{ +- ReadGuard<RWMutex> lock(m_containersMutex); +- return m_containers; +-} +- + auto Sandbox::GetSandboxConfig() const -> const runtime::v1::PodSandboxConfig & + { + return *m_sandboxConfig; +@@ -409,27 +403,6 @@ void Sandbox::RemoveLabels(const std::string &key) + m_sandboxConfig->mutable_labels()->erase(key); + } + +-void Sandbox::AddContainer(const std::string &id) +-{ +- WriteGuard<RWMutex> lock(m_containersMutex); +- m_containers.push_back(id); +-} +- +-void Sandbox::SetConatiners(const std::vector<std::string> &cons) +-{ +- WriteGuard<RWMutex> lock(m_containersMutex); +- m_containers = cons; +-} +- +-void Sandbox::RemoveContainer(const std::string &id) +-{ +- WriteGuard<RWMutex> lock(m_containersMutex); +- auto it = std::find(m_containers.begin(), m_containers.end(), id); +- if (it != m_containers.end()) { +- m_containers.erase(it); +- } +-} +- + void Sandbox::UpdateNetworkSettings(const std::string &settingsJson, Errors &error) + { + if (settingsJson.length() == 0) { +@@ -1009,8 +982,6 @@ auto Sandbox::LoadMetadata(Errors &error) -> bool + m_networkReady = metadata->get()->network_ready; + m_taskAddress = std::string(metadata->get()->task_address); + m_netNsPath = std::string(metadata->get()->net_ns_path); +- Transform::CharArrayToStringVector((const char **)metadata->get()->containers, +- util_array_len((const char **)metadata->get()->containers), m_containers); + + ret = google::protobuf::util::JsonStringToMessage(metadata->get()->sandbox_config_json, &config).ok(); + if (!ret) { +@@ -1120,8 +1091,6 @@ void Sandbox::FillSandboxMetadata(sandbox_metadata* metadata, Errors &error) + metadata->task_address = util_strdup_s(m_taskAddress.c_str()); + metadata->net_ns_path = util_strdup_s(m_netNsPath.c_str()); + +- metadata->containers = Transform::StringVectorToCharArray(m_containers); +- + google::protobuf::util::MessageToJsonString(*m_sandboxConfig.get(), &jsonStr); + if (jsonStr.length() == 0) { + error.Errorf("Failed to get sandbox config json for sandbox: '%s'", m_id.c_str()); +diff --git a/src/daemon/sandbox/sandbox.h b/src/daemon/sandbox/sandbox.h +index 13ee4958..20a8e338 100644 +--- a/src/daemon/sandbox/sandbox.h ++++ b/src/daemon/sandbox/sandbox.h +@@ -104,7 +104,6 @@ public: + auto GetCreatedAt() -> uint64_t; + auto GetPid() -> uint32_t; + auto GetTaskAddress() const -> const std::string &; +- auto GetContainers() -> std::vector<std::string>; + auto GetImage() -> const std::string &; + void SetNetMode(const std::string &mode); + void SetController(std::shared_ptr<Controller> controller); +@@ -112,9 +111,6 @@ public: + void RemoveAnnotations(const std::string &key); + void AddLabels(const std::string &key, const std::string &value); + void RemoveLabels(const std::string &key); +- void AddContainer(const std::string &id); +- void SetConatiners(const std::vector<std::string> &cons); +- void RemoveContainer(const std::string &id); + void UpdateNetworkSettings(const std::string &settingsJson, Errors &error); + auto UpdateStatsInfo(const StatsInfo &info) -> StatsInfo; + void SetNetworkReady(bool ready); +@@ -203,9 +199,6 @@ private: + bool m_networkReady; + std::string m_networkSettings; + std::string m_image; +- // container id lists +- std::vector<std::string> m_containers; +- RWMutex m_containersMutex; + // TOOD: m_sandboxConfig is a protobuf message, it can be converted to json string directly + // if save json string directly for sandbox recover, we need to consider hot + // upgrade between different CRI versions +-- +2.42.0 + diff --git a/0002-sandbox-adapt-UT-when-del-m_containers-and-m_contain.patch b/0002-sandbox-adapt-UT-when-del-m_containers-and-m_contain.patch new file mode 100644 index 0000000..97a3302 --- /dev/null +++ b/0002-sandbox-adapt-UT-when-del-m_containers-and-m_contain.patch @@ -0,0 +1,167 @@ +From 415d7dca6175136ca4c1c780f1e512fd363d01c4 Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Fri, 3 Nov 2023 14:27:45 +0800 +Subject: [PATCH 02/14] sandbox:adapt UT when del m_containers and + m_containersMutex + +--- + .../design/detailed/Sandbox/sandbox_design_zh.md | 16 +--------------- + test/mocks/sandbox_mock.cc | 11 ----------- + test/mocks/sandbox_mock.h | 4 ---- + test/sandbox/sandbox/sandbox_ut.cc | 14 -------------- + 4 files changed, 1 insertion(+), 44 deletions(-) + +diff --git a/docs/design/detailed/Sandbox/sandbox_design_zh.md b/docs/design/detailed/Sandbox/sandbox_design_zh.md +index 86acd70b..109094cb 100644 +--- a/docs/design/detailed/Sandbox/sandbox_design_zh.md ++++ b/docs/design/detailed/Sandbox/sandbox_design_zh.md +@@ -99,7 +99,6 @@ auto GetNetworkSettings() -> const std::string &; + auto GetCreatedAt() -> uint64_t; + auto GetPid() -> uint32_t; + auto GetTaskAddress() const -> const std::string &; +-auto GetContainers() -> std::vector<std::string>; + + // 设置和更新sandbox的变量值 + void SetNetMode(const std::string &mode); +@@ -108,9 +107,6 @@ void AddAnnotations(const std::string &key, const std::string &value); + void RemoveAnnotations(const std::string &key); + void AddLabels(const std::string &key, const std::string &value); + void RemoveLabels(const std::string &key); +-void AddContainer(const std::string &id); +-void SetConatiners(const std::vector<std::string> &cons); +-void RemoveContainer(const std::string &id); + void UpdateNetworkSettings(const std::string &settingsJson, Errors &error); + auto UpdateStatsInfo(const StatsInfo &info) -> StatsInfo; + void SetNetworkReady(bool ready); +@@ -252,16 +248,12 @@ public: + auto GetCreatedAt() -> uint64_t; + auto GetPid() -> uint32_t; + auto GetTaskAddress() const -> const std::string &; +- auto GetContainers() -> std::vector<std::string>; + void SetNetMode(const std::string &mode); + void SetController(std::shared_ptr<Controller> controller); + void AddAnnotations(const std::string &key, const std::string &value); + void RemoveAnnotations(const std::string &key); + void AddLabels(const std::string &key, const std::string &value); + void RemoveLabels(const std::string &key); +- void AddContainer(const std::string &id); +- void SetConatiners(const std::vector<std::string> &cons); +- void RemoveContainer(const std::string &id); + void UpdateNetworkSettings(const std::string &settingsJson, Errors &error); + auto UpdateStatsInfo(const StatsInfo &info) -> StatsInfo; + void SetNetworkReady(bool ready); +@@ -347,9 +339,6 @@ private: + std::string m_networkMode; + bool m_networkReady; + std::string m_networkSettings; +- // container id lists +- std::vector<std::string> m_containers; +- RWMutex m_containersMutex; + // TOOD: m_sandboxConfig is a protobuf message, it can be converted to json string directly + // if save json string directly for sandbox recover, we need to consider hot + // upgrade between different CRI versions +@@ -410,9 +399,7 @@ std::string m_netNsPath; + std::string m_networkMode; + bool m_networkReady; + std::string m_networkSettings; +-// container id lists +-std::vector<std::string> m_containers; +-RWMutex m_containersMutex; ++ + // TOOD: m_sandboxConfig is a protobuf message, it can be converted to json string directly + // if save json string directly for sandbox recover, we need to consider hot + // upgrade between different CRI versions +@@ -430,7 +417,6 @@ std::set<uint32_t> m_vsockPorts; + + 1. m_mutex: 保障并发sandbox的生命周期操作（start, stop, remove） + 2. m_stateMutex:保障并发对m_state，m_statsInfo，m_networkSettings的修改与读取 +-3. m_containersMutex：保障对m_containers的并发操作 + + ## 4.2 sandbox manage 模块 + +diff --git a/test/mocks/sandbox_mock.cc b/test/mocks/sandbox_mock.cc +index ab6c2d60..e5aefdda 100644 +--- a/test/mocks/sandbox_mock.cc ++++ b/test/mocks/sandbox_mock.cc +@@ -77,14 +77,6 @@ const std::string &Sandbox::GetRuntimeHandle() const + return defaultStr; + } + +-std::vector<std::string> Sandbox::GetContainers() +-{ +- if (g_sandbox_mock != nullptr) { +- return g_sandbox_mock->GetContainers(); +- } +- return defaultVec; +-} +- + const runtime::v1::PodSandboxConfig & Sandbox::GetSandboxConfig() const + { + if (g_sandbox_mock != nullptr) { +@@ -154,9 +146,6 @@ void Sandbox::AddAnnotations(const std::string &key, const std::string &value) { + void Sandbox::RemoveAnnotations(const std::string &key) {} + void Sandbox::AddLabels(const std::string &key, const std::string &value) {} + void Sandbox::RemoveLabels(const std::string &key) {} +-void Sandbox::AddContainer(const std::string &id) {} +-void Sandbox::SetConatiners(const std::vector<std::string> &cons) {} +-void Sandbox::RemoveContainer(const std::string &id) {} + void Sandbox::UpdateNetworkSettings(const std::string &settingsJson, Errors &error) {} + void Sandbox::PrepareSandboxDirs(Errors &error) {} + void Sandbox::CleanupSandboxDirs() {} +diff --git a/test/mocks/sandbox_mock.h b/test/mocks/sandbox_mock.h +index 6b46dca6..341042e9 100644 +--- a/test/mocks/sandbox_mock.h ++++ b/test/mocks/sandbox_mock.h +@@ -31,7 +31,6 @@ public: + MOCK_METHOD0(GetName, const std::string & ()); + MOCK_METHOD0(GetSandboxer, const std::string & ()); + MOCK_METHOD0(GetRuntimeHandle, const std::string & ()); +- MOCK_METHOD0(GetContainers, std::vector<std::string>()); + MOCK_METHOD0(GetSandboxConfig, const runtime::v1::PodSandboxConfig &()); + MOCK_METHOD0(GetMutableSandboxConfig, std::shared_ptr<runtime::v1::PodSandboxConfig>()); + MOCK_METHOD0(GetRootDir, const std::string & ()); +@@ -46,9 +45,6 @@ public: + MOCK_METHOD1(RemoveAnnotations, void(const std::string &key)); + MOCK_METHOD2(AddLabels, void(const std::string &key, const std::string &value)); + MOCK_METHOD1(RemoveLabels, void(const std::string &key)); +- MOCK_METHOD1(AddContainer, void(const std::string &id)); +- MOCK_METHOD1(SetConatiners, void(const std::vector<std::string> &cons)); +- MOCK_METHOD1(RemoveContainer, void(const std::string &id)); + MOCK_METHOD2(UpdateNetworkSettings, void(const std::string &settingsJson, Errors &error)); + MOCK_METHOD1(UpdateStatsInfo, StatsInfo(const StatsInfo &info)); + MOCK_METHOD1(SetNetworkReady, void(bool ready)); +diff --git a/test/sandbox/sandbox/sandbox_ut.cc b/test/sandbox/sandbox/sandbox_ut.cc +index 494fb543..dd84d8fb 100644 +--- a/test/sandbox/sandbox/sandbox_ut.cc ++++ b/test/sandbox/sandbox/sandbox_ut.cc +@@ -49,7 +49,6 @@ TEST_F(SandboxTest, TestDefaultGetters) + ASSERT_STREQ(sandbox->GetRuntime().c_str(), info.runtime.c_str()); + ASSERT_STREQ(sandbox->GetSandboxer().c_str(), info.sandboxer.c_str()); + ASSERT_STREQ(sandbox->GetRuntimeHandle().c_str(), info.runtimeHandler.c_str()); +- ASSERT_EQ(sandbox->GetContainers().size(), 0); + ASSERT_STREQ(sandbox->GetRootDir().c_str(), sandbox_rootdir.c_str()); + ASSERT_STREQ(sandbox->GetStateDir().c_str(), sandbox_statedir.c_str()); + ASSERT_STREQ(sandbox->GetResolvPath().c_str(), (sandbox_rootdir + "/resolv.conf").c_str()); +@@ -85,19 +84,6 @@ TEST_F(SandboxTest, TestGettersAndSetters) + sandbox->RemoveLabels("key"); + EXPECT_TRUE(sandbox->GetSandboxConfig().labels().empty()); + +- std::string containerId = "container_id"; +- sandbox->AddContainer(containerId); +- auto Mycontainers = sandbox->GetContainers(); +- auto it = std::find(Mycontainers.begin(), Mycontainers.end(), containerId); +- EXPECT_NE(Mycontainers.end(), it); +- +- sandbox->RemoveContainer(containerId); +- EXPECT_EQ(sandbox->GetContainers().size(), 0); +- +- std::vector<std::string> containers = {"container1", "container2"}; +- sandbox->SetConatiners(containers); +- EXPECT_EQ(sandbox->GetContainers(), containers); +- + StatsInfo statsInfo = {1234, 100}; + sandbox->UpdateStatsInfo(statsInfo); + EXPECT_EQ(sandbox->GetStatsInfo().timestamp, statsInfo.timestamp); +-- +2.42.0 + diff --git a/0003-Add-Readonly-Masked-Path-and-RunAsGroup-support-for-.patch b/0003-Add-Readonly-Masked-Path-and-RunAsGroup-support-for-.patch new file mode 100644 index 0000000..072a5e5 --- /dev/null +++ b/0003-Add-Readonly-Masked-Path-and-RunAsGroup-support-for-.patch @@ -0,0 +1,642 @@ +From c1eb46b00ea65fc5601f0d843bc485d087f687e0 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Mon, 6 Nov 2023 17:31:58 +0800 +Subject: [PATCH 03/14] Add Readonly/Masked Path and RunAsGroup support for cri + +Signed-off-by: jikai<jikai11@huawei.com> +--- + .../common/cri/v1/v1_cri_security_context.cc | 111 ++++++++++++++++-- + .../v1/v1_cri_container_manager_service.cc | 16 ++- + .../entry/cri/v1alpha/cri_security_context.cc | 110 +++++++++++++++-- + src/daemon/modules/spec/specs.c | 74 +++++++++++- + src/daemon/modules/spec/specs_extend.c | 17 ++- + src/daemon/modules/spec/specs_security.c | 19 +-- + 6 files changed, 294 insertions(+), 53 deletions(-) + +diff --git a/src/daemon/common/cri/v1/v1_cri_security_context.cc b/src/daemon/common/cri/v1/v1_cri_security_context.cc +index f6441f42..930710e0 100644 +--- a/src/daemon/common/cri/v1/v1_cri_security_context.cc ++++ b/src/daemon/common/cri/v1/v1_cri_security_context.cc +@@ -19,15 +19,28 @@ + #include <memory> + + namespace CRISecurityV1 { +-static void ModifyContainerConfig(const runtime::v1::LinuxContainerSecurityContext &sc, container_config *config) ++static void ModifyContainerConfig(const runtime::v1::LinuxContainerSecurityContext &sc, container_config *config, Errors &error) + { ++ // none -> ""; username -> username; username, uid -> username; username, uid, gid -> username:gid; ++ // username, gid -> username:gid; uid -> uid; uid, gid -> uid:gid; gid -> error ++ std::string user; + if (sc.has_run_as_user()) { +- free(config->user); +- config->user = util_strdup_s(std::to_string(sc.run_as_user().value()).c_str()); ++ user = std::to_string(sc.run_as_user().value()); + } + if (!sc.run_as_username().empty()) { ++ user = sc.run_as_username(); ++ } ++ if (sc.has_run_as_group()) { ++ if (user.empty()) { ++ ERROR("Invalid security context: runAsGroup without runAsUser or runAsUsername"); ++ error.SetError("Invalid security context: runAsGroup without runAsUser or runAsUsername"); ++ return; ++ } ++ user += ":" + std::to_string(sc.run_as_group().value()); ++ } ++ if (!user.empty()) { + free(config->user); +- config->user = util_strdup_s(sc.run_as_username().c_str()); ++ config->user = util_strdup_s(user.c_str()); + } + } + +@@ -42,6 +55,7 @@ static void ModifyHostConfigCapabilities(const runtime::v1::LinuxContainerSecuri + if (!capAdd.empty()) { + hostConfig->cap_add = (char **)util_smart_calloc_s(sizeof(char *), capAdd.size()); + if (hostConfig->cap_add == nullptr) { ++ ERROR("Out of memory"); + error.SetError("Out of memory"); + return; + } +@@ -54,6 +68,7 @@ static void ModifyHostConfigCapabilities(const runtime::v1::LinuxContainerSecuri + if (!capDrop.empty()) { + hostConfig->cap_drop = (char **)util_smart_calloc_s(sizeof(char *), capDrop.size()); + if (hostConfig->cap_drop == nullptr) { ++ ERROR("Out of memory"); + error.SetError("Out of memory"); + return; + } +@@ -74,7 +89,8 @@ static void ModifyHostConfigNoNewPrivs(const runtime::v1::LinuxContainerSecurity + } + + if (hostConfig->security_opt_len > (SIZE_MAX / sizeof(char *)) - 1) { +- error.Errorf("Out of memory"); ++ ERROR("The size of security opts exceeds the limit"); ++ error.Errorf("The size of security opts exceeds the limit"); + return; + } + +@@ -82,6 +98,7 @@ static void ModifyHostConfigNoNewPrivs(const runtime::v1::LinuxContainerSecurity + size_t newSize = oldSize + sizeof(char *); + int ret = util_mem_realloc((void **)(&tmp_security_opt), newSize, (void *)hostConfig->security_opt, oldSize); + if (ret != 0) { ++ ERROR("Out of memory"); + error.Errorf("Out of memory"); + return; + } +@@ -98,12 +115,9 @@ static void ModifyHostConfigscSupplementalGroups(const runtime::v1::LinuxContain + + const google::protobuf::RepeatedField<google::protobuf::int64> &groups = sc.supplemental_groups(); + if (!groups.empty()) { +- if (static_cast<size_t>(groups.size()) > SIZE_MAX / sizeof(char *)) { +- error.SetError("Invalid group size"); +- return; +- } +- hostConfig->group_add = (char **)util_common_calloc_s(sizeof(char *) * groups.size()); ++ hostConfig->group_add = (char **)util_smart_calloc_s(sizeof(char *), groups.size()); + if (hostConfig->group_add == nullptr) { ++ ERROR("Out of memory"); + error.SetError("Out of memory"); + return; + } +@@ -114,6 +128,64 @@ static void ModifyHostConfigscSupplementalGroups(const runtime::v1::LinuxContain + } + } + ++static void ApplyMaskedPathsToHostConfig(const runtime::v1::LinuxContainerSecurityContext &sc, host_config *hostConfig, ++ Errors &error) ++{ ++ if (sc.masked_paths_size() <= 0) { ++ return; ++ } ++ ++ if (hostConfig->masked_paths_len > ((SIZE_MAX / sizeof(char *)) - sc.masked_paths_size())) { ++ ERROR("The size of masked paths exceeds the limit"); ++ error.Errorf("The size of masked paths exceeds the limit"); ++ return; ++ } ++ ++ char **tmp_masked_paths {nullptr}; ++ size_t oldSize = hostConfig->masked_paths_len * sizeof(char *); ++ size_t newSize = oldSize + sc.masked_paths_size() * sizeof(char *); ++ int ret = util_mem_realloc((void **)&tmp_masked_paths, newSize, (void *)hostConfig->masked_paths, oldSize); ++ if (ret != 0) { ++ ERROR("Out of memory"); ++ error.Errorf("Out of memory"); ++ return; ++ } ++ ++ hostConfig->masked_paths = tmp_masked_paths; ++ for (int i = 0; i < sc.masked_paths_size(); ++i) { ++ hostConfig->masked_paths[hostConfig->masked_paths_len++] = util_strdup_s(sc.masked_paths(i).c_str()); ++ } ++} ++ ++static void ApplyReadonlyPathsToHostConfig(const runtime::v1::LinuxContainerSecurityContext &sc, host_config *hostConfig, ++ Errors &error) ++{ ++ if (sc.readonly_paths_size() <= 0) { ++ return; ++ } ++ ++ if (hostConfig->readonly_paths_len > ((SIZE_MAX / sizeof(char *)) - sc.readonly_paths_size())) { ++ ERROR("The size of readonly paths exceeds the limit"); ++ error.Errorf("The size of readonly paths exceeds the limit"); ++ return; ++ } ++ ++ char **tmp_readonly_paths {nullptr}; ++ size_t oldSize = hostConfig->readonly_paths_len * sizeof(char *); ++ size_t newSize = oldSize + sc.readonly_paths_size() * sizeof(char *); ++ int ret = util_mem_realloc((void **)&tmp_readonly_paths, newSize, (void *)hostConfig->readonly_paths, oldSize); ++ if (ret != 0) { ++ ERROR("Out of memory"); ++ error.Errorf("Out of memory"); ++ return; ++ } ++ ++ hostConfig->readonly_paths = tmp_readonly_paths; ++ for (int i = 0; i < sc.readonly_paths_size(); ++i) { ++ hostConfig->readonly_paths[hostConfig->readonly_paths_len++] = util_strdup_s(sc.readonly_paths(i).c_str()); ++ } ++} ++ + static void ModifyHostConfig(const runtime::v1::LinuxContainerSecurityContext &sc, host_config *hostConfig, + Errors &error) + { +@@ -123,6 +195,8 @@ static void ModifyHostConfig(const runtime::v1::LinuxContainerSecurityContext &s + ModifyHostConfigCapabilities(sc, hostConfig, error); + ModifyHostConfigNoNewPrivs(sc, hostConfig, error); + ModifyHostConfigscSupplementalGroups(sc, hostConfig, error); ++ ApplyMaskedPathsToHostConfig(sc, hostConfig, error); ++ ApplyReadonlyPathsToHostConfig(sc, hostConfig, error); + } + + static void ModifyContainerNamespaceOptions(const runtime::v1::NamespaceOption &nsOpts, +@@ -196,11 +270,18 @@ void ApplySandboxSecurityContext(const runtime::v1::LinuxPodSandboxConfig &lc, c + *sc->mutable_supplemental_groups() = old.supplemental_groups(); + sc->set_readonly_rootfs(old.readonly_rootfs()); + } +- ModifyContainerConfig(*sc, config); ++ ModifyContainerConfig(*sc, config, error); ++ if (error.NotEmpty()) { ++ ERROR("Failed to modify container config for sandbox"); ++ return; ++ } ++ + ModifyHostConfig(*sc, hc, error); + if (error.NotEmpty()) { ++ ERROR("Failed to modify host config for sandbox"); + return; + } ++ + ModifySandboxNamespaceOptions(sc->namespace_options(), hc); + } + +@@ -209,9 +290,15 @@ void ApplyContainerSecurityContext(const runtime::v1::LinuxContainerConfig &lc, + { + if (lc.has_security_context()) { + const runtime::v1::LinuxContainerSecurityContext &sc = lc.security_context(); +- ModifyContainerConfig(sc, config); ++ ModifyContainerConfig(sc, config, error); ++ if (error.NotEmpty()) { ++ ERROR("Failed to modify container config for container"); ++ return; ++ } ++ + ModifyHostConfig(sc, hc, error); + if (error.NotEmpty()) { ++ ERROR("Failed to modify host config for container"); + return; + } + } +diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +index 70629591..1f20d2d2 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +@@ -128,8 +128,22 @@ void ContainerManagerService::DoUsePodLevelSELinuxConfig(const runtime::v1::Cont + return; + } + ++ const char securityOptSep = '='; ++ + const runtime::v1::LinuxSandboxSecurityContext &context = config.linux().security_context(); +- CRIHelpersV1::ApplySandboxSecurityContextToHostConfig(context, hostconfig, error); ++ std::vector<std::string> selinuxOpts = CRIHelpersV1::GetSELinuxLabelOpts(context.has_selinux_options(), ++ context.selinux_options(), securityOptSep, error); ++ if (error.NotEmpty()) { ++ ERROR("Failed to generate SELinuxLabel options for container %s", error.GetMessage().c_str()); ++ error.Errorf("Failed to generate SELinuxLabel options for container %s", error.GetMessage().c_str()); ++ return; ++ } ++ CRIHelpersV1::AddSecurityOptsToHostConfig(selinuxOpts, hostconfig, error); ++ if (error.NotEmpty()) { ++ ERROR("Failed to add securityOpts to hostconfig: %s", error.GetMessage().c_str()); ++ error.Errorf("Failed to add securityOpts to hostconfig: %s", error.GetMessage().c_str()); ++ return; ++ } + } + + auto ContainerManagerService::IsSELinuxLabelEmpty(const ::runtime::v1::SELinuxOption &selinuxOption) -> bool +diff --git a/src/daemon/entry/cri/v1alpha/cri_security_context.cc b/src/daemon/entry/cri/v1alpha/cri_security_context.cc +index 0535b438..57ec3a63 100644 +--- a/src/daemon/entry/cri/v1alpha/cri_security_context.cc ++++ b/src/daemon/entry/cri/v1alpha/cri_security_context.cc +@@ -20,15 +20,29 @@ + #include <memory> + + namespace CRISecurity { +-static void ModifyContainerConfig(const runtime::v1alpha2::LinuxContainerSecurityContext &sc, container_config *config) ++static void ModifyContainerConfig(const runtime::v1alpha2::LinuxContainerSecurityContext &sc, container_config *config, ++ Errors &error) + { ++ // none -> ""; username -> username; username, uid -> username; username, uid, gid -> username:gid; ++ // username, gid -> username:gid; uid -> uid; uid, gid -> uid:gid; gid -> error ++ std::string user; + if (sc.has_run_as_user()) { +- free(config->user); +- config->user = util_strdup_s(std::to_string(sc.run_as_user().value()).c_str()); ++ user = std::to_string(sc.run_as_user().value()); + } + if (!sc.run_as_username().empty()) { ++ user = sc.run_as_username(); ++ } ++ if (sc.has_run_as_group()) { ++ if (user.empty()) { ++ ERROR("Invalid security context: runAsGroup without runAsUser or runAsUsername"); ++ error.SetError("Invalid security context: runAsGroup without runAsUser or runAsUsername"); ++ return; ++ } ++ user += ":" + std::to_string(sc.run_as_group().value()); ++ } ++ if (!user.empty()) { + free(config->user); +- config->user = util_strdup_s(sc.run_as_username().c_str()); ++ config->user = util_strdup_s(user.c_str()); + } + } + +@@ -43,6 +57,7 @@ static void ModifyHostConfigCapabilities(const runtime::v1alpha2::LinuxContainer + if (!capAdd.empty()) { + hostConfig->cap_add = (char **)util_smart_calloc_s(sizeof(char *), capAdd.size()); + if (hostConfig->cap_add == nullptr) { ++ ERROR("Out of memory"); + error.SetError("Out of memory"); + return; + } +@@ -55,6 +70,7 @@ static void ModifyHostConfigCapabilities(const runtime::v1alpha2::LinuxContainer + if (!capDrop.empty()) { + hostConfig->cap_drop = (char **)util_smart_calloc_s(sizeof(char *), capDrop.size()); + if (hostConfig->cap_drop == nullptr) { ++ ERROR("Out of memory"); + error.SetError("Out of memory"); + return; + } +@@ -75,7 +91,8 @@ static void ModifyHostConfigNoNewPrivs(const runtime::v1alpha2::LinuxContainerSe + } + + if (hostConfig->security_opt_len > (SIZE_MAX / sizeof(char *)) - 1) { +- error.Errorf("Out of memory"); ++ ERROR("The size of security opts exceeds the limit"); ++ error.Errorf("The size of security opts exceeds the limit"); + return; + } + +@@ -83,6 +100,7 @@ static void ModifyHostConfigNoNewPrivs(const runtime::v1alpha2::LinuxContainerSe + size_t newSize = oldSize + sizeof(char *); + int ret = util_mem_realloc((void **)(&tmp_security_opt), newSize, (void *)hostConfig->security_opt, oldSize); + if (ret != 0) { ++ ERROR("Out of memory"); + error.Errorf("Out of memory"); + return; + } +@@ -99,12 +117,9 @@ static void ModifyHostConfigscSupplementalGroups(const runtime::v1alpha2::LinuxC + + const google::protobuf::RepeatedField<google::protobuf::int64> &groups = sc.supplemental_groups(); + if (!groups.empty()) { +- if (static_cast<size_t>(groups.size()) > SIZE_MAX / sizeof(char *)) { +- error.SetError("Invalid group size"); +- return; +- } +- hostConfig->group_add = (char **)util_common_calloc_s(sizeof(char *) * groups.size()); ++ hostConfig->group_add = (char **)util_smart_calloc_s(sizeof(char *), groups.size()); + if (hostConfig->group_add == nullptr) { ++ ERROR("Out of memory"); + error.SetError("Out of memory"); + return; + } +@@ -115,6 +130,64 @@ static void ModifyHostConfigscSupplementalGroups(const runtime::v1alpha2::LinuxC + } + } + ++static void ApplyMaskedPathsToHostConfig(const runtime::v1alpha2::LinuxContainerSecurityContext &sc, host_config *hostConfig, ++ Errors &error) ++{ ++ if (sc.masked_paths_size() <= 0) { ++ return; ++ } ++ ++ if (hostConfig->masked_paths_len > ((SIZE_MAX / sizeof(char *)) - sc.masked_paths_size())) { ++ ERROR("The size of masked paths exceeds the limit"); ++ error.Errorf("The size of masked paths exceeds the limit"); ++ return; ++ } ++ ++ char **tmp_masked_paths {nullptr}; ++ size_t oldSize = hostConfig->masked_paths_len * sizeof(char *); ++ size_t newSize = oldSize + sc.masked_paths_size() * sizeof(char *); ++ int ret = util_mem_realloc((void **)&tmp_masked_paths, newSize, (void *)hostConfig->masked_paths, oldSize); ++ if (ret != 0) { ++ ERROR("Out of memory"); ++ error.Errorf("Out of memory"); ++ return; ++ } ++ ++ hostConfig->masked_paths = tmp_masked_paths; ++ for (int i = 0; i < sc.masked_paths_size(); ++i) { ++ hostConfig->masked_paths[hostConfig->masked_paths_len++] = util_strdup_s(sc.masked_paths(i).c_str()); ++ } ++} ++ ++static void ApplyReadonlyPathsToHostConfig(const runtime::v1alpha2::LinuxContainerSecurityContext &sc, host_config *hostConfig, ++ Errors &error) ++{ ++ if (sc.readonly_paths_size() <= 0) { ++ return; ++ } ++ ++ if (hostConfig->readonly_paths_len > ((SIZE_MAX / sizeof(char *)) - sc.readonly_paths_size())) { ++ ERROR("The size of readonly paths exceeds the limit"); ++ error.Errorf("The size of readonly paths exceeds the limit"); ++ return; ++ } ++ ++ char **tmp_readonly_paths {nullptr}; ++ size_t oldSize = hostConfig->readonly_paths_len * sizeof(char *); ++ size_t newSize = oldSize + sc.readonly_paths_size() * sizeof(char *); ++ int ret = util_mem_realloc((void **)&tmp_readonly_paths, newSize, (void *)hostConfig->readonly_paths, oldSize); ++ if (ret != 0) { ++ ERROR("Out of memory"); ++ error.Errorf("Out of memory"); ++ return; ++ } ++ ++ hostConfig->readonly_paths = tmp_readonly_paths; ++ for (int i = 0; i < sc.readonly_paths_size(); ++i) { ++ hostConfig->readonly_paths[hostConfig->readonly_paths_len++] = util_strdup_s(sc.readonly_paths(i).c_str()); ++ } ++} ++ + static void ModifyHostConfig(const runtime::v1alpha2::LinuxContainerSecurityContext &sc, host_config *hostConfig, + Errors &error) + { +@@ -124,6 +197,8 @@ static void ModifyHostConfig(const runtime::v1alpha2::LinuxContainerSecurityCont + ModifyHostConfigCapabilities(sc, hostConfig, error); + ModifyHostConfigNoNewPrivs(sc, hostConfig, error); + ModifyHostConfigscSupplementalGroups(sc, hostConfig, error); ++ ApplyMaskedPathsToHostConfig(sc, hostConfig, error); ++ ApplyReadonlyPathsToHostConfig(sc, hostConfig, error); + } + + static void ModifyContainerNamespaceOptions(const runtime::v1alpha2::NamespaceOption &nsOpts, +@@ -179,6 +254,7 @@ void ApplySandboxSecurityContext(const runtime::v1alpha2::LinuxPodSandboxConfig + std::unique_ptr<runtime::v1alpha2::LinuxContainerSecurityContext> sc( + new (std::nothrow) runtime::v1alpha2::LinuxContainerSecurityContext); + if (sc == nullptr) { ++ ERROR("Out of memory"); + error.SetError("Out of memory"); + return; + } +@@ -197,9 +273,14 @@ void ApplySandboxSecurityContext(const runtime::v1alpha2::LinuxPodSandboxConfig + *sc->mutable_supplemental_groups() = old.supplemental_groups(); + sc->set_readonly_rootfs(old.readonly_rootfs()); + } +- ModifyContainerConfig(*sc, config); ++ ModifyContainerConfig(*sc, config, error); ++ if (error.NotEmpty()) { ++ ERROR("Failed to modify container config for sandbox"); ++ return; ++ } + ModifyHostConfig(*sc, hc, error); + if (error.NotEmpty()) { ++ ERROR("Failed to modify host config for sandbox"); + return; + } + ModifySandboxNamespaceOptions(sc->namespace_options(), hc); +@@ -210,9 +291,14 @@ void ApplyContainerSecurityContext(const runtime::v1alpha2::LinuxContainerConfig + { + if (lc.has_security_context()) { + const runtime::v1alpha2::LinuxContainerSecurityContext &sc = lc.security_context(); +- ModifyContainerConfig(sc, config); ++ ModifyContainerConfig(sc, config, error); ++ if (error.NotEmpty()) { ++ ERROR("Failed to modify container config for container"); ++ return; ++ } + ModifyHostConfig(sc, hc, error); + if (error.NotEmpty()) { ++ ERROR("Failed to modify host config for container"); + return; + } + } +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index a7751d1b..95346603 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -2133,6 +2133,58 @@ static int generate_security_opt(host_config *hc) + } + #endif + ++static int merge_paths(char ***dest_paths, size_t *dest_paths_len, char **src_paths, size_t src_paths_len) ++{ ++ if (dest_paths == NULL || dest_paths_len == NULL) { ++ ERROR("Invalid args"); ++ return -1; ++ } ++ ++ if (src_paths_len > SIZE_MAX / sizeof(char *) || ++ *dest_paths_len > ((SIZE_MAX / sizeof(char *)) - src_paths_len)) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ ++ size_t i; ++ char **tmp_paths = NULL; ++ size_t old_size = *dest_paths_len * sizeof(char *); ++ size_t new_size = old_size + src_paths_len * sizeof(char *); ++ int ret = util_mem_realloc((void **)&tmp_paths, new_size, ++ (void *)*dest_paths, old_size); ++ if (ret != 0) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ ++ *dest_paths = tmp_paths; ++ for (i = 0; i < src_paths_len; i++) { ++ (*dest_paths)[(*dest_paths_len)++] = util_strdup_s(src_paths[i]); ++ } ++ ++ return 0; ++} ++ ++static int merge_masked_paths(oci_runtime_spec *oci_spec, char **masked_paths, size_t masked_paths_len) ++{ ++ if (masked_paths == NULL || masked_paths_len == 0) { ++ return 0; ++ } ++ ++ return merge_paths(&oci_spec->linux->masked_paths, &oci_spec->linux->masked_paths_len, ++ masked_paths, masked_paths_len); ++} ++ ++static int merge_readonly_paths(oci_runtime_spec *oci_spec, char **readonly_paths, size_t readonly_paths_len) ++{ ++ if (readonly_paths == NULL || readonly_paths_len == 0) { ++ return 0; ++ } ++ ++ return merge_paths(&oci_spec->linux->readonly_paths, &oci_spec->linux->readonly_paths_len, ++ readonly_paths, readonly_paths_len); ++} ++ + static int merge_security_conf(oci_runtime_spec *oci_spec, host_config *host_spec, + container_config_v2_common_config *v2_spec) + { +@@ -2180,6 +2232,18 @@ static int merge_security_conf(oci_runtime_spec *oci_spec, host_config *host_spe + } + #endif + ++ ret = merge_masked_paths(oci_spec, host_spec->masked_paths, host_spec->masked_paths_len); ++ if (ret != 0) { ++ ERROR("Failed to merge masked paths"); ++ goto out; ++ } ++ ++ ret = merge_readonly_paths(oci_spec, host_spec->readonly_paths, host_spec->readonly_paths_len); ++ if (ret != 0) { ++ ERROR("Failed to merge readonly paths"); ++ goto out; ++ } ++ + out: + return ret; + } +@@ -2205,11 +2269,6 @@ static int merge_oci_cgroups_path(const char *id, oci_runtime_spec *oci_spec, co + return -1; + } + +- if (make_sure_oci_spec_linux(oci_spec) != 0) { +- ERROR("Failed to make oci spec linux"); +- return -1; +- } +- + free(oci_spec->linux->cgroups_path); + oci_spec->linux->cgroups_path = merge_container_cgroups_path(id, host_spec); + +@@ -2228,6 +2287,11 @@ int merge_all_specs(host_config *host_spec, const char *real_rootfs, container_c + char *userns_remap = conf_get_isulad_userns_remap(); + #endif + ++ if (make_sure_oci_spec_linux(oci_spec) != 0) { ++ ERROR("Failed to make oci spec linux"); ++ return -1; ++ } ++ + ret = merge_root(oci_spec, real_rootfs, host_spec); + if (ret != 0) { + ERROR("Failed to merge root"); +diff --git a/src/daemon/modules/spec/specs_extend.c b/src/daemon/modules/spec/specs_extend.c +index 5ede7936..199cba54 100644 +--- a/src/daemon/modules/spec/specs_extend.c ++++ b/src/daemon/modules/spec/specs_extend.c +@@ -136,28 +136,21 @@ static int make_linux_uid_gid_mappings(oci_runtime_spec *container, unsigned int + unsigned int size) + { + int ret = 0; +- +- ret = make_sure_oci_spec_linux(container); +- if (ret < 0) { +- goto out; +- } +- + if (container->linux->uid_mappings == NULL) { + ret = make_one_id_mapping(&(container->linux->uid_mappings), host_uid, size); + if (ret < 0) { +- goto out; ++ return ret; + } + container->linux->uid_mappings_len++; + } + if (container->linux->gid_mappings == NULL) { + ret = make_one_id_mapping(&(container->linux->gid_mappings), host_gid, size); + if (ret < 0) { +- goto out; ++ return ret; + } + container->linux->gid_mappings_len++; + } + +-out: + return ret; + } + +@@ -180,6 +173,12 @@ int make_userns_remap(oci_runtime_spec *container, const char *user_remap) + if (host_uid == 0 && host_gid == 0) { + return 0; + } ++ ++ if (make_sure_oci_spec_linux(container) != 0) { ++ ERROR("Failed to make oci spce linux"); ++ return -1; ++ } ++ + ret = make_linux_uid_gid_mappings(container, host_uid, host_gid, size); + if (ret) { + ERROR("Make linux uid and gid mappings failed"); +diff --git a/src/daemon/modules/spec/specs_security.c b/src/daemon/modules/spec/specs_security.c +index 08db8d0d..e78cc744 100644 +--- a/src/daemon/modules/spec/specs_security.c ++++ b/src/daemon/modules/spec/specs_security.c +@@ -879,13 +879,6 @@ int merge_caps(oci_runtime_spec *oci_spec, const char **adds, size_t adds_len, c + + static int make_sure_oci_spec_linux_sysctl(oci_runtime_spec *oci_spec) + { +- int ret = 0; +- +- ret = make_sure_oci_spec_linux(oci_spec); +- if (ret < 0) { +- return -1; +- } +- + if (oci_spec->linux->sysctl == NULL) { + oci_spec->linux->sysctl = util_common_calloc_s(sizeof(json_map_string_string)); + if (oci_spec->linux->sysctl == NULL) { +@@ -904,6 +897,11 @@ int merge_sysctls(oci_runtime_spec *oci_spec, const json_map_string_string *sysc + return 0; + } + ++ ret = make_sure_oci_spec_linux(oci_spec); ++ if (ret < 0) { ++ return -1; ++ } ++ + ret = make_sure_oci_spec_linux_sysctl(oci_spec); + if (ret < 0) { + goto out; +@@ -1004,13 +1002,6 @@ static void free_adds_cap_for_system_container(char **adds, size_t adds_len) + + static int make_sure_oci_spec_linux_seccomp(oci_runtime_spec *oci_spec) + { +- int ret = 0; +- +- ret = make_sure_oci_spec_linux(oci_spec); +- if (ret < 0) { +- return -1; +- } +- + if (oci_spec->linux->seccomp == NULL) { + oci_spec->linux->seccomp = util_common_calloc_s(sizeof(oci_runtime_config_linux_seccomp)); + if (oci_spec->linux->seccomp == NULL) { +-- +2.42.0 + diff --git a/0004-network-support-version-opt.patch b/0004-network-support-version-opt.patch new file mode 100644 index 0000000..db3a845 --- /dev/null +++ b/0004-network-support-version-opt.patch @@ -0,0 +1,482 @@ +From 7a2dd92a527c1f5ee79239d93b792dc9a9758e27 Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Tue, 7 Nov 2023 20:38:22 +0800 +Subject: [PATCH 04/14] network:support version opt + +--- + .../network/cni_operator/cni_operate.c | 16 +++ + .../network/cni_operator/cni_operate.h | 3 + + .../cni_operator/libcni/invoke/libcni_exec.c | 86 +++++++++++++- + .../cni_operator/libcni/invoke/libcni_exec.h | 2 + + .../libcni/invoke/libcni_result_parse.c | 29 +++++ + .../libcni/invoke/libcni_result_parse.h | 6 + + .../network/cni_operator/libcni/libcni_api.c | 106 ++++++++++++++++++ + .../network/cni_operator/libcni/libcni_api.h | 8 +- + .../cni_operator/libcni/libcni_result_type.c | 18 +++ + .../cni_operator/libcni/libcni_result_type.h | 12 ++ + .../modules/network/native/adaptor_native.c | 3 +- + 11 files changed, 281 insertions(+), 8 deletions(-) + +diff --git a/src/daemon/modules/network/cni_operator/cni_operate.c b/src/daemon/modules/network/cni_operator/cni_operate.c +index 62249f18..6db6db51 100644 +--- a/src/daemon/modules/network/cni_operator/cni_operate.c ++++ b/src/daemon/modules/network/cni_operator/cni_operate.c +@@ -926,6 +926,22 @@ out: + return ret; + } + ++int version_network_plane(const struct cni_network_list_conf *list, ++ struct cni_version_info_list **result_version_list) ++{ ++ if (list == NULL || list->list == NULL) { ++ ERROR("Invalid input params"); ++ return -1; ++ } ++ ++ if (cni_version_network_list(list, result_version_list) != 0) { ++ ERROR("Version CNI network failed"); ++ return -1; ++ } ++ ++ return 0; ++} ++ + int detach_loopback(const char *id, const char *netns) + { + int ret = 0; +diff --git a/src/daemon/modules/network/cni_operator/cni_operate.h b/src/daemon/modules/network/cni_operator/cni_operate.h +index 150c1154..7750ff00 100644 +--- a/src/daemon/modules/network/cni_operator/cni_operate.h ++++ b/src/daemon/modules/network/cni_operator/cni_operate.h +@@ -61,6 +61,9 @@ int detach_network_plane(const struct cni_manager *manager, const struct cni_net + int check_network_plane(const struct cni_manager *manager, const struct cni_network_list_conf *list, + struct cni_opt_result **result); + ++int version_network_plane(const struct cni_network_list_conf *list, ++ struct cni_version_info_list **result_version_list); ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.c b/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.c +index c4bc81c0..4908565e 100644 +--- a/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.c ++++ b/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.c +@@ -28,7 +28,7 @@ + #include <sys/wait.h> + + #include <isula_libutils/cni_inner_plugin_info.h> +-#include <isula_libutils/cni_version.h> ++#include <isula_libutils/cni_version_info.h> + #include <isula_libutils/log.h> + #include <isula_libutils/cni_exec_error.h> + #include <isula_libutils/auto_cleanup.h> +@@ -183,10 +183,10 @@ static char *str_cni_exec_error(const cni_exec_error *e_err) + static char *cniversion_decode(const char *jsonstr) + { + __isula_auto_free parser_error err = NULL; +- cni_version *conf = NULL; ++ cni_version_info *conf = NULL; + char *result = NULL; + +- conf = cni_version_parse_data(jsonstr, NULL, &err); ++ conf = cni_version_info_parse_data(jsonstr, NULL, &err); + if (conf == NULL) { + ERROR("decoding config \"%s\", failed: %s", jsonstr, err); + goto out; +@@ -198,7 +198,7 @@ static char *cniversion_decode(const char *jsonstr) + + result = util_strdup_s(conf->cni_version); + out: +- free_cni_version(conf); ++ free_cni_version_info(conf); + return result; + } + +@@ -466,6 +466,84 @@ out: + return ret; + } + ++static char *get_default_version_stdin(void) ++{ ++ char *stdin_str = NULL; ++ int ret; ++ ++ ret = asprintf(&stdin_str, "{\"cniVersion\":\"%s\"}", CURRENT_VERSION); ++ if (ret < 0) { ++ ERROR("parse cni version failed"); ++ } ++ return stdin_str; ++} ++ ++static int do_parse_version_info_stdout_str(int exec_ret, const cni_exec_error *e_err, ++ const char *stdout_str, cni_version_info **result_version) ++{ ++ __isula_auto_free char *err_msg = NULL; ++ struct parser_context ctx = { OPT_GEN_SIMPLIFY, 0 }; ++ __isula_auto_free parser_error perr = NULL; ++ ++ if (exec_ret != 0) { ++ err_msg = str_cni_exec_error(e_err); ++ ERROR("raw exec failed: %s", err_msg); ++ isulad_append_error_message("raw exec failed: %s. ", err_msg); ++ return -1; ++ } ++ ++ if (stdout_str == NULL || strlen(stdout_str) == 0) { ++ ERROR("Get empty version result"); ++ return -1; ++ } ++ free_cni_version_info(*result_version); ++ *result_version = cni_version_info_parse_data(stdout_str, &ctx, &perr); ++ if (*result_version == NULL) { ++ ERROR("parse cni result version failed: %s", perr); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++int get_version_info(const char *plugin_path, cni_version_info **result_version) ++{ ++ __isula_auto_free char *err_msg = NULL; ++ char **envs = NULL; ++ __isula_auto_free char *stdout_str = NULL; ++ __isula_auto_free char *stdin_str = NULL; ++ cni_exec_error *e_err = NULL; ++ int ret = 0; ++ const struct cni_args cniargs = { ++ .command = "VERSION", ++ .netns = "dummy", ++ .ifname = "dummy", ++ .path = "dummy", ++ .container_id = "dummy" ++ }; ++ ++ stdin_str = get_default_version_stdin(); ++ if (stdin_str == NULL) { ++ return -1; ++ } ++ ++ envs = as_env(&cniargs); ++ if (envs == NULL) { ++ ERROR("create env failed"); ++ return -1; ++ } ++ ++ ret = raw_exec(plugin_path, stdin_str, envs, &stdout_str, &e_err); ++ DEBUG("Raw exec \"%s\" result: %d", plugin_path, ret); ++ DEBUG("Raw exec stdout: %s", stdout_str); ++ ret = do_parse_version_info_stdout_str(ret, e_err, stdout_str, result_version); ++ ++ util_free_array(envs); ++ free_cni_exec_error(e_err); ++ return ret; ++ ++} ++ + void free_cni_args(struct cni_args *cargs) + { + size_t i = 0; +diff --git a/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.h b/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.h +index 60b1c972..48d8d8b6 100644 +--- a/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.h ++++ b/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.h +@@ -40,6 +40,8 @@ int exec_plugin_with_result(const char *plugin_path, const char *cni_net_conf_js + + int exec_plugin_without_result(const char *plugin_path, const char *cni_net_conf_json, const struct cni_args *cniargs); + ++int get_version_info(const char *plugin_path, cni_version_info **result_version); ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_result_parse.c b/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_result_parse.c +index 164b2e29..aa4f75cf 100644 +--- a/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_result_parse.c ++++ b/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_result_parse.c +@@ -741,3 +741,32 @@ struct cni_opt_result *new_result(const char *version, const char *jsonstr) + ERROR("unsupported CNI result version \"%s\"", version); + return NULL; + } ++ ++size_t get_curr_support_version_len(void) ++{ ++ return CURR_SUPPORT_VERSION_LEN; ++} ++ ++int get_support_version_pos(const char *version) ++{ ++ int i = 0; ++ if (version == NULL) { ++ return -1; ++ } ++ ++ for (i = CURR_SUPPORT_VERSION_LEN - 1; i >= 0; i--) { ++ if ((g_curr_support_versions[i] != NULL) && (strcmp(version, g_curr_support_versions[i]) == 0)) { ++ return i; ++ } ++ } ++ ++ return -1; ++} ++ ++const char *get_support_version_by_pos(size_t pos) ++{ ++ if (pos >= CURR_SUPPORT_VERSION_LEN) { ++ return NULL; ++ } ++ return g_curr_support_versions[pos]; ++} +\ No newline at end of file +diff --git a/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_result_parse.h b/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_result_parse.h +index 547bc915..438e1332 100644 +--- a/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_result_parse.h ++++ b/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_result_parse.h +@@ -37,6 +37,12 @@ cni_result_curr *cni_result_curr_to_json_result(const struct cni_opt_result *src + + struct cni_opt_result *copy_result_from_current(const cni_result_curr *curr_result); + ++size_t get_curr_support_version_len(void); ++ ++int get_support_version_pos(const char *version); ++ ++const char *get_support_version_by_pos(size_t pos); ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/modules/network/cni_operator/libcni/libcni_api.c b/src/daemon/modules/network/cni_operator/libcni/libcni_api.c +index 781759e8..7f62df78 100644 +--- a/src/daemon/modules/network/cni_operator/libcni/libcni_api.c ++++ b/src/daemon/modules/network/cni_operator/libcni/libcni_api.c +@@ -843,6 +843,112 @@ free_out: + return ret; + } + ++static int version_network(const char *plugin_name, cni_version_info **result_version) ++{ ++ int ret = 0; ++ __isula_auto_free char *plugin_path = NULL; ++ ++ if (plugin_name == NULL) { ++ ERROR("Empty plugin name"); ++ return -1; ++ } ++ ++ ret = find_plugin_in_path(plugin_name, (const char * const *)g_module_conf.bin_paths, ++ g_module_conf.bin_paths_len, &plugin_path); ++ if (ret != 0) { ++ ERROR("Failed to find plugin: \"%s\"", plugin_name); ++ isulad_append_error_message("Failed to find plugin: \"%s\". ", plugin_name); ++ return ret; ++ } ++ ++ // cni plugin calls should not take longer than 90 seconds ++ CALL_CHECK_TIMEOUT(90, ret = get_version_info(plugin_path, result_version)); ++ return ret; ++} ++ ++int cni_version_network_list(const struct cni_network_list_conf *list, ++ struct cni_version_info_list **result_version_list) ++{ ++ int ret = 0; ++ int i; ++ cni_version_info *tmp_result_version = NULL; ++ ++ if ((list == NULL) || (list->list == NULL) || (result_version_list == NULL)) { ++ ERROR("Empty arguments"); ++ return -1; ++ } ++ ++ *result_version_list = util_common_calloc_s(sizeof(struct cni_version_info_list)); ++ if (*result_version_list == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ (*result_version_list)->result_versions = util_smart_calloc_s(sizeof(cni_version_info *), list->list->plugins_len); ++ if ((*result_version_list)->result_versions == NULL) { ++ ERROR("Out of memory"); ++ ret = -1; ++ goto free_out; ++ } ++ ++ for (i = 0; i < list->list->plugins_len; i++) { ++ if (version_network(list->list->plugins[i]->type, &tmp_result_version) != 0) { ++ ret = -1; ++ ERROR("Run version plugin: %d failed", i); ++ goto free_out; ++ } ++ (*result_version_list)->result_versions[i] = tmp_result_version; ++ (*result_version_list)->result_versions_len += 1; ++ tmp_result_version = NULL; ++ } ++ ++ return ret; ++ ++free_out: ++ free_cni_version_info_list(*result_version_list); ++ *result_version_list = NULL; ++ return ret; ++} ++ ++/* get the latest CNI version supported by all plugins */ ++char *cni_get_plugins_supported_version(cni_net_conf_list *list) ++{ ++ // init to default version, if no found, just return default version ++ char *cni_version = util_strdup_s(CURRENT_VERSION); ++ int i, j, version_pos; ++ struct cni_version_info_list *result_version_list = NULL; ++ struct cni_network_list_conf network_list = { ++ .list = list, ++ }; ++ size_t curr_support_version_len = get_curr_support_version_len(); ++ __isula_auto_free size_t *plugin_version_count = util_smart_calloc_s(sizeof(size_t), curr_support_version_len); ++ if (plugin_version_count == NULL) { ++ return cni_version; ++ } ++ if (cni_version_network_list(&network_list, &result_version_list) != 0) { ++ return cni_version; ++ } ++ ++ // count plugin supported version ++ for (i = 0; i < result_version_list->result_versions_len; i++) { ++ for (j = result_version_list->result_versions[i]->supported_versions_len - 1; j >= 0 ; j--) { ++ version_pos = get_support_version_pos(result_version_list->result_versions[i]->supported_versions[j]); ++ if (version_pos < 0) { ++ break; ++ } ++ plugin_version_count[version_pos]++; ++ if (plugin_version_count[version_pos] == list->plugins_len) { ++ free(cni_version); ++ cni_version = util_strdup_s(get_support_version_by_pos(version_pos)); ++ goto free_out; ++ } ++ } ++ } ++ ++free_out: ++ free_cni_version_info_list(result_version_list); ++ return cni_version; ++} ++ + static int do_copy_plugin_args(const struct runtime_conf *rc, struct cni_args **cargs) + { + size_t i = 0; +diff --git a/src/daemon/modules/network/cni_operator/libcni/libcni_api.h b/src/daemon/modules/network/cni_operator/libcni/libcni_api.h +index 878cb1bb..f94ab3f7 100644 +--- a/src/daemon/modules/network/cni_operator/libcni/libcni_api.h ++++ b/src/daemon/modules/network/cni_operator/libcni/libcni_api.h +@@ -28,9 +28,6 @@ + extern "C" { + #endif + +-#define CURRENT_VERSION "1.0.0" +-#define SUPPORT_CACHE_AND_CHECK_VERSION "0.4.0" +- + #define SUPPORT_CAPABILITY_PORTMAPPINGS "portMappings" + #define SUPPORT_CAPABILITY_BANDWIDTH "bandwidth" + #define SUPPORT_CAPABILITY_IPRANGES "ipRanges" +@@ -87,6 +84,11 @@ int cni_del_network_list(const struct cni_network_list_conf *list, const struct + + int cni_check_network_list(const struct cni_network_list_conf *list, const struct runtime_conf *rc, + struct cni_opt_result **p_result); ++ ++int cni_version_network_list(const struct cni_network_list_conf *list, ++ struct cni_version_info_list **result_version_list); ++ ++char *cni_get_plugins_supported_version(cni_net_conf_list *list); + + void free_cni_port_mapping(struct cni_port_mapping *val); + +diff --git a/src/daemon/modules/network/cni_operator/libcni/libcni_result_type.c b/src/daemon/modules/network/cni_operator/libcni/libcni_result_type.c +index fd1091de..8a0ce1dd 100644 +--- a/src/daemon/modules/network/cni_operator/libcni/libcni_result_type.c ++++ b/src/daemon/modules/network/cni_operator/libcni/libcni_result_type.c +@@ -129,3 +129,21 @@ void free_cni_opt_result(struct cni_opt_result *val) + val->my_dns = NULL; + free(val); + } ++ ++void free_cni_version_info_list(struct cni_version_info_list *val) ++{ ++ size_t i = 0; ++ ++ if (val == NULL) { ++ return; ++ } ++ ++ for (i = 0; i < val->result_versions_len; i++) { ++ free_cni_version_info(val->result_versions[i]); ++ val->result_versions[i] = NULL; ++ } ++ free(val->result_versions); ++ val->result_versions = NULL; ++ ++ free(val); ++} +diff --git a/src/daemon/modules/network/cni_operator/libcni/libcni_result_type.h b/src/daemon/modules/network/cni_operator/libcni/libcni_result_type.h +index abbc22fe..36640e63 100644 +--- a/src/daemon/modules/network/cni_operator/libcni/libcni_result_type.h ++++ b/src/daemon/modules/network/cni_operator/libcni/libcni_result_type.h +@@ -19,10 +19,15 @@ + #include <sys/types.h> + #include <stdbool.h> + ++#include <isula_libutils/cni_version_info.h> ++ + #ifdef __cplusplus + extern "C" { + #endif + ++#define CURRENT_VERSION "1.0.0" ++#define SUPPORT_CACHE_AND_CHECK_VERSION "0.4.0" ++ + /* define types for version */ + struct cni_opt_result_interface { + char *name; +@@ -73,6 +78,11 @@ struct cni_opt_result { + struct cni_opt_result_dns *my_dns; + }; + ++struct cni_version_info_list { ++ cni_version_info **result_versions; ++ size_t result_versions_len; ++}; ++ + void free_cni_opt_result_ipconfig(struct cni_opt_result_ipconfig *ipc); + + void free_cni_opt_result_route(struct cni_opt_result_route *val); +@@ -83,6 +93,8 @@ void free_cni_opt_result_dns(struct cni_opt_result_dns *val); + + void free_cni_opt_result(struct cni_opt_result *val); + ++void free_cni_version_info_list(struct cni_version_info_list *val); ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/modules/network/native/adaptor_native.c b/src/daemon/modules/network/native/adaptor_native.c +index 4c63dec1..45288d7e 100644 +--- a/src/daemon/modules/network/native/adaptor_native.c ++++ b/src/daemon/modules/network/native/adaptor_native.c +@@ -26,6 +26,7 @@ + #include "linked_list.h" + #include "isulad_config.h" + #include <isula_libutils/log.h> ++#include <isula_libutils/auto_cleanup.h> + #include "utils_network.h" + #include "network_tools.h" + #include "cni_operate.h" +@@ -1301,7 +1302,7 @@ static cni_net_conf_list *conf_bridge(const network_create_request *request, str + list->plugins_len++; + } + +- list->cni_version = util_strdup_s(CURRENT_VERSION); ++ list->cni_version = cni_get_plugins_supported_version(list); + if (request->name != NULL) { + list->name = util_strdup_s(request->name); + } else { +-- +2.42.0 + diff --git a/0005-doc-support-version-opt.patch b/0005-doc-support-version-opt.patch new file mode 100644 index 0000000..39ebd5a --- /dev/null +++ b/0005-doc-support-version-opt.patch @@ -0,0 +1,87 @@ +From e314c2ba64b6f7a8b88566e6c01fac791c8e4186 Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Wed, 8 Nov 2023 16:03:50 +0800 +Subject: [PATCH 05/14] doc:support version opt + +--- + .../detailed/Network/cni_1.0.0_change.md | 24 ++++++++++++++++++- + .../detailed/Network/cni_operator_design.md | 8 +++++++ + .../Network/cni_operator_design_zh.md | 8 +++++++ + 3 files changed, 39 insertions(+), 1 deletion(-) + +diff --git a/docs/design/detailed/Network/cni_1.0.0_change.md b/docs/design/detailed/Network/cni_1.0.0_change.md +index 35dde2f7..a91225fb 100644 +--- a/docs/design/detailed/Network/cni_1.0.0_change.md ++++ b/docs/design/detailed/Network/cni_1.0.0_change.md +@@ -33,7 +33,29 @@ cni_net_conf_runtime_config; + ## 2. Execution Protocol + ### VERSION + +-VERSION操作用于检查插件支持的CNI规范的版本，在spec-v1.0.0中，它增加了输入参数cniVersion，iSulad未使用VERSION功能，因此不涉及。 ++VERSION操作用于检查插件支持的CNI规范的版本，在spec-v1.0.0中，它增加了输入参数cniVersion。 ++ ++整体时序： ++```mermaid ++sequenceDiagram ++ participant conf_bridge ++ participant cni_get_plugins_supported_version ++ participant cni_version_network_list ++ participant version_network ++ participant get_version_info ++ conf_bridge ->> cni_get_plugins_supported_version:post cni_net_conf_list ++ cni_get_plugins_supported_version ->> cni_version_network_list:post cni_net_conf_list ++ loop for each plugin ++ cni_version_network_list ->> version_network:post each cni_net_conf ++ version_network ->> get_version_info:post each cni_net_conf plugin path e.g. ++ get_version_info -->> version_network:get version_result ++ version_network -->> cni_version_network_list:get version_result ++ end ++ cni_version_network_list ->> cni_version_network_list:comb cni_version_info_list ++ cni_version_network_list -->> cni_get_plugins_supported_version:get cni_version_info_list ++ cni_get_plugins_supported_version ->> cni_get_plugins_supported_version:find the latest CNI version supported by all plugins ++ cni_get_plugins_supported_version -->> conf_bridge:get version ++``` + + ## 3. Execution of Network Configurations + +diff --git a/docs/design/detailed/Network/cni_operator_design.md b/docs/design/detailed/Network/cni_operator_design.md +index e77f4f94..64aaf2ed 100644 +--- a/docs/design/detailed/Network/cni_operator_design.md ++++ b/docs/design/detailed/Network/cni_operator_design.md +@@ -73,6 +73,14 @@ int detach_network_plane(const struct cni_manager *manager, const struct cni_net + * Return value: return 0 on success, non-zero on failure + */ + int check_network_plane(const struct cni_manager *manager, const struct cni_network_list_conf *list, struct cni_opt_result **result); ++ ++/* ++* Description: get the CNI version information supported by the plugins required for the single network plane of the container; ++* list: network configuration; ++* result_version_list: record the CNI version supported by the plugins; ++* Return value: return 0 on success, non-zero on failure ++*/ ++int version_network_plane(const struct cni_network_list_conf *list, struct cni_result_version_list **result_version_list); + ```` + + # 4. Detailed Design +diff --git a/docs/design/detailed/Network/cni_operator_design_zh.md b/docs/design/detailed/Network/cni_operator_design_zh.md +index ac88806e..6aa3c51a 100644 +--- a/docs/design/detailed/Network/cni_operator_design_zh.md ++++ b/docs/design/detailed/Network/cni_operator_design_zh.md +@@ -73,6 +73,14 @@ int detach_network_plane(const struct cni_manager *manager, const struct cni_net + * 返回值：成功返回0，失败返回非0 + */ + int check_network_plane(const struct cni_manager *manager, const struct cni_network_list_conf *list, struct cni_opt_result **result); ++ ++/* ++* 说明：获取容器单网络平面所需的插件支持的CNI版本信息； ++* list: 网络配置； ++* result_version_list：记录插件支持的CNI版本信息； ++* 返回值：成功返回0，失败返回非0 ++*/ ++int version_network_plane(const struct cni_network_list_conf *list, struct cni_result_version_list **result_version_list); + ``` + + # 4.详细设计 +-- +2.42.0 + diff --git a/0006-2242-disable-grpc-remote-connect-by-default.patch b/0006-2242-disable-grpc-remote-connect-by-default.patch new file mode 100644 index 0000000..8fa1bb0 --- /dev/null +++ b/0006-2242-disable-grpc-remote-connect-by-default.patch @@ -0,0 +1,26 @@ +From ee928d5af7ab7c42ee4597e1b6ae5871767c165d Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Mon, 13 Nov 2023 03:04:35 +0000 +Subject: [PATCH 06/14] !2242 disable grpc remote connect by default * disable + grpc remote connect by default + +--- + cmake/options.cmake | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/cmake/options.cmake b/cmake/options.cmake +index 8f1dfbbe..aeb24662 100644 +--- a/cmake/options.cmake ++++ b/cmake/options.cmake +@@ -106,7 +106,7 @@ if (ENABLE_SELINUX STREQUAL "ON") + message("${Green}-- Enable selinux${ColourReset}") + endif() + +-option(ENABLE_GRPC_REMOTE_CONNECT "enable gRPC remote connect" ON) ++option(ENABLE_GRPC_REMOTE_CONNECT "enable gRPC remote connect" OFF) + if (ENABLE_GRPC_REMOTE_CONNECT STREQUAL "ON") + add_definitions(-DENABLE_GRPC_REMOTE_CONNECT=1) + set(ENABLE_GRPC_REMOTE_CONNECT 1) +-- +2.42.0 + diff --git a/0007-2244-Save-task-address-of-shim-v2.patch b/0007-2244-Save-task-address-of-shim-v2.patch new file mode 100644 index 0000000..c7bf391 --- /dev/null +++ b/0007-2244-Save-task-address-of-shim-v2.patch @@ -0,0 +1,209 @@ +From 23945e20c418595a7a4037e9258f23aa7bed6b48 Mon Sep 17 00:00:00 2001 +From: jake <jikai11@huawei.com> +Date: Mon, 13 Nov 2023 08:15:12 +0000 +Subject: [PATCH 07/14] !2244 Save task address of shim v2 * Save task address + of shim v2 + +--- + .../v1/v1_cri_container_manager_service.cc | 6 ++ + .../v1alpha/cri_container_manager_service.cc | 5 ++ + src/daemon/modules/runtime/shim/shim_rt_ops.c | 86 ++++++++++++++----- + 3 files changed, 74 insertions(+), 23 deletions(-) + +diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +index 1f20d2d2..f635df2b 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +@@ -1013,6 +1013,12 @@ auto ContainerManagerService::ContainerStats(const std::string &containerID, Err + if (error.NotEmpty()) { + goto cleanup; + } ++ if (contStatsVec.size() == 0) { ++ ERROR("Failed to get container stats"); ++ error.SetError("Failed to get container stats"); ++ goto cleanup; ++ } ++ + contStats = std::move(contStatsVec[0]); + + cleanup: +diff --git a/src/daemon/entry/cri/v1alpha/cri_container_manager_service.cc b/src/daemon/entry/cri/v1alpha/cri_container_manager_service.cc +index 6f8ca114..9da25768 100644 +--- a/src/daemon/entry/cri/v1alpha/cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/v1alpha/cri_container_manager_service.cc +@@ -1019,6 +1019,11 @@ auto ContainerManagerService::ContainerStats(const std::string &containerID, Err + if (error.NotEmpty()) { + goto cleanup; + } ++ if (contStatsVec.size() == 0) { ++ ERROR("Failed to get container stats"); ++ error.SetError("Failed to get container stats"); ++ goto cleanup; ++ } + contStats = std::move(contStatsVec[0]); + + cleanup: +diff --git a/src/daemon/modules/runtime/shim/shim_rt_ops.c b/src/daemon/modules/runtime/shim/shim_rt_ops.c +index d348dfe1..550b17f3 100644 +--- a/src/daemon/modules/runtime/shim/shim_rt_ops.c ++++ b/src/daemon/modules/runtime/shim/shim_rt_ops.c +@@ -16,13 +16,17 @@ + #define _GNU_SOURCE + + #include "shim_rt_ops.h" ++ + #include <stdlib.h> + #include <unistd.h> + #include <sys/wait.h> + #include <sys/stat.h> + #include <limits.h> +-#include "isula_libutils/log.h" +-#include "isula_libutils/shim_client_process_state.h" ++ ++#include <isula_libutils/auto_cleanup.h> ++#include <isula_libutils/log.h> ++#include <isula_libutils/shim_client_process_state.h> ++ + #include "utils.h" + #include "utils_string.h" + #include "constants.h" +@@ -318,16 +322,46 @@ bool rt_shim_detect(const char *runtime) + return false; + } + ++static int save_shim_v2_address(const char *bundle, const char *addr) ++{ ++ int nret; ++ char filename[PATH_MAX] = { 0 }; ++ ++ if (bundle == NULL) { ++ ERROR("Invalid input params"); ++ return -1; ++ } ++ ++ if (addr == NULL || strlen(addr) == 0) { ++ ERROR("Invalid shim v2 addr"); ++ return -1; ++ } ++ ++ nret = snprintf(filename, sizeof(filename), "%s/%s", bundle, "address"); ++ if (nret < 0 || (size_t)nret >= sizeof(filename)) { ++ ERROR("Failed to print string"); ++ return -1; ++ } ++ ++ nret = util_atomic_write_file(filename, addr, strlen(addr), CONFIG_FILE_MODE, false); ++ if (nret != 0) { ++ ERROR("Failed to write file %s", filename); ++ return -1; ++ } ++ ++ return 0; ++} ++ + int rt_shim_create(const char *id, const char *runtime, const rt_create_params_t *params) + { + int ret = 0; + int pid = 0; + int fd = -1; + const char *task_address = NULL; +- char addr[PATH_MAX] = {0}; +- char *exit_fifo_path = NULL; +- char *state_path = NULL; +- char *log_path = NULL; ++ char response[PATH_MAX] = {0}; ++ __isula_auto_free char *exit_fifo_path = NULL; ++ __isula_auto_free char *state_path = NULL; ++ __isula_auto_free char *log_path = NULL; + + if (id == NULL || runtime == NULL || params == NULL) { + ERROR("Invalid input params"); +@@ -337,29 +371,25 @@ int rt_shim_create(const char *id, const char *runtime, const rt_create_params_t + exit_fifo_path = util_path_dir(params->exit_fifo); + if (exit_fifo_path == NULL) { + ERROR("%s: failed to get exit fifo dir from %s", id, params->exit_fifo); +- ret = -1; +- goto out; ++ return -1; + } + + state_path = util_path_dir(exit_fifo_path); + if (state_path == NULL) { + ERROR("%s:failed to get state dir from %s", id, exit_fifo_path); +- ret = -1; +- goto out; ++ return -1; + } + + log_path = util_string_append(SHIM_V2_LOG, params->bundle); + if (log_path == NULL) { + ERROR("Fail to append log path"); +- ret = -1; +- goto out; ++ return -1; + } + + fd = util_open(log_path, O_RDWR | O_CREAT | O_TRUNC, DEFAULT_SECURE_FILE_MODE); + if (fd < 0) { + ERROR("Failed to create log file for shim v2: %s", log_path); +- ret = -1; +- goto out; ++ return -1; + } + close(fd); + +@@ -367,13 +397,13 @@ int rt_shim_create(const char *id, const char *runtime, const rt_create_params_t + * If task address is not set, create a new shim-v2 and get the address. + * If task address is set, use it directly. + */ +- if (params->task_addr == NULL) { +- if (shim_bin_v2_create(runtime, id, params->bundle, NULL, addr, state_path) != 0) { ++ if (params->task_addr == NULL || strlen(params->task_addr) == 0) { ++ if (shim_bin_v2_create(runtime, id, params->bundle, NULL, response, state_path) != 0) { + ERROR("%s: failed to create v2 shim", id); +- ret = -1; +- goto out; ++ return -1; + } +- task_address = addr; ++ ++ task_address = response; + } else { + task_address = params->task_addr; + } +@@ -392,10 +422,20 @@ int rt_shim_create(const char *id, const char *runtime, const rt_create_params_t + goto out; + } + ++ if (save_shim_v2_address(params->bundle, task_address) != 0) { ++ ERROR("%s: failed to save shim v2 address", id); ++ ret = -1; ++ goto out; ++ } ++ ++ return 0; ++ + out: +- free(log_path); +- free(exit_fifo_path); +- free(state_path); ++ if (ret != 0) { ++ if (shim_v2_kill(id, NULL, SIGKILL, false) != 0) { ++ ERROR("%s: kill shim v2 failed", id); ++ } ++ } + return ret; + } + +@@ -614,7 +654,7 @@ int rt_shim_status(const char *id, const char *runtime, const rt_status_params_t + return -1; + } + +- if (params->task_address != NULL) { ++ if (params->task_address != NULL && strlen(params->task_address) != 0) { + if (strlen(params->task_address) >= PATH_MAX) { + ERROR("Invalid task address"); + return -1; +-- +2.42.0 + diff --git a/0008-2233-add-runc-append-function-design-doc.patch b/0008-2233-add-runc-append-function-design-doc.patch new file mode 100644 index 0000000..c1fc023 --- /dev/null +++ b/0008-2233-add-runc-append-function-design-doc.patch @@ -0,0 +1,110 @@ +From 426b309efdee934f61a6fb27b278711aa5419dd5 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Mon, 13 Nov 2023 08:22:46 +0000 +Subject: [PATCH 08/14] !2233 add runc append function design doc * add runc + append function design doc + +--- + .../detailed/Runtime/runc_design_append.md | 75 +++++++++++++++++++ + docs/images/runc_isula_attach_flow_chart.svg | 5 ++ + 2 files changed, 80 insertions(+) + create mode 100644 docs/design/detailed/Runtime/runc_design_append.md + create mode 100644 docs/images/runc_isula_attach_flow_chart.svg + +diff --git a/docs/design/detailed/Runtime/runc_design_append.md b/docs/design/detailed/Runtime/runc_design_append.md +new file mode 100644 +index 00000000..aa17f558 +--- /dev/null ++++ b/docs/design/detailed/Runtime/runc_design_append.md +@@ -0,0 +1,75 @@ ++| Author | zhongtao | ++| ------ | --------------------- | ++| Date | 2023-10-19 | ++| Email | zhongtao17@huawei.com | ++ ++# 方案目标 ++ ++isulad当前默认的runtime为lcr + lxc，具有高性能的特点。但是该runtime是通过oci规范转换为lxc规范实现的，而且在lxc中进行了较多适配修改。 ++ ++随着oci 规范的日益成熟，oci规范实现的runtime能满足各种场景需求。因此iSulad实现基于oci 规范的runtime对接，具有底噪更低的优势。 ++ ++需求目标分为以下5点： ++ ++1. 对于isula 命令行支持 oci runtime 容器的功能进行全量排查，识别缺失功能。 ++2. 补齐缺失功能，保证切换默认runtime为runc之后功能的完整性，并针对新增功能补充对应的单元测试与门禁测试。 ++3. 梳理isulad-shim依赖情况，解耦isulad-shim依赖，减少runc容器底噪，并且提高稳定性（静态编译isulad-shim，从而使得isulad-shim不依赖isulad版本，支持isulad热升级） ++4. 重构社区门禁CI框架，支持多 runtime 测试。 ++ ++# 总体设计 ++ ++由于isulad与runc之间的交互存在gap，且将容器创建成功之后，容器进程的生命周期与isulad进程的生命周期没有必然联系，因此isulad设计了一个isulad-shim进程，用于isulad与runc的交互并将isulad与容器实例解耦。 ++ ++isulad 与 isulad-shim的关系以及整体结构请参照：[runc_design](./runc_design_zh.md) 。 ++ ++由于isula attach 涉及到attach中新建的fifo fd与容器首进程io进行数据交换，因此需要isulad与容器首进程对应的isulad-shim进行通信，完成建立io 连接并进行io copy操作。 ++ ++isula top 命令仅需要在isulad中直接调用runc二进制。 ++# 接口描述 ++ ++## isula_rt_ops模块 ++ ++```c ++ ++int rt_isula_attach(const char *id, const char *runtime, const rt_attach_params_t *params); ++ ++int rt_isula_listpids(const char *name, const char *runtime, const rt_listpids_params_t *params, ++ rt_listpids_out_t *out); ++``` ++# 详细设计 ++ ++## attach 实现流程 ++ ++### 流程图 ++ ++ ++ ++### 详细流程 ++ ++isulad端： ++ ++1. 创建容器时（rt_isula_create），传递给isulad-shim attach socket path； ++2. 进行attach操作时（rt_isula_attach），先根据 attach socket path与isulad-shim的socket server端建立连接，获得通信的socket fd。 ++3. 将attach 的stdin stdout与stderr fifo路径写入到socket fd中。 ++4. 从socket fd中读取isulad-shim是否成功将attach的fd加入到epoll中成功连接，若成功则直接成功返回，若失败则获取attach-log.json文件中的报错信息后报错返回。 ++ ++isulad-shim端： ++ ++1. create isulad-shim进程时，若传递的process 中包含 attach socket path， 则创建一个本地unix socket文件，用于isulad与isulad-shim之间通信。本地unix socket文件打开获得attach_isulad_fd，将attach_isulad_fd加入到epoll需要监听的fd中。 ++2. 收到attach_isulad_fd的事件后，调用do_attach_socket_accept函数，accept到通信的conn_fd后，将conn_fd加入到epoll需要监听的fd中。 ++3. 在收到conn_fd事件后，调用attach_fd函数。attach_fd函数中从attach_isulad_fd中读出stdin stdout与stderr路径，之后在shim中打开fd。将stdin对应的fd加入epoll监听列表中，有事件时调用stdin_cb。若容器有输入与输出，则除了写入到初始isulad fd中之外，还需要写入到attach的fd list中。 ++ ++### 新增文件 ++1. /run/isulad/runc/{container_id}/attach_socket.sock 用于isulad与isulad-shim attach通信 ++2. /run/isulad/runc/{container_id}/attach-log.json 用于记录isulad-shim中attach的报错信息。目前所有的attach操作共用一个attach文件 ++ ++### 未来规划 ++1. runc容器的attach支持魔术符退出（CTRL + Q），退出后不影响容器首进程的运行，也不影响其他并发的attach操作。 ++2. 由于支持魔术符退出，在GRPC版本中，可设置了ISULAD_INFO_DETACH错误码，用于标识用户输入魔术符退出，而rest版本中由于实现差异，无法识别exit退出与魔术符退出的差异，因此在魔术符退出时，会存在一条INFO级别的报错信息：`INFO("Wait container stopped status timeout.");` ++ ++## top 实现流程 ++ ++isulad端： ++ ++1. 直接调用runc二进制ps容器：runtime_call_simple(), 设置选项" --format json"； ++2. 直接解析调用返回的stdout中的容器进程pid，将其写入到rt_listpids_out_t结构体中； +\ No newline at end of file +diff --git a/docs/images/runc_isula_attach_flow_chart.svg b/docs/images/runc_isula_attach_flow_chart.svg +new file mode 100644 +index 00000000..e57a4ec0 +--- /dev/null ++++ b/docs/images/runc_isula_attach_flow_chart.svg +@@ -0,0 +1,5 @@ ++<svg version="1.1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1339.8517879909941 1669.333441840278" width="1339.8517879909941" height="1669.333441840278" filter="invert(93%) hue-rotate(180deg)"> ++ <!-- svg-source:excalidraw --> ++ ++ <defs><style> @font-face {font-family: "Virgil";src: url("data:application/font-woff;charset=utf-8;base64,d09GMk9UVE8AAO9AAAkAAAABO1AAAO73AAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAADYTJHQZgAIluATYCJAOQWAQGBY9lByBbpzpxQ1QZOitEWvl1ExHg7NqqxsS6RlCwSkciyuhqMfv//89KOsZwUAeGmJXV32EeijNbRUcYuVUTmWNvG5FWRBfkAW4YWVueXU6tq9d0pZufMcACcUzc7AmXTyBwYmw0WojBScItqLq853lRPMu/i21F/1VrveHFeyj+LdHE3AfsFiWacmNotmSIhpJesVGSFIFWJDgsMxNmRIsQQzGF4rvOX33MPtY+LeOF490dgR0G/l0zmGbP8ku9beGBOf2+EcKFm8Md59ESM5GZKYtFok/F14jmMBPlbsKEWTmjIcDMVCgM0RQv1VD4IZddn+x6g2Ff91p6BriTOHYqRgBPP/Z8O3Pn/WXBXUTBKoZoSbZg6N3qwPP/937uvc+bD5huHSziTSgio5mQNpAsr235cGq9P8/r5vc1H/KatrSSHj8fGwzDhavuia24cMyBvutkjYQtKFtwTcS1wVWcODY7KMSFOM69OS/+4fm59f7vRf5FsY0aMXLQQgsGrRwggmTYYCJVEmVBm4V5Z0ThidFYPeyrtRKEoeme3dvtgT6iOeJnlC8MsAKKT0UoFD469oV+l7DD8pn6lvkVc++0kSEfkCM0Vauqh+RnHNszHtYIqREKELpb3S0eAhMvIXmB7+tkb9/wpNO+5fOjmT/aU+6YYwu6wVlib1xrbGxcwNgduqmiNNOESimjUWmogISEAOOOe086XjvNccGlpjQnb8gjOY3Gyr1sLlmavf8vKSIJwtUoBiEBfY1jpaqrq1ET/z2/7f+/4/hmD/yU4cZxDvyseBGJ7/W9rVwMSlAkJUsk6hzqQCsqJVbd9NbifX/4r6pJ4LDfsD5vl0a3Nl1TSqk09SUxpviFT9KCpj3jFmScMz4t9f8VWXkadd4bSR35QqxNH33Ai5BewBI5hQAbQsZFe2nQZ84A787uLKHXduwYMHUawjI9wuP+7R+jZdW/WpL3hdof92mkcoac8YBwj3gJB7CB3aayqzgrsSIikYrAZYK2u3l6AJfgdm+OybOvB8+JTFAi4hU026JOz7fHCxQujUCxVVutY8Wgje+dK4CA+afpvl+9Nyecky85rLBnngvwxHEpBa9ATSI+MxnxjMcbadI6LaWIPXHlOeusTrrDBkAFCAzMc6UTOFEFKTY1Nm4GuAPasdAah4+DcVmNg/pfU5Nao+e7QykouG0KYR4ewkKQVmtP1n91ike6Kvt60/WySWeBpeOKr7QCg0hoKAwgPPOvqrl+UPI78CqVirROp/RhKm2YMy0C6CJQdAHdBNBOBOmKyGuC5AZe6X1LKQUflBN+2M7DB2VHoBs/KOcRvCZQfn6i0lsdM5Ypw5otoyCn0UlftixbMiaTk2XKy5Q1Y17G2zJOB7nKUm0/PkgM7J3K/X5azIxlS8eShkmeIYrIIuvN47j+8Xk5xmbzq/fd0AYlB/aifvZu/3or31eH2eCosZNM23gKUvMyo25K7fx8x2ZRqLM4/xxLbOST4x/7aOuym35y8olHctP4b8V/TX/YCP2p/1mult99+Gj0u488j59Q7/DTaWN4EkPx7itzRBhgghZ7XPAiQJAIcTLMtFCJKku1WqHHWlvsst9xLnKDYR4xwicQZ7tG+knSE5NnLF69LLqrR0X89vtHVudoGtiw59gDkletTr9hov04IcNsWAhCFGnUMYNrWMA6cqigBQoCbHTgQRhxDOMAjuIkzuIyzuAybuEuHmMEbxC773VEWIigqlrWqT4NbHCjmtiM5nd+y1vXlq5ob9d1c/f3eC93qA/7pk6NEihFn0Rmmnk61thin7O8wilOc4krXGeGedbYJEaSKhN6GOUQD/IKz/ACr/IW7/ERn/Il3/ITf/B1//9y0DjoHH5x+M2hIRCCxhEpTKlFmUZcYipGDGMmFmBNbIBtYYfYXewxq9h9hvn6/vr3thQeZ+KUArHScmqorznNakXbKqkpSqIM+SLlVlB9GtJ+HdUZXdZZXdGQ7uuZ3uizfnlBkASHEFk57pwrbnvKU17wunOuGjNrxbYj2+1zj/u92yM+4fO+5gu+7jt+6Od+40+uMZH/QTH0f6P/F2WjGegMdD6aieagxWg5WoU2oG0oH5WhWgfrEB2GI+UoOOqOGcc5+Rat6/9k/8+mUCCeJiIC5sZapHx9QEX2IMLTPuIbQjZoyoY9XgDCc1CSUcEfzRVFJcPJ4YfarTTXnEEp9Fll4bmCyF6huXD0c2NkyBrUclqFDS0fCcldmnBxq5ZzUdjQ0hBhra3noZkdrY2QIXdtwDXbzuJpombNK3hcpdld/aaHaGW4yOQ0Uq2KuSpqqHgkIlXMSxG5hZWvipLO5Yja+dGe2Ec0Xl7hJVeLriry+vKRHMnapHSSWh3Na/r0xosnKF8vBiOnHHm02DmrmKh7JKa/6Kx4A1EGB5L5gPZja3EC36Px8gVmpEMfpVo7eKal3dCDQk3OQWrtEfywxsP7kXKdc+xllnS10CSorh6UNHRFJNThKpI3Uj12NefH5EBdrYWSRruZkfa+i7mL+aR7b+irdu+/Svdr602Xpz3AJtUl38juYpi/pk0HYWKZtRHy//WcNGHEROmgcepGC9vOyzGRY1+qfJxX8Ka4G2gxbNKvLqrqBjSlQRMYHTrDDGsYBjAs71XrIXW0LYetKxhHqGHoNTTSyi1NsvSW9pYPZAY4gkxWtrU0z9vbRMU08fOJQxc+Syy+gtk9G54gA+xj+zrZX5Y0rjZrdBmGy68X/sBnzCF5AdMtKp4mqPbyQez0lZGifaq47Tf7SIZExoiy9kkO2JjFGs8VnQMdK9nMR0/nKEUbGADCEcjocG6DUGjz+CoAhQFwBBIDEwuFxo4bN+++CUBhAAhHIDEwsVBobBxhCwAAwIcDAomBiYVCY+OISUAQBH0SBIEgCIIgCIKgcvQdCMIRSAy0MCC+M6Y5BcVI/gkYkMn8sRZzoj6jjymGesj9+I2a22/ocz8NNPA2C+fgZtOvbe7ZbLKKuW1NchzK6mJ1Pf/1IF9PGNzuWk+fWr5AJLsGbXx6WqXtz7Sw5sK/2TRigdQEnKWCSwHOJDBxwFWcy/MuPpBExwGTydS++B38ygHYhoswSGJKao0cwRU+cxa87AMKDql645JDKYO+5m4QVzcJ3y7stzuYTaZTHAbNHXHpPfKmuHugSHKiUk/ZPC/TXrC0xqt/dsGQ57sgXd3hs/6nDbLRY/DknG/yxVziZVz+ldwy0Iy6cNfv9o1UVPeO7SRI/Eej3M5tWU/0ydP7mv8W+vb7QPcr3ytNdXF9bro22Y7bj7Zf+/5e7C/73n67/xrB1Iy9URj6uDPOjzvj1fgeZsnDRCrKMR3LsR2VaAcbarhhDXcEIhLDcSROx6U4E5fjZgzHoxiJt3PJ9GZwpV151l4c+73B+fGXExniT6JM1AELXkAM6XgMP5YTZEIWvN+8x96yP86/HmkfeR30BDeCq6FruBPGwn1qRw26oBG6Ql8N9qqpBupSDaqaqtbOeqAPtE9H9a6ua4yag2bZtBiT0RqTSZtKYzeKbdmu2Fors26rtyFbtJX2mzudd3CO+7zBVdzBKraziKWsZzf7OcUPuIzfccG8beSYgWdMypiWsTBjeUZ2xraMHRl1GW0ZwgxVhjlDyrAzchn1jJlUWRX48WpiZWxuhR6qmw57kBDolGzsgvY89rhb7XvJzwQfY4+20SX4OtWreKpuSFaXlECpiOVamcv4JA4/x8nKhM2qKCxtMjn4oyulp5+2+DzPxYtYzPSoPf9Yvlvd4vcuYU3142xsRkmAfbwfK6ODt6039/4JpW8VEqlHIZIunBL+6JmzfHyE3o3mF33fTPtyd+kt8NrwT5yY3tXcECqX0RZRjsN416y3bBO8IjDJcPLMjeX3UFc4USotsEQLOQvWwDz+tuofrKKIxg39EBCg4K6oeW4pSCLqm7ih+kAy7/Fydux5Ldh5nZZU05IuqzLvripqXcA6pZ2RPKH/d7Vhy+50yl+yFh/UIyLAjn8fv/fx2Z7SNsWc2qzj/U7z+cFG30qoPh7Z5qC3Ze5YhCSBZYH04b4xhJmKKLbQSuzNojJnYgVKYcrTV4jLFCCbX7QrsD8wOFpEHcVC6m7/T4/rDbOCMYvv9JxgP8P+5bnnhxblt29k79MERhrZfpNruPWEn9zyaWm9D6Yhlysalgwi4BGLoGwXLltLytTuk/Fts6n1vh2+DbWgbnrIDLRawWQ8nikM/EuPyg9/viTjuTpg3B1Y+hDUoC9ieivGQlh0cicgzznLfcQVZ3dVKCuRSmjBAmXfyhNu9pTGGImlCL53o0K42CnEDXXqbCl182f/Bf89gv5vwu2e7vJ4iWYg3CGwkKTThtD0GoseIwAWO58TjMhWCDTwf4n/MWnupK2x/YCzIig48kILRHftWSKYgi+uRo3xYmrCurpHP+wg/v01/53uutoVJiUQVDulJgTOfnkrR0ZI1k1tENAMdR8iv1SLeO3Yf8I/i8/b9WdwvtwVMZlpeDIIzYdmpyAtzZMJSHtCcxcsWTFrteeNzqfiMmfScU8kIho1b78owsx+mq4uQutyR6w+9Yo3PmDBKfNIugWybM4TIVcd0CWURQ0Sc9kHMwiRyovyick6zb2ikU77TH+AKiu2Gf+Ahqq76ShpCk2KlUBjcPCK11f9dj+F7vSs8H0W2JSabHDxn24YNFev43yHqeq6XfE4R3rTOnZgFh8dEP9Q12qAHDk022XQ1pUSyjS/MSNipVuDYUYp0cM2GyEIDULG3HlLVE2Auyq87HOQvZquGjVgYq9KECHaIH0yk/2vr37SWfBV8B39wF8uFIxNfTsoDBvADTTNVVQ7OCrfY6L0G9yaU8aGZUKY7KcewrBtVffi6ub8GXxbdpI97b8hBoFQUgYJ00i0awM2XFgyVBGcFpXLAuR+2yS9JkI/gib26ZWf37eeNzxCOI7hrUwsKtpDVjRC3jQBB+N9CokJWXc0xw4tO43puOVMxB3S5XQnpNSwKH7otNIEjQiM4uWWjq3nF5hOi5K7xSdq3G9gYjUCzpPda9tRcOW32Y1gO7B4VKSrcYl2suSDeHI+STDgD5mdKorFl+SBLm5wuaWCyjmlZRMIsqBi5kErzkGOJ4nBBsasiHSBSPGEh2GWK1XRkbr0idAO/blDD3UvNXmqJJmxpciGbqkJFkRNIUlDf+tr0yuVBmbNePeTRpvLyLH+s3osJPvlgTLqh3bYTHbiV3M6AGRNwjDRIuSterpZLzTWgcuy62rHrJUFzGaxIq+ArjzfVthhOorC0AiCppSU1uIeWScjvuX2saqaGLe8Fh3SttzX2jZ4L/t29aumKWwl4SBuLsVe6wvvnzl7f2Fj60x2nQvYoDXgEEtrWEGMMhekBIg0R6qwt5tu7aJsxFY80wzcmiOQL9SbhS37pegG7yUM0NERxJnuG9BGqdPXDMtrch5+Nbp6qOhNVG7d8BXcqZcV5coyFjc48CSzVlnjgNh2I9tCF1i0cmNaz0tWcdOWmnZQokq6pKW7cjrIj6FvBr7l+gH1AYTeF4etWFaoxef12lP4cFlvw7p8amqeUgDZqPYsC1MlUuNK8Mrdazc29gBZsdmRjPBrIuC6r/YdOIIzHVFYFJYrS6tbwdJCTvZNziE5kR7IqdUadOyGS3ESUHqK7/9NM3cbbyjUVkSsAOtQcaUIuezfbaVYT2V/XL5KOV9TQ9fG3WezS8MLr2cJBkVOfn9tH754Wao4QR+eb7wYvd2+107VAeA12lEZIBOssvLIFFCpkDpjRTY7Ovv0+7XIpQne3FTbfpFkF4sx2A8e2b/y6YM3OW4SCexYtGl0iDncNe3uxpa942A1SXQP5Cf61mjfYINvow1RZVm/4BbL9Xyj+fTag1N1gDdXtRHSP7w5j1qrmZ5xB3ApbsCAIKQ4zRDLkDfbE3FVSPJxnsxGk+P9wytWacTijZsSdBKq2ytb/prJ+O2mCchsWw3g0mDn9m7b4DN2d5seOaA066UNdPbUzuH6nQfXAunVgqo2ekuBJiumUfT9cxcjX6hdfdEzvjRfml7aBmEgqFVDoUAjb8kGGy6WB4uKKOG8vup5eFWweTsT9ItV3gQCp1IF8dX6Ok5ox9gPGraY2+n1iq7ZkiVZ5Dm9YDs7FyRqz+xpVVUolMUgZ/h2DpOmplbsrFN3TFtxNEArmqxBPWcSCxlJGisugPhwyHi/GDsfHF06QQKjxAP8pKgompEleE9YZHz32WddH0hyGDqskcSu9dgEuaqpMCjrBRHiIZeQsMiKnimZr0syQ+0SvxE9AFhxhQTs/AE/sanpzViT/b1wMD7ZvX1rBxCO8WcNLUCqHPuaarv1pOIBIHGalH2hk4EPA4jG3RO5MbMxXOhX7kx99f2ds1Vs7czum7f7DmvB5YZGY71ArWtIxnOaBSRdxAprUkE9EkEqH4Y6kuSWwazEblObCGPKRPDEqNqV1ZqTdw0xlgornRgwgOezRgQpEMGwA4+rQo3NbYvbG1gBuLaPrA9GzVMv9HbRsnav4EhDT3mZOlKNK+/e+Fx1s1pz/M9+UXf4Ja1DUcdd6aYJ8L++VzqQssWypCkghOsGqraGxEuQ2mgxXrGzpnFHzPRhhCT4J3DIYv5KjPnM8J3hYgyJRUFRERCiihSmv6uN1bjpzcsbV/NL2xfqs00vXq6mMiCiD4gbnSqNoxUtBqPV1/g+LIxqtIXWiofbm1thsPl6Tpp+Np3aKwM27ZYdKGpYoyiRVXwlWj4WDsx7Sn4iIWHC0NlwyGTXXr6wn4uzjyhq6JJ29+I1ABL4ss3q49ZkuZsSPlHwQmoZbRecMh/egNGDZJQi3KCtimV30adH2/PUB1KNGxptAPUUU7ajIqzWOquk8wrlY3tfbHumeo8VEOyryY9PbkV+J1BKHY2KWtbXAMeel6CQTZtybf3V627dbiPd71q46Ddo1HO0RRCRHyo+zBQ6YTv23devSKVABnQx4MJ/5Tlb/kCNO2giJf1rDycrANLWTQHC9x/Z272Dh6XbUfmlokgarA96u5NbV5DMzZEIEqnHX51L1IoVK7ndbzlCfP0n1UX1AtgGWfbL81d6m/JpVqBUQQS48gdViQ3PSvZD+VA4KO8s3VKOzdv7/ejA1bbLR/zvRoPib+DjXfcMJQd2I+MnyDoe/DePZDoN1QCCWscaqmlTQKd1AtMAB7bbbXfVsTEmNmOQ5EIvYqiRay296XeXArEJZJYnNeiPqoaL4seQk549GuJ/GTLKLduXFx+/SCHfYgMfIXRFrLHzo25iixS8ZaBbAuc7Cett9j01p1gzBodStOCUNZCCwIubdyN/utMxWO14HRr2m54XGI7nOpR6FsVAzvk0gLiVrFY4QZKKQSnXCC1RAq4xJbwHvP8Nyy0kkV7GC9pOVkK5M6vCwqa0z74TqahlwTSNmZKWbEHNZ6K6hPrG5FaXGRswXApnlnYWSWn6diVOZ79nOvEtn+Ox0RQ6xrJqA1cxLbjQJHiEZXWqYnb54dX2zEoxAQcYyxtX5JSdZTobF3KdImDFfgmUvYKII2bqGld+gzJ1Sapw1KJKJmbE8qJDxZTF08osFb/EMhMWs+mPDlXMCwmssLjEqUIZM9Ut8a2YS5QME1arzFoJ5lck07VmWSrXpid4zLMi0ww0pN7udPnlN1/hvUYAl62L039i1+t4moKRhtpjPKdhhJMXOPL0MpLpz0T2r4vrH581ODlaJd/z5uxALVOmys4gc4g4tnkm0yt9Yk0AMSu0KvP62glaZpuc/2JzG9suZErr5XUgDRhB1PifHrq0pvMFWDgK+c4wj14B5Om3O0g/F3lco3RR/aQgWEeRUE/slB77cjCLjTzqvCmGSrPM1EwiJDTNyALmpLmIgGrQ2pXdTLbk7vLBQQvjmUcktrqu72KCXYVU0+t1lYKg/gArCHwpVrmaQL0yckjHi1APcQhyykKTXX9tS/qqYwCZr16RoNHrtq9ceWLl/S0rn7jmlkW/waImOdjpXOn4bKbQpUscuG9fGb+xzmZMxYR6klAHLRZ/qJA2mjDlf+3hX0OMsUSsPQVhOPHsYmjyqGpQD4YJCCSMPCQv2q5IGDmI31OLvrQ01dvWVIanBA+Njw15M/XUSGnW/6I+tzKLo9Fj1+xXGPPUprENKxNidYwuiSiZbFutTvZlMVs5l9sqMQaR8LGJZeJUV3OWPWBkUfftVXC87aB2G39YDFYflLVYi5pIQDDNFBxKBlE1ZOZrWzjmgpJnu0o0AaR6neXUqQkJ19loSiYKkfHMwYnKfhMEUXOx02VkOGu+xrm6x6OvNc+9DajfWqqbU5OzpSUNwPN1zKNjWoHzF6kuw+EcRTkak/k0VudILpcSJXaPdFg+zHgJuWznOoTERhyYB5y6SCPWjrpkd39NaLQajfWd/uHWHoASlSRGba2322QquVondGQV20ZoNwLfO7l751szicl4a26KyEdv7bSPATtgJwUb6GRJtSGKRl4JhcO8qCVFZmPBKUkck2FO76rN17PktdM99RuLxKbn5gs3J3FHSuk4aqAk1ZACcNFGkJFaYxvBhvVyQ90LPqguF7fHJZ1+zh76WuBJPmLUuJoaNjBbPhdW4Abx+f51GmgAZIOYVBBv8i7WgQIt4w0bU4yN8UMzmMxcimVAllVR70rk+mJb1x1wRcsWr6SuCegtXmcFbZXI4pazvJwVljqqZGiqagmkGOQkoc5QGbD5S1rE/m9oYZeVhKXYlyqQFwoul/d5Nrp6SiKVmlThv1KiIog52ffqXvnT3xxQlwGT9OGGu9bWtn0+Dyqyyk9G51mb7sw/akWV73//dJNdcvmSQDNd/m6qUZ9XHRhWK0kRVetydUlKiYgFYr/8kEnFR2xXNiI/QK9CACmZ/+YXnf3LiOjMsS8ZnEgG63G1TZsNvzeGc4oeqmGeBLxogCYm2XWeAfNvJLvNO5S/onTtM7bnt9NTY6Rds26C4bbw/urcKix4S9t5hWLznuKpgW+FhAY3TMCEqVDWsJqeaX7XIyBw19HqWn68/u5qx/PVytZMTO8CGZaKfXMnuNX8WinQ8dW260lAKBdV2DWmsePLDauxHGxbE4d0lFBYwuzTG/gjJBPBQ0io/Z12b2eodtoTARgDpdIxSo5+/4l7YUDGSCgX6ha4JVZg4upSBZBjXQOimGEU1DcuUqbPWXsd/MczAA48M4B+0gpp9TEqiOV2NFhVIct11HPA3pfGoCQu+SIU8A0mREVOuSKXZNJys9d6O5kBWbq07svP1xb5aq4qn/nEGlSDh+O2g28bS6CG2abuNYKw7PjKDbQPhnlDt3AYZMLk87dDwAMyto8/2B+x8bvNlNB3R4DT6MZHUtVtlAr3xh0mpQ7aKnJAJSshCpAgdz2uiiAG/rD99o/XUODaTWr5BKoOdiMZ1Y0vug2o1ShyFVF2MZQNMMf84E66z0huVdwVpZcJeHZLG3DV0RwtJ1H6bzdWnZ87BkUEktYTxVcdByuZp1u352nwaA64cpHGX1aA2Pm3jjxjy9gGCbqilSAW9cM8wslgTjApO7u5Dfenl6eYOWrGoMI2wYTCoLyj7iub/VcZdDk6C4sywsci7sbhk8hs18B+32UvzeL7EfPPThkDuoYxqaeNzWhT9pdGnXHObIsHEEhWEMWpZYT7RoYhiGVN4sKs3B466Q88G8K/PDkP9IkN7Iz1rzCAmf77F10WCEy+W1mHfweg8htltqr2EqKSSNuco1kxc6ksA19TpZGFKQrTCvpJwff+YqgZbvttM8tTT5c0rLFVibKgWXl9dQLDI5NSoWLG85xuYziabFhAzljEhIlcrewjQnPG2JIPdCPjGA09JQRnNgLtVExzqGGWC3q32rimghfAFmMmuxwCFtd6EhrP7n49Pikn+aZp86ouacvA1mn3IePGfB5gDn3vkOwjCdNwINloGRy2xsRY2mQHAoyRCV2DVxhW5eUmnJwyej6yx6xUC4AFeVUyJfzK1gSvclTCoA8QxHkuhm18w3WELQ26hn0kcLfuCMexDQkAqbLKQ0aa4sHhljGG4RerELEjFJ+H9wGSr0eBevkxnn9Yj+AAEBlFqaAH6TGe5BUJUrcxwkWEAT4uJRw6HLZjXYjFmShkWEliKEhru1wMFQCNOrbRFVcPSBt1Aqe/GuP0BfzzGz9cW3I+s3JH6d7zPNHtRqA9bJS4IkTpohqK7cI6qzTpkji3uiYyGhZTZXFVE3ukSre9cPlO7S0ip6Bh7JQaNYUOwXblfCEELP1CJYAPUlOVjV22Q3ttPSpzFb4McMBEtXLzVyjNsgKRczcssaTDEjBDutnTgaqiTweDrT8SZi+58DlOCJA4XNTg6pG3U/rroamj7WTT52noPBolRUWR0KhujBe20ZBi/F4jVCOhWJwwKeXeqiF7lu/lHJE1eS3Fgno3UW1k+1QXR85btD3d3fid4lJjHUo3/rYpFQj5gWawRvTge6osbDe2V7rAwld3NSiXRX4eSTWdOV7Ob2nz9QKO9coZJknLzPQ6e/Jt+VZ4Mt1paA0aQ+thM1FdlCP1gXeqa7C620j7KLPb3JG2iqcmK8GUhSfWUbuXEtFXMZXtSe1Z4xlrg/nFGcjYOAWV63RehdtXnTvwHDIGuUElACLra3wCP9kk8z6Kp+485PnsZb+/93UWFVKh32aqSlfTEFNvkIBF4hRgVPUdi4tcKSxQDROKmCafwvL+3tp9tB7gt9AuTstW7Sf33LtyyzO4hcy2GMQ4HlBH3PWx/GOYBBg6pAoCzzBh+wGv0xsrLcud0OmyRYgILA719aT4ChbQnzhreVORUK1hc2QgKVp5pL8dMgHUrY2l5sZyFTOYl0Ie3AKZEO/arV6ApISCJVZEp2ShrijMQSgQTlGz9Ugrm2pkBKqe0ibpmTOQh/rAHv43T2BXA3BtpQT20woXrkiKQLjapGZ7h4vfq1XxZVLDHWtO2+bnFdWc4i8PrmBwRK1beJqSkhdouYDTZVGSrLkdCcwjdqTN9Ne9LNbR39Pzd4rTalZ93HbVhLdXXCCRZVqCgaAr8Gj87tPL7j3n7G9njhWADt4REmjnqRoioDdkDTLdubxDL66vjczXBWQu5CDUuenOC8ThncfkIbtmnAze4Cknur452dvk7onBc28W1MBtfGGvALu64uQCktJsnRNQHztGC4oC8VS1egptGIEFYKWtUI0AhPxo1jBcxcFaEp+t5ZUtEbD9N1IWtncO9q4hTrnohxYZcT1vNB3L5gxTN9tEM7meAeUFsOfSEBcB1tYFzCW0oOLCSvJiuKrd7NS6xIm6O/wtfvYGmRkRmJ6hTbcPrV96dZKYc0uxRmqTod/YN1QnUYBQWZV30MiB71ndge0w0Xx02zV7k1tlVu8PnhVE2BQEaXppK1/ZeMwaq2H9yW2jO5kVAjeIOQT+/dMN+LKlBy5ITd6ND6GVkyUH1cn9l0p1t/C0VvcJqUhZXU5qo/0JIKKDVEucSEAUUmwEouWEXvoJNWkLHblbNbhmvkXKnKyVmaxl+irxlHkwL/5Ra5I9SgxJVTBQwMcrFwu1tfhmdkFyAQSgwNckiY0Rl64mycf2XBdaxDnwBMstw5oUfzL21uQzxDz3RzFe8C1Mzaaqzi4Zh7UjbSzv1cbSQZQhONLcDC/ev/rS5iHAw5nPly+VltPTfd2eU7+qmzN5wlkJo6boBQ0yCh8oXmKKtEACgJtluJp6ALTs2NRoia7QexrLVMRfjfgkEFUqJmHYye/+FBiF7+HvZiIxE+IYVmdEJUI2HdQDDFyz1g8Z3yjfb5NpU4wr89e8J0aYqZCzVD1CY204dedr+ZHwA3nr3NjNrgs3e42TyL+kKQnl7yaf1wD+jxs3TCvVV1rHrbdjPpCNr9wSoULmayuo1SpVxEr1WkYulKtpEAt0bw7Hu4mSZkGtVhIqiLGUikkCcBAAcijrypeONwGX9CUfWlmqmUhN1zXFvviwT8YbCvtLxD3UHczdsNP5nHTQVGI/JR2+t0U7JK0P5LTmTAONui5V9bSYqPqsW6fzUlGcxfUq4IQyFqaxDSNnZi9TJQNrFMQgLI3JJoR78LIpAFfQe8H8NgFSaHZPhjiWRkW09lBpfpXIt4Gx/BH8ouFFgpmqEdCqrzRly+h5gK5+fVQAX6JqVtWopemdzmFeg0jjeBvZice2UQGpMHlDy2mBM91p0soCroBx1/GyD4tOEsKoFHT7ndb0n3iHBclEgKorBFnGKUFWMEzGd9xpIFOV4sTu+q6vH108hVZa1xwrwxOfZBVqPOVBm28UF7/OGhEQq7OKAZhmjNNcE+hyQTGEZTdRdZT1hul0ZzWEX3bNlsk7GdfaTeiWevtnKtqfMOVk1tegShNkeVSCRADUxkJMZlILhLjULCWm7Xl5ycQdqVJylBWKGpQRQvWGHBQH0bA2BLh3lzx1iAGLGksqHCV0bDyP0tzXVOUs3izGtqJIkq1NU70al0hAq+uiATV/cRFplJfYqQkspiM7UPViNi3jWxLH3z/JBY9ZuzfnIJFlu4uzTHYuY9ulEtHp5wS9QlvNZykyhU3FUrzVSjCNuRKm4rDMre2zU93UGNwL6RmwiyHvdrRl+bFONeLAdUsNvRUnTtXp/6mdEZrhQk8ypNZtbJYfS+mJ7TmOLQTYE43poPYIKwi//Qino7vU69FW3xQKX6mieISp2kuwQ8hwKOuRoRcfYcbSFzpdnVdPiNe4JHOgdMFTsco7JUmeJr9GX4pwBzsT+VMH3cF7JZoApHN8GUTX8mlDdsXuUj/tTm9ctLy9XEoYrUnanfZ1OvSVh2GrhngG7h2b22A8MMjgOB7bvhyRiFOuALGSfH9Fd1+DySdEwIph4gg9j4yBh/gIzpdNp8S/QXy07fKAqQpu/dOuGm1hZdocSc5cThCbiapoAv6jjzFDot/8ocOukugMCyVJPDrJtAopaTE4eaC+igv68rTrlyDpI/UFnyK0oBbUpttnn9b/mMxvM5kUNSWAmYENQVUB0A5kArSLFgF+v1Iq0AuUARuBOiAbdB1YAxQDfUAnsBrYACwGHQCzgAqgA2gE5gHdwCpgAFgIbAU2BWwHtgDbgG+CHWCQ2XlgP7PnwEpwABxiDgYcZv4D1oOl4ARzNHASdDHtwHnmjOAiOAcuME3AJXCNuRy4FX49sA4YAsPMbcFd5g6gB7SCR8z9wGPQr/0E6Mbv1Ay6+wnqFd4CevABGsLbQG+/gCPm7wEd/gCngH71h9OAp0D/3n8oAqoVXhJQDlSZXAJUgnerTQdqAF+ABMxVeBZQCBSAOcw7wFv35gfkBN4ErADfPLAWfOcbRAcgUAhoMFjobQdeXB7mKJfzlL+39BHAl+uXXvl/so5N/2RJKH9Sj7U5/fmhxseQzubqP3GyV3+yNnYcHb818XdREE6URRU0WPiFd/AeFsudUi8d0iU9clrOeDmvwh/o5/j3I+uDlACCuSAn/MtJnGbQK5KQllx0mWyGALVe/0NHdV5P6S3mr2ab+WTKzRv7k31i7dbLI3zgMT0qi2XGJmM/4qcTH02OJJk8mxxv7pOxZi5nVO+uXE++5Fv+Yx5b48U9xfcOczlXcPvO4MKlT5b+tQ+tuuv31P/sfKK5o+ltzbT/2rZ2/tWxdnO9W72S/o/9N5O/Tm6fLE0+TSV+8sNbXT16oe9i+pq/1lSu2V0rvLzm8quXT1xJy36Q8+DUZC7kKtf+3wTt3+9u8Jn+3Y2/nf3vQ3mz6dxffR1zIOSnYSML3sXCxcriT27//tLS8v/Ie5LPW21f/bWCduFHC+8WLhedXv89qlOx70Z3YzzaWNKxZPPmNyX/714rdS/tVPpqx8mMNptWbsr+FKedkpVby/m5k3l2ZbvKPpXh+ZdVn6iaVRAKSSFYeJ1wi/OFmlY1FyqFlX+te6L6/1M718+snaSNq7fqUuOPmz7YdKn1VvOXrFNtmL0QE7C72H5sPKdJ6yLiibarbQvtR9v7tP9DZkmW9Mkf8ydSN6ljtID+GS7VVeYxeC15vXhDuC2O5Eyuw33H/wX/C4KHBFHwBYfwSsgV6/t90ZD8jPxnknsURvk36eelRWpDlbVPyWq0P5V/Ux7X/2ZRT0WvsW7cNd5R/F8JytnK5UqbmZhr1suWaF232tahdU9VUq2orhW9J//ZTd2+2cZKRJ1qsiurIt89p76Ke843onpSvvxW/hFv4V/zNaJGxERCpIkMkSV5sn04J5nLOXN/cN58M77FhFwvs/VRpThMfU28hHnkBZWKr54RT6h3tX8vLZNeN/8snzE+biwZP2h+2XzTes76Bf1LesLJGDONa+af3H+2Br2u98D+tX8nmA+3w990RuJ/m5qXWpDKia/Hf5w+mv6lx8n9X1ea6unfujvZd2XLLaXdbnfb/fawPW5POjc6XEfoSJ2d7i91Z3vfVXq6PFeBDctGh6O36k+vbU6+0Jzf3LP+S+uP1rsba6eg81dTxVsT7i+5X7j/5+nxaDxaD7WzsPslb937ZW/K96Ozl2aUfmHgSOB/ULBUH7pywo/8Yu2X80+iumggmr9Tv2uIfyauENNH7xEqkRJbxJ+RWEG+YFCwST5PJol/UVgvfEg8SFSTv02+k/rctavUakpIM9fv03/Oo6irWTH3H4s+5HZz+7n7+W/dXimyd36VXOX5yn+8/302NfeT6t99+Hn+9/N7Cx97Ip8WycyaeM32n5fm6OW6L8pJ77/t1m/ojXI4q7U/EMXqXibzWQOct33KO7p78qQWcK2sTGv+kkDXjNWp2M49I1Q6L/q2WHP2BWDFnVwzBqFVqTVW48phjWgW1QgAq2uyBhveX4cDCDiwzY8u4tMErObflbyTvAPyR09/8Vrtj228o58s7AB/nZZNdnyWpFfGOpHuAfL1l505dxgxrAWyDzLMOTeGYxpvdRXF6aVDVqrJGoBWyAWyvl7CNcAdbjReNhsNNjaI79ewimqdXvT1XIQQYG6A9LGwTo9MAmeLkxvgZkmGleW0z8L4QB4biCACM6Q6Nwwi7OqW2+6F7+lOABJYISyTmeKVOsaMDKH829PqIqiLLw7X2FLIcZsqmBjpU8aYsnaVU9rmU2b++BFIPTHEGGm6HegMNZ4tcybgY4uhCgmKaDXkzOC76dpWsK5sS4DmV3nIm/OcndbuBmHXfJJvsTD+sdRuKFZ2T222Pgk0olNlHwdc6GaACrecfCB0VwvmHD5TsaSkXlNAr3mngRIfgBxbDY9VVFEFpwHK7VYqVwELH0G33Wss5cPV1bCslXILKfg2kHP+5RrUnzUSh50HK+9mhw+RztN6hTnR/xx32e/84W71GRAWuNBih/zhvlsGCENv0CWHUFbKHqtMJRP0DHYmvU23MNgfPIPzSIB66UhwB4TY6wYiwMKof9r1UcqjlqO+UvNJebyXL9okq0QcOxkmOrHV0NzS7zgb3ckuv976+mx7Y3LTnPnCfrNBJCWLbgHtXWurUmpJdvGoMwoqk+sm+BEgOUqR8UW62ZzHXHreFSGOx14FDUX7mkCJV7Nn0dVPNN/mu3d7xD7XEzdse8+Wzfjc37VU5IZ4LMe7kiuCMRhfJ17Q5pJA4mriTy9/nV5OQLhf20HsYMeyAzf19y6ZmcJig9+S6m8UNlRAndgQN+DZ05p5gshG6UbPSKFmKoKGvl2rA/fIHzQtcaKNkkSgCpWwBEaRe4I9m0dz/naSkFW/pwPUTGHwvETmjRclZYYI5U3VYsUcT8Q9XNxziuUgIGikhWj8/AIafZRmBY0jM7GUnbmFLFBkP5r6QLWpAdb272zUNGoBTDNqR3eN4RaowxPBagWjUW1Rcch/v7zyUuNtMhs23/gIkMQfsdI4V+a80am/guRmhiA/+R1MghTO3rpSA4EMWng7xLo+Y0MGC68IMN/7WpLVtlIHgT7t98OjKDb2GhTI4MyGe0s91U8bi+EWAX2ShzuIgKc4roCd7Mz33B7K+3X3PWnvXZu1EDjwb7N8QIvLJtjNUItzFCpqGs5IBcCC9itFn9xmXf4/dIhriZBnqyUZ1pFzGkHh1yudZljbjoBWtUMkXOaEX7xdbErxhW9KFBMPjPEyJ7HziGbku9OpneY97bc2JDJDIapthguGa/1MpaP+4q+szFdH5Op3URYsmnPCFAns1NnkU3GkZ09orWn62jd3Lu72nlLxexunYJG1Ya8AuDOBmsAZlKb4/XkH5Xpla0DjDlGIHJYzgkgUrKTZ74x0Kd61xe7T0lZrdvkrD3U59jYXwxBxrFCQIKJg9OMIyYq8+ilkjKGQk6lgVrxzpGDX1Sha1RJdSeo6jiKtrTbBO9oz/FZp3VAju6YpQBpSqYoSuhJgX+8vamqDCxuq3Aa8TChmHDlW/Youh4ZKXDf1lZhMVid1UuHblWYO0EHxUEIfIzZGHMoGdK9ay//aOOlXV4sikKS6wvapZIvQAU9hCQ5I5NyQkSWnpIo0tlaMxK5ZFVdtk3bYIoC4fJFzmvdNv9gpmmwt+dG/EdyAIpAFWqNSks3wOXndgtrphouiZyv3FAuQx31ihYZwBAvZMDSRVFm17TQTH/P/WHSwKtenDHx1uxjtpw6rVmEZ6ZUpPz9TRfahB/1nKNgXb+5ijOxxmdQGTwFtqiLtw7QoiRoC9fzx5xkXvEwt3EY6bKk0J20cyD6NVWjQlzEbBqtIfp3gmBgtwwvvaHv2qzDjxPTH3ITZWDUBpKukCof8jlRCyaAuyiJx+Lys5+vxdq0EXOSEWfW68pSK9UefUzldBRweqbbr8wMxwoBsnE733ZEKZkBOWKuktOnFjh+pUS1Y9g2jotCE08QyTwDdhrM21PkrQwq6GzGrk5RQkMRXUxGAwkogvxpfw86oxs00akFFXmWXXMUfqHhMRO15Rr/TPzXHsDfA6bN+XMFHmvXa683j6P1xLxiLyb4os9qZ01fSVCpjTriaUQj1chJ8+gmzMxT8Dake7+S+C9EbQjILkBE/bDlakqLfbu2v7gABfwBPTy2Zaf83aFabWmtZmf4ddn6M79SPrpbdU1GqSYnovxh+/gRbAib0/300uO+LdzdhW6vvXEZWi6qiGYRRVhoB94lK9b90rkT99iZQHjZ2K3PW86SXaE7bG8cuMMdG+gAu7135ygfo+hZxZDprXeEzeeufat9iVH+48h2ZNdkXj9K5Ao/W7n+jnXz9f7CmxDivEBeENMmyWUkXMBvNOJILtTShNjKyTlsKARilp8Z7FwgJBwesZEEOMYW6TRjUoYW8KWFWoGs4YanKfPYmWrdl9UkKB5TYXunDiVO+ZSKIKaoMMbvKo6bxjAuoI37cXIKhmt5P6Q9qMWL6iyOoAawnxEN6nV06fP8mQKfuCwOYKGCSopywHvvJcBJSb+OxRA3427Z/+Lzp2kXNNRO7thsKgX707IBog2dnyO6M/8PfLged30mDZafp5iYwA1eHTejpTeIraIVX87JfEWS3DYKgSm5K3d2OomhGDMgL9mGtQazxO+u810A4e1eqzq9+qe4OXz8QevXVkSPmcoXE62UxX90GFjFqfrg0li9ve2NoZ2XBQn58TAj1+BARufLyUqCKgEBIBLFTjx17Sm+a+CP8wcC1mnI6IaaVWHd9cHwsvk7PPx+qHeZ3Xtrczo+pQLABWjPv2OK3TVk5kaP/sX576na9XV0v1SOVU6vWqmDskY6cGpmhU4fmAXQKihyg1qms+B4ChJwrDZrLX/M1maLaLYacT+p7w8rveoc1rzNPOYmHpdMLRbIqMxogW16Ka7B+5/71L6L92VKnqDGlPi/jwHUvh3NaW7PN/APqrOrYEmAZcGj5ch5qgD6eiyTW/fHLiiY1JQ2DibypqaxqOqnD0FOloD8BDbXjjFhJ7lGNgtTgYYyoyuqaEb/ypl5pUpLECwWPdLPrBM+p7Fwq8FMvfcG3spNUq5IC2B7egfTBC0kTdnX3hxcQaSzbjjsXf/+Gloz27t4e7eoxF67aDBFqyvKVKxN9bdUtK99p8tUR1xIey7oAoEJH15ne3/y4BvnZoTNW6finnjU7vVvRo0zoVv9ucpd8dSAK3pr03AdPdt1KK6X/RMa1PVu2sE5TfNPo+68rGiAr/sCE/7LYQby4OOHH0AVfjzOwLg8YaVxtzPlyLGBaEEoTTPe5tA7r64a7zyMNze6ZMIRJgb8MBPZ8AZLDFVUSl6fixvWZhMF0RwdcMZZimKmqaoruhl/Hv5v2K3Z1eOUw713+Sr3UpEvn0ltUS89vAb0HP7wKu1X1+X1O9drnwsBSugt3LDV+GBXImeTD0bJeQTm1oOB1UhbFWFmH3MWPELhx/wuz+btnP6uoW5fZhY3o3uzeY6ulYeA2PX2rAIWi2F9GrVLFbGdv/Fxxpr+7e2/Nf/vW5nBPPgpBPk8WEsSXoH1aIKiWJXzqcoUhuDMr8zIbISoDPrmA6+MZeRQNuuvakN8qPLaf/HuDq/K0RAX0lx+8uogs1MJJSgHRQu99uq3NOUD0ItbUPTcS1uX01BdzigMHVitX9XNYx41CgIXKutwetHzSEVtAwhs7DPterpkpywo9NZNN48AK1N0wMsKExpVg3gVyeK6LwSioPpcnMwlR6zekdvXki1JDBDyvKDw6Ro7xrKTI3BMyrBn2cHgo3tT5+3SoAjITSsnmPaPcNQ0SgqcIOdB+cM6vsXFPNc1zVfX0QJwURMdUo/wT+vdb+dZTnyOeelHft+pTaAZKQNiwdvXrMWILoRLAxdNuaxe1xnbXtGNwfVJqXo/EjRUvtjHNvP7Fbu5gR4bXwKlmWqEhBiTKOybd5Jal7FxeyRSK/KcX/oYBB0ZzMd4NraWkZ4WDsT8IHlUj+ps0AnlJUhEcKxK4xAz7fA9Mqy/323BF+srclLgurTazBKF1iy69k9ktv6qPxFvzlrv6xa2NdAVY6snzvaGbr+iIO5McTvoAJLEKU2q3NMcNza5KRBKUSq59Bau5JQ0QeTXv6UEGWDw3AWXhAXvCnmcM1Cl1e7akO7pGvp4Gp9h3MWh2VAaptFa93NcY7rA1WO4ntOTF9Pm9S7VV6cvD16/b/Oy+fdHVoywQdFlT2JYnSo7lN+OzF3Imtbp92BRA4Oz8VJj2uZVMtHwwdBv777xsWcialW07ua7gzzOyoOuLhTB9cXnDLOwDgu5v5Zz5DF8CE4nFoGOsyZi2NGXxjj3ALeCwOdOAFCkJNTTg95sYB3bSf4hOveq2qdFSnBgklFaYMmKu3rNs3fMdxRcHfPu6DcaAR2bnHYlE6S7RFOXjejSTfh39E3i5pETJNRHqKWndh5Ztsx8haUZJL8F6kSqhkUvfIAksaq3w/R2psvFN2+nsulEwABYMPm+fKDnAC0hhadW2mAMNbTBm06h2oypKYjNAgli9s/XCSWsasY/88S5JrN3RQDty4D3ZGjeM6+Hdl7UIMKYtWdDpMWIVGXEp4VoLb8+dKFg721ycae/rnII38MQkaunht94ECG2qbqr1/SZeA3SBi7K12RVDyq0+NWP5v+bMb2/ONfcyKubxPeVB+24nxUeskxZPufbbbZKoFfSf+RLfwidfXWhkfCUFpvMjbgRlgLhvqq65bkZOmn3KaERyPUTG7ik3QS8QpKJ0/9IG2vMcpchSjPJSUYD8ijBy2G47AurTP4wprJKeMUFAORXuz2u/IZthDXyQb9EJmpAtyrisf85FRLpk+PBxkBDqMmZH9MWzB4CA8x0JPYDPecUq4MJ8DH8LDuF2uMW4Fs7B7Kc/XGdmwqo4dB0xci1No7t7mvZnr4k1p/Q7ifxMcrqRCgPRq4TnLD1kiOl/8WMIPfF8DQ0DEJS0KkrB9MOTaJ8GdPLx0WH4Wh4pua4p4hOGX0sO9AY2QooRGGEBe4Bwr121diEct8KBF7mRhE0Ibc1VVasmA1EDBmO5jHHywdz0EEj6H70vx+zZ6tbVLtgc/ZObl2H9Dj/8ElpayMitIIpFNA+st60L6B2GfqYi2SgZM0xR5pxnn3V8RRFEXnx2boLlRUmWn7lYbnVmwR+IDz043+zTIA67coeX1+Am3vtp0ejsLo6qNiBwMR9l0h8xHmL3GjctLz+ZebzYllD87A9xt2B9SrXH6HVH2a2pMi01LF9x7dEiBop/cUeEBxrc6PNoE3uRm8cLwRW//VobKtzhUCmbMY+FhOxbt+3xJjBBqzOpPXYu06YP3NxliCEI6ymusPZElicujdA2syBrOTSTwe/94PpSAf1g5GDbqIqrpHq9hFs+d8NKQNkJZQszmdGl8mK85b/Ve5jcHg2dra/IrqIgMnYbMDNVqQpDqr27GAk2VdFZ5vAUb9Q1zERi2Vh2QNuFVMAz88F+39rd6a8lXacZ9GsAqpKE0RFx0hDqJbqpEx1AVE6+PNj1CErlXtZk1zOdb6p9xVQt0Ve6O8ACCNtqs4MteqrFyPqTAMjh76caeccfpvfuX9o2CxnIpYw30PGMXUBQDgs4Z01AmsM89OKOUkWVvT361G6SGif39qXOsNR1fmfF+k7gZv1OTAu1XXjqCAeCFgUPAnYq0ljrVNGwEOEqzuHTJ6WOqUx1o3warTj3lKiSoNvVlIYUFJbMDcW78kM+Qm/pk+xhbIjhaONP/evFxgaxVVgcv4DFjCB03I0Tl6/EyU8+8+XAQrZFmSBWIoq2dOZpa/fGJSavrtCABcHQjUreTqTrWs9XJkoXKDByBL8FvcaxDtXYKQUIylakwrXC9xNxV2++C3V+GkvdeQ9IWH5Hh56+KJlDK+jywkwIWLaOa2i3NgYYvi5BtiD+ehK7gli5BgM1x4xwzNdshJCjFrbfogdqHVVWJYfLKJwpWSTUvYiPQCN5YZ2EGlMkBmzb6/U5dJCfLzWM5zz3cvcf7zFVbdAaFzMukJf40bt47edOoDqUIm7ZnrsYDgFMYQwnh7THjrNz3Pq+FlN2b+w3G4vjM2CZo9nzlmgAnqJUQqfHm2WMCOV4AbMiabvp8rq9tbOes/TA8KyYWSBYO6Nsylv74AFl6k7wLpOf8x/uv7UZjv2wgNJgujR5whx3NxJ3JG4V2lhCCZH8yooflYoTvHg5n6afuTTTKuan8xdn8YvNK4Pzh7NHK9/NN/bbfDlZXSFr49/1kaAwrlaaqE7uGvQFrZKj1YoldILCCnD1RRY73+fFbPyR1X9m1FIC71PVSPfMnBLA3dj2/JF8y7gz9K9DW9+uHkm/Gw8pP4oCnvSNws9A1SJ15rJyqnurNtKGhjEEZf3mkII2Hr9YTkn/RvjtD5Vv6K+IXXjiWKrttggIAEmURBH5A3FPuA76h7TEZE0xqFsYpC1x3ItdiiorOgI6X0dsQOUpJ5NkFBe3poSZgMTJBtvP4ZYMm8uldjUJ1Ax4wPFoKmI8H0tAEhc5LhCxGBtSQHK0deSKe1YTHGwdsYYAUvNPoyMQk7wXtqJqkjsLLqcOUXUFo2rtqMdBt15z2D60skWACbwX9TSR5ao1sfP8VvZevT4RsyN2O+309tod/3Z4p/SGGCidIk1fn3fKoWC5vgGDoSjKoh6c5vRGGhYuJPGXz+FMOUNi9PNKvTi788JMa+K+/bw+BdRgU27Cs2dMYwfV9dH6RAbtN1+Ey6fcdXxEQ5KYzgf/Ickt7xmtJA1KrX4TWLlQjaFb4cUYLUaOVwf5NB+t3Mx6ZkUsewupOVLamm+qqmyUtUIkKbRIRDwwo42AFlR7dScUlpeIPiUCNl7ulOHuhlfeeBa5ft0UK2tyPp14fJL8xDwmneLmu76Lapw1GtsCaA0ND5ok3KvWohN54VucbbF7xRofQACjsOdJ58foyJrkG1ybhpkRMw0M5T50y9hwkbdCVM66tbPViscnoP7k2sBiTNeB4/k7O3vUX9lw3KyvE1EyyoodrU5wu5Oi60uKxy/RBUEr0so3Qu1cQTJZWmDbYSMHWDIrydDXP9jPov3IyT8YRq2zNJQtlfAFx6y69QuSDUTu5SdYxAKisPcTI6m7dl6Tr10APJzqVPS4Dqkx6Ux4tTEfz42X0nn6QVbDirHOcI06oRv6LoD1ZYMGpkA0BhpTfO/m7b4XSef2b2UCmjneNU1MbQtj2637FbfUwQRe8aTVciiAkX675rSunp8fceYtU8ROc17lX/Ue+kzYH2lN+bKpcTtzhDeUoqkAF/otmfz5B9fzKkrVKtj7gZOGrr/3aHqpShWSuXolbZNzvuAuoIp6uuoAA88PLBiLhV4ZBYJjInMNa9k4yUbT14hN7dJp0ajijLgaOHw5mM0/3/uN1S3jTkuw0Vs/mXx0HFjsSb1uiB0lj8MzNXV13zXPktUNOKq/OJz2Jf984eQLJ1U6F6/SdXcDlrtICH6IffTDx1+6SNWNOqTOt6X+ir8LHFBS436tbWx/cdQmNanO1dWytEWINZCE5AXXjrVN77Bt73bSwsiL9XGmRXFPfcPQSqns0NRteq6ndpaStkZjX8kpsjKShv1m9KGRwbBE7LnIVGiZDoSIDNpGW1ww18wFbSNfSNerFb3dzAs40MerlmyBUsTfvhuPha64LTQJ4EZO4eQjkHDSUcon63FqderCH1YC3uFVJjBaAZYPwKPN5R296Q0yrF+0ZQKcyOq+AYVSXbRRPac107KHd4b/9eI2YFPwl59t0I18dwou3bOAWVNlFdqB1NUQ8MuaYANiLfhl0ahf6zypwIxlOI5djuSuMpGbVibLGK8PascMxUdFa4P2rUe9rd2jJtNeJw7dCWlX0nIKChCW+IYUA+FjFwxepmkvyyH6VWFc8ks20CPZvgy7O7z4KqqjN6sFVazVqv51TpEMR/O11GjtAflsosO88vuyXVYVr4Bj7We4y/yV7MjKhdxa2myVwyQvBspl7tsTLRIjs1XnBnAtr3HwgmTXCjS1qRG3D6QkE1t6Q+X35eZLk/ZJY4OaUF1qQ+wLRW6PWqQbLMvjvMAIAPXpqkikotSOfGAmedtBi08c3Dk+iUgaxcq3jpEj2uPFyIUthB5xDq7Zn2zLxXh2ynrNh7ydLGvY7rb0Vx1bVmji4dz0ym/9ZPN/E8On5zuoYf5LtN+0C6CCOfRKqEKGkQQeRckSFxeUT/ttIuyKxL/x9f0MlzpXbv04Ryr8sSNpRTd1WhZ66Ywv0N6qDtnyhg1I1QlH7N+tYAn6hLgsN2F0Sv3f9/Ce0nyCF8qdslgCTCKvqCxvsx12RK6bHHR48Y57FiKuJudd6eCZT7Eb/iBPx7NsrokQe75t9VwcSF87Htv7JiYkKGVAdQomKGGkYkhCmC5LYoqos/f9QdjKYuqinOy3hPRmC3cI+zlnLXkmjRsT4U+EttZSvOP1/eKad+9hg+o4J5+1XRDXrpCAVaOW2Rh2ymTT8ZRUS8qx4Cx3QvOn35q6+NRRhxhbGWjQ5w/kHBrtLq8PClK5XCBV0KxMr0ABxgxFicXOdZWvFe9pDk+V3BKNATHwkpny049pZnNn5tgfOtBlLF181v4axl9ImuWmCdWODqK9pjZ6efqRo7UD42ZtrT23HXL3dFdqPg5oDI8te7jaamnT1I5DvWvvLLb8IK8YmDOjjFAmn+q03xRTQIaIcTNzXCYm8bS21dZ7ipcSe3XsW0CKl3fKsLdXNTB0d+yw+0Q6gUt5VVbBmcTuDiMJB5LPspXKmyIWL+fXON7AgVQr4YrYsMvA3qGFMU2WDaUpmHOKQ+FaxNhqI0lo0/O6diuZCLeNTaCCJLosyGBIQdeB663BvWRxq69Yh7je28Y5/VX4oBgOQpJe9nUYJ4yn9Gi7Ssslr2kBK6P9XkgANE2VotqzrfiZBAZkTcbwLyCpFmjMyv5OkKpNy2lGKSaqphmbx5lTMFXDxHFu3zdHgWoU+B2kblXXulum7mpxHIqpSmISSRHvP17t5Lb7u8Ixd3xzX2/IJkPBCC7xhkw22TreDnYzf4vECuV2VuM0siTZOwqnFQ2TlAVRCO+Wu2oMiAgdkR8Tw0DqSawYuNcRsXqgD0oT/wUglYrBbDU6/4a47qpgqbjdUVrXPaCRgL4YBnizDcJJS5q6pn3htSqFPPR62MCOqFpyYEQlb+to83XvAp+GqVY4jI1AagF1Yr4tOECq+9zeBlybYxXn0FLRVuP6PM6k6ycCUEQfiC2YiakkRSuB443DJmiXXHgqNhSkHftxLpenRAQLomQg6BYUig5uu3b1JXMH33LCNUBAI3z4sUtzcY6ONQv7uPjzuAHEliSDB4YD2T7rDaB26Tgmapm1VtiOxE7QVUBq0FVDhNA8I1p715TXm7RRC+TNqwKH/VJNkcLaBRxc/pzff7YMOKqm1GBA9syyDYafHoBJzoLwWWPzgJHMSG/qKZ4R5HtG2L+rzuiKYVqW3mnQAc+TicdMIr8cJbioXv1v0dv/elwmJ4Wm6fCaLmrZdGTvwtCfq5Sqy8rp3l7PrlOaOZOPO2oYNiXXb+Bx2KN71BBIgfgAxNX5POQyV5fOnPw92vK8Y8sZt0JUqlSYY5upmRnWHv88y508uMB/3KN7KuDyjZIPKzo1KEqUa2m5c/Fo50i/h6llp1cbE1k0U0mWgWuSfkJImbVOOJTdD6goYbHsSu2KKdl531VV6Wwm1cMqwAr9ehPmRawm/tGosz05vH/DU1hoYXLe6ziMizY0ioRkRXXKV16w15kStlArbC24Xkm0aN42vmrYjMdWPslQyvmxxIFTUlhc0YArL/Zh+K7xnLU7bDbC56/HldpPTe7QU5TXY13Z8Q3OJ55sGI7VTpvDdmfQ60y2A1UxCJW3b6yXdzhQjbf32jJIG57ZZGC+6tdczlrAFg1oTk/opx9NSFpdw+pbt0q67fmyhymWhV8WF+bW2lUtU24IQGIUCbIcrqODh6Wj9mZg5KMEA4GpahykOVkQUZQqlJMy2APT4SEeqbGcGp640qzKLCXbiiH4VkXMeTZwhSCPOWuyoPmCWlc0VOOmeE2tY1hRDkv9gWmGoeAJjed93QLO5l87mIEtckVo2AmuWgpt9YbeIDzuf3fJn8ZhGKkdhWTiY8VQcD5v3ZC9FjomOk1HY8NKw3G4UJIsnk2F2Ny0XrV95IH9WuyEg/jdZwVhtRglP/n8zPL5miW5PboIPAbdc9hM/CAa1gARc+X/i3H+rFG9JoGRuI6w/m0gwUhwz4aX1WyPUS/+tfa+GFGlSIhhqqxZCbLTui9EAAoqZoZKvMY5VTs7kbf8Lu63GsPbh73x4WA8frfX4zdWVcuMybVmTEE856aKL+68HNJtPXL7UXp48HuVuGN39sxR0NKDRq1ZcVcMXQP3q5/f+7B3c/HSDv1TVvO43qKIb3M1jrt50/UWa6KBabJD7O4Mqm0nY6qWbPON0oZLll+ZXcWLa8ClmqcHzxz5oqPUmfc+KPbyFcKJrYE6iIeG6YdDp2WNlbaWZiM7as8akLGwpL+3Iy7HGwlhoAQa8YSjDa8MeRuIvEAEOIa6SRWxQrWkxJWfM1cqVzvjmFMfaz9KpWMfd3rxiNM40HuE7OywYtbGNiwOc+oQ1VOtsNFZ73C3uPWlA+bAO5Rj3PDdN5RI7nYf8BmPd+tX13YasgYmzWpXTcvNUggS3sWUgniwl5ZRXK2YnKmM+mvOSDvi+rlb5L31bMaRUE6v6sDEsrsKRNHYLaGg75hsiqG1q3ve/uV+yd0i3ItEKFdNPi+6y9fKkXRNbN4q6DWQbzJLg5Ymq6IkcvFN0SFARZf78KlQqU6TCiLcyJfOy0QPpB/3sTP9c5Zg4a5svueCabLzqstW+lK3y9gadS5rSxkRiU70QGhtuLa90OxkNigzQcIp4T4TAWib1jxodlZq9qUEiS2GS0NLUBZkkFwZT2T/C2RXz9ufCVCx+/+0MbGP74od/O859pzCf9NDClLEe68zaUTOfsNUCv2TmToicctpAMmxWXfgo6ZFDfCYzzANJN/bP34nYjKdrUqR4ZqICen3MFcgLdJGZq/+ggnSZQmy7YNK0sBiogH14ANABw6v2ScEvLzTpqmwbaZVyCLzenewu8qyQFrEn/+YAw593NbBnmtAjWVADV5t6JVZXjEEDGsgNeOrFC0OjP1NS/0kXZA2a2Fmt/N33VyovqABLyJqhVFCgcTzhIcRQpQ4dJwTX4pI0Me/fHUAI9wx8ht29Pe7vnIPKR/cNyWt19ZL652ZBACqSilyhlVZ6yu9+UjxfJKvewoYuhmTBVS/Jjr+IgojSlVsS21gSbfkezvPM8zZmuAhOFTS4PzU6452MbQ19IfSBmWZ66zLQyUCMuQLuIsDUNLHXApno4fCtmrGmVLItQcJyEyFTRUBpWkywpCYrASWp3A6KehFp9bIpVl/u0avFuyo5uS1b2v7AJWo4grbR+PlVgHw4WIC0+BJUuVioVzK8JjcyzGugsf+YlvKMoOicsc80R3sEocACgJMMSTjF9wJLiwk8A+ghXSElxgO7k//AKawQtraGgKnbGMLURD0aqn+VswRfaiPABhbJzGsgnfytJpjX9sgTJJ1LtgIpMuawFNcY+kWdlQDNKopURYQLJCA+38WzYzJCkyIbWlb03HG0l1V0xw3IxlamzC8WfQo25LPwCYBiAlNZfiRKSmhqgwAj101MatYsa9nbFfll1JsJekhPlHGIgtwTR3HZmIaqlM5nB8LWMqTmz+5TZU/6TRADEpyIBU2CpOeX8R+ivLP1s0Tb8DuYLTrIxkhRw2hbPVPdzDG9KBYCGQJwEweYjX3OuwyKcUY9SLWBK/owhNjyRRJ13mZIfbXbikAsSRayPVGA+tLm4wlHap9gLUl5syr69HcKdHhPrlENYsLU/v1JST3f64FITEsRM9w8LJAKcPZq1uyCgziiuXAng534eNoZnvxyEbhgHtEgXLf0QfZcaAAUCEyY1lvfPNTbV3dihG3hbq5HUUW9Of2VO2CfqPY/TEm80E69vu1H9pAyLQjjeQLYUiQ6Zh5KhRCAVl/m8AIKKB26HftmgsFk0aW/tLsXIIuXzGnJ9lp4K+8WR1AO080F03DX79IJjtccXA9bFVBEOtFF6/fYGAsFukraLOXLOyVwB+2MvXJdTpy3AW7cV0HaDjYyUJWLLjROiAsSCm19eyMdYVDEMs9+LwF57C+OrJd+8Bdw3uyosgEZZgpQaR1hRkYuaMQmAJ62rFQDfK6FEu05E50P7Fprm2Ck6meFzOFAf2Ny6xyWdd7rubPSKa/YTBlX8SzBDgdKokYyhSoGkQu5aPT/uizSwSofZLk4H9i667CftfWg06+vdCBy+GGHwhABlXDjpwJSEPVM+xe/qkPdpfZlvLYSLoc7DGsKpkKBha0Jt202wVQTdYpLI8HE9MEVsnwbEblnCplveAAKnPlcmBlPG2lzSAAsQ0vMp1OXoCI7vgRZ6K/TduWapAQ38CvXZXRPd2av+ulvUVeL1qJUujc3Seo3312OERUF/0nAizQOuL5PjDohB5keU82LUzYkdl+ZR8j3uSA51UHf4Y2b9ItwNingP6kCMSDC45rlfaOCxaIvWeKEEEJK0sHH3hF1uAFUoOROO/zaJDmBYNtb9hNd19IbtTMFtUaAnU4l7wFp0Xe3IMvvcjyaY18XoAYcuN+JbFuwVMBRHh83fdZt2/U3WBkRtbb3P4PaRL5pu3WtndX2S6v4VMtQIDJAqntwUkCNwRtOcaCjtxf+RjJbpxo0RV0NpE6xQlHigD+jyf7qlGm8j4pBUvwEyTV6M/KZmBBMPXWTQzz8lQ1uxchHTaY8RpjwKT+FrTANQBzvEKRCbJm1G0w+UDjooXEgdDxvzQJeo2xCfYwYVahIWStBHoBFcs76C6Yuy3tsmN4KxRhA+glqVUcCqldtcKmw1OMadZ9aFuCQ7qH5HWJuCQx7TQagEMTtJ2wNZD+WJZA8t7Raa8Fp19ODpsIqBirqJKagqmhZqgmF9dP9lxFhSEItN3mzV9uolKMu/Eyfkni63UYApyP1eWnr3M47M5x13gHUoDctYO5jkEMEAeWmqgLam1h7NTxuSCor3+/9v0cQqsv8x1YGjfMNgKKQqBYbRoGeo1hqGkwKGmYi0Z0TEUr+ydlkBOc71AYri7CQYoRIuQ1GSIdKg0NNiJt4TQAEum/IkHApsgojjCoELEq//IeATu7QmMVs9KS1ygfzmCVBNRW2HfOiG6um50PdpquZrkd4tXeaF3dB61tt4OAlRF3AvtKD+g972w0I9WuXzwoQ7FMkMXvE/o+mo5cJHAMCa+RvRusSt5x8wzPTX6Nzzqs6qSrjI6TMxCF5dyEgoJ9Gu9cHmLBc/ws9ktTVr7m+BxlxFhcDC7HVuU0hp16iExN2zwOnKHyMGRNRktT5PAY/K3WEQ3abAej2bT9LOVh1lSunF4kBFxyUls19FbwOfAORkI4aLWdZONY+cELzkxrgoDL+DblQ8uf8LjZDdPQBYOQd3QepgrcvxzAJaewkzK+u9SNjhLMBZsBqrJCynppqAAWnMDf42TzCeGMZz0ZH4E7321Z1Z3pYVB84N3y9X2SztzWkgI/kzjnPTruSyi3JpPLY8XdTsHh9IcvQ+/WrSBFNdsbzMf8tY1tBm2odi/Ff34e7Utjv1qE3QTHY8Kj7tjP7A8iYjSK82MHv1FcOH++3lrYfe8mEFJ1MYT5RLhpoxWLVfc6G4SyG46dCeKb6fx756NLr7pB/AaDAdAfi3aynwE4TPXtCTxUH8nyu2ij3rZUhwC0/SQoMay8IpnsLpsxP1q7mITcZTkE7b0BT8W1w7kYngYJzdXVULolhJPbbyIooOtXfhM5RevEZMDsE+au5fdAc1f+jTqMRDwZDa/9XjZRseAZSeYl5uT2EUmyjI3AhZp+YW7rh35Y+GI58R6K1hrPe50hWTxw2DQonPxuToOAanmUdjoZoNoBxnfuZOzycGAxamZ8oJn7+7e/3bA+On/sQ5PHQavQibYMDRkAtPDKdQeDu+815nHy+fOqWtrs7wIOUIKvK8iBgN121eJgyNSCvxuPsn3GmA/EuHhuDuqCpFQwZbbJekisy451w0iutQDdAsnILENknnVK3Vcm6uouLbHdTkmEtgZTnnoolQ1z24ohJ2cUFeM6hro8BUHxD9StgU9kXCpsegzUqIspqENvecyMyb0s7BA6r1xHJb78s7UAjJFWUYIJUdeQAfXBqMHkyMEFF+kHfVHKrkBUX5m/AgXHa3s0q87wHkwDoOBooUxySpAprjOgNOsEgqQkIgLo1pIkhTJX7j7hGZb/PTt+igOrm4r+F2hp+NG0C+WzD9d58ID9j5RjZAT5TatjfuPNTgPLj52KkEz8RYHrFOpASSoYTmVVap+ANEyH1bsvQBxwwmotkQT52uU4vjF/5kGCAKmAm/keHObWYvS3yLz/UU6AjIMw69XEK4CBXO6SxLhzH4/2viJCeyC/5udpGqHpSzeSBDztVZ0FRo5OCbM7D3TPj1naVLj5l/sRX6TSBJL95j2MoWA2LW5JJqdGMCinBVG0ugXLwK0oKrzod2JkwWeeWs2+D72DCBJVdDY6M0UZFBUzDX7yjIX0gWi07eMMBjnjphGwboh49tMgDt2zgiKKCmxTjoowq04/UNuptuk/DyHrbBD1xazXPXH2zlNDxtsVA256IpMhWn4NiCaTra2MO9gdq85Xbx8bOXx46Fn9rmXNOON+rwlTOcuwRwAkMEEAtmIBZSJeEOnNURQaj69sWVZoZ5UxnitWOzDa2R9ICAhYJuxIZkwQZaLAiLrY7DIyrExoXbUBZBh8pX5e02O8aEYUZBuu3tZgADAYH0B3NQnFFw2sn7L9dJlN0ppUY8LouUhn49BsUaUgArS+zUlUwxhOhwFdLjVjMLuZGcMLYQOKqCP1x/FkGGemwlhBCyAvFHQeSyyZU/HFuCdiwT2RQjI3MklP0FA6FcCYd6Ry/05v6DGFxuna22LmrtHLFSismVIAp7V4DdVgK7e7T/UAM8YsHvgboNoP1JSqUw/bTQWp+s2bz0J/tF9eQm3uqsC2afRvV9losn6nYoOcXzqchwezOzeHckzhWgodhmarnrkjAlHbzpYlk+1Sj9Qp1EBhyOhBCdRAayM0xG+GGJLWcfERSjOvJ2jbVKspL7gxq4Ck1rQDRpf7tPh9rwcI4nmtD9sr1VLoX0c+cWYxNn8I8Gb8KEfNlp7qXx2UC9CgStSAgo5wcB5VVHR2cmhK97wo/9RCDkDzozuV5aFoRT29n6yL3cfaxDuOZ1re6sfOI0ZmPuChTiuLHDotTRSp/CuNEWd+yxXA+lrkbz+1ZYGYBtyzZbsCR193ldnXtff55VYg7uuLfZgPKGtrRZm9i568IrR8yCR1y4YEO9ppwRQjtD30vQ+GC0Kzx74f2EKys8+2zpzmTOhFqmLMmU601O61CMgUD3T0GsKkL2951hdOVRGzpb8go4vCaxKG3LQJLGCe5oQssggKd4lr1gZ+oZ7j5he+vPbuO9aTcSZKV8pFxnqJu0BIYMy+A9wcXNzdJzErIBb2XHussr5Ufhp61kExc+c1W/NtMMTOxuao7b9/izU7FRq/OT9CfQ/T6M7HBVxeWv7m7gMrgKY/W28yl+Z+D4PmEyL5YXgtnkH0q4VSaeYjH1HcgSLJkym3899Ce355DvFaU/UnKPGpbhsVYfT0r34ztoup35dCYHRFfWa/JUq2xgRaKnmIzdoy7O1UeAvl5WIJ6hMyZZMNbuCLjctXRoYDbACzB8tmnmNaN6+vgloCrwbHR76sHwSe9pRiObYuwD5p7B3cfa2+vOQHRJnt/Sxceaa+bjF7DrM3d7jnwZ+/EEJbPlE/TVvYFsFI/9bFngX8lZcrHXj7/qWwBwlqkhIVR+pmlrbF9iVSBt4eqtGGWQbTGnq8oH21HGLDls+bbV56mtT4RrOcZEGzfH+1DQtyHaN0xvLMbjBsTjoWkJM6NaCeqpVs9P0/lX2d7krm+6ZY8mnzJr8c38DeMbffwUblm90UYwBX1TANKT/DA9tcz/2X136bl9GAnvZ3fWn3DSNw/AbqsFzNl9nW3O+n0RNsp4fh8SmpCcNW2gmA3a7lk2XwWM9LItQj6cg3jkGz5h1VQXv++NlBtxuji2pTPW0yZkCibTC3Hp5PixpVk+GwFikRoBiziSY6PSd63BQVE9WDevhr1cA4aPsWpbiEyQnrfIKbAmlYTMUKWK/AGSKSx880ndMvq9SSs+89GwBHbkzGvQ39gdQDwK4hBKetCJWf9L98O4GAdfYSEi5i4Fq/EEmYqcxCOUtZPb8Nuf7XF+1IwiMOL8MIHxbKKAUhY8zRCLQ19lroe5LQERkspdTtCWOPJc4GBNoRUbKRnALaQ5VA7fmy+BhWz234u9edKeS51+OH/eKszKihl7mFTFSSiBXKngrtxx+k7NAp1XMuGhjYURyYAGrR72hIhrFqfXdVB9lAdpCFDYBtyYYwy10Sg+oXiAtB4qgTy+aneAW3MNPInKp2JCH32L63DfW0IluIMw1Fh46+YVS7IB29u8JeL9zku8nnaAGSVMcSCtZm5RdqF9IU+SJJdJ3pTznxCmuwxtU8dB3UsT8BJnljUkJURJbuRnN1SWaImZ3BE5dP+uLae32wtiD/n7UEHS663cd/D9KFB3Sn3Rt9OAuEUTfdQIIyiDCcBm2GHo5nZKgJxWYC4+BIK6IaXt7rP1r8ZWC7dpPg/oPjkO9vG8faZGLMN+LO0KRxufGXvQgl/HlTOGK0ZAeYguiV4JLUxSsXYjC27XoX3UYEPYlMuKY8yMUwMFMFD3M1JLLuto7kyeuJGGyZFxY3eiDitwbzyP25TiQOXWblmLQdsWsDJ8ZdCYxqt7IRPFv8zgTbhCvkl61V6P7Tat6Tzy+PwF80q1gW6k132/Os+/eucynPMQTHZwI8C5V6PH3wx7EtiAC7aTvdI2ALJTajlfXKbx8jzoMPghkJb3twW0L/ypTKdXYRgV2JWBr/4zakFOXAdnxt3h6BU8cfrEzg7LDYvtmXd2e/szIAgu0mga7+bwLkNjZiW6rHB38pPS8dLL/WNS74A08h8M+tOoRCdkGBpVRCFgx85exllrAbKRIPioaTa+4YWjmZ99Co0ckmiqSXrAz7vVKPtVDe8EriZWOCu3dKwU+4KdcfvRJ3qKGsq0CaITqH4uB5WiMF5uBw3Kixq/x3pmkyLVdyavF3wrvSHgH3lGTzouhczo/wPeyvAtejRZyPY5Ft+puesfuzxFWrW2iBqb4UIQUAz240GujF/cS8RCVkr6kTYKitcFjBl5Eo1iU4LI0Fw5wAh0J4wM8D5WoskR0hbpoU0mZOU5kvdhLQ/LEMMtQustuB1m/y7QjMvZcVTxA069hCLzl7QNKoU9iYbnMj92tJinK/iwHNyWpbxTY4tr/PYvAK14JZEGe3Y4SstOcz3xNErGxMdbE+Fkvg+BKTX72Nxg1Sd2CVc+8zavPhr6UcBn3RgIY2TbFJH5sIdLg/lpzYm4P93YKLPg7JI0s2AgqKBpcFJ/iN++SUJFFk4VNp8d7YA3cwzU9xPbYKRYWXKlV4Z6RPC1FGrGaEOM660sZFLQz5E8c9sWYEL738W6GEZ4SfkVihfvfUcYfpH0EW0rjE+5bRq7bjS9S/MaD+n/qt+D8IWCzbdhAMxRi35IHe00AffN7gcTr0f3L8Q9gUt4Dmvz7gIIqmmYIZifZyhwPWFsovnNxOiOQ/a0PvFU5gwmS4phDSpp94QaomtXA50sySQhHAAphEEiipKFNFyD813AF+4v0lE7NYcw5IaFAryoBrDYNODuTA4shv/fPxdSgFltk/fGpmBwhPGNrWjcD1fqLoP6JtIb5qzuauYXMGCBGJnUPNWTe9qQPb35eqEuDGdQoroM2eRWZ3Cdgy+1YYz9V693bf2jSwZv56CSx3/YgJCtU0aOd/voAO24LcfW982yHJ3eIjWBjx3Bg8GnePRRy3yH3/GqicvEma5UBj5mKqa8dEAsotmc59qBmai79hF24oagXt+frQ8t3WaVzwrzGZIW/0Kubgy4cSjEUCs4j23dtvwyvuPnCD57rddzs0gZzHMq5eG++EAYN1sdB6YxMT6H0THMhEKTvzA9fAjqXLoezft63rN7U/bUOm6S1pE66c0/xdhNDjR08ev7YHKiMIIW0WGlo15Bwcj8IQZYOTgUmBMBYRgvgieNL9GBftlVTOkwvBBBgQsknBjPi9uJ0r+tllTABGPBtrItwXTV1uHvDLl3RkbUBf4yHJXgTinxZmrxVQYjDATTgEj5VP6Vs+Gm6c++2zW//5lW27wbqH7+Pu63po4EHlzK4/Qt89fraoQXn8ZAt8DVzaxDM++NzaUeEP+0S3Fzdc4Nv1NjeGn3xXiMFaYnQCQomvDedpQczPVTUgWVhYSx7W03dSRb8ICg8L3G0v5xBGQaBfqZoEmcOiGthpf7UwVsCmLPhc226kneypiQGkdBdlBfiqh9pLVaIJQVcXFX2GDuA/O6A1XWwyMr9SBG9JgGMvmCOWqKopwAQ5rdnyGAUROZNrsWPRsd59jbXjA21pwrV1dv1Cpx5FtHK3MAYF6JAC1+gi3NlhclU07pNlHWYOeY6hXR8a+QuUVk9TwMsFEr8KGAh4rahAFk6T0Q01nek9fLzigx25BYnx+309hxzaancoptvAUqQO4FtvjI+zshJgGaqAHyxzJiK+MMz6VyefmvvDpJGPOpCJOAraPqfTvrJVfn+EHk8CPaGn0BpIRo6+4FeLclrxkm1pCc74rc3LhwyADDUIrJC+DOL3PeNZ0fgdFnFpS3Hg3wGW5XmUPX9n+qAIBSFfHqptUoKTapNDDyy8CMWjp5FFH3trqhDVxpdjAZbn374lopNNpFNsAqTSHst+CwyzjJ3/mihTgrziKpcK9EtODmDt/Q8jbTvahD0Yv8U3hk+UmRt2FjpNF9MfvnNs1H5rrKrHg5ofpkBBRzqrYkjNmYW2BozgF5mFGv6/oHAmWcBOaGaGuq8/XeTaI7+4J8sW+wmxEVKfvvZuGLSph/zbstHbWj7u7IEcbPrXGAuRtGVSgcMSm4rQhbd7ILI6Q1S6+kKuYcSK3Hwqj30g2oxj/EkmAlq+tiAjp5PikoID1/65T0z65/em0GPuCA2oZDXdQXoqjb1erzOTHZzidnBfMsomfGtjbPjBqHbYNvQ2yMNA3rYz+qZbt6pbIM1Zq00kEc0NdYg26G107yr7VuafOLLtOq979sxlFsEGqbCX+bTaq39q8D8iSReKOcATtZLjgnjWrwZQ1jHVkWsGnhZO9h8umI+ACvnKsk/U2LJILd9YzukjgGyDh/4bkzBeo3WDZ5vEBf8PxP6/YonaXADWFvzCDQ9zSG0dfhiMemz0PUZBvoYA9VQL/fXGAr7CZbEPg368ewE2NBhMG+WFKVs8dYoYYstsqSPDQ6z4PCpw9MlMUwhegEqSf6UgvK3OuD/2CTHREVunW8oW8MYFkYKaqklXYAGEiiIhXq7sKq5dBsADO2FQRacsTt8BzUiONMMZwtZ273T3YKvn8+dW8KliFWUdcyJHNIEDnjVrl6mY52w+DGYY1VD99J8/+DCMXt356LNvo16XHhRGMk5N9MySSMaJx0PI/IwlEBlDBb1X2Kzwj119zAYENV/02h2XI2Ua5LwjcSO+KE7K6ObzqzCDHI7CCYNWRlNXjZz94bTIUczxUbxsVeCp64IHm7OTtRJiapcTacSBpUsfe5zMu94hH0bxyVOaiJXGy2qSXjrBEwcn10/BPJxd80jMIH0v+lP5fbdffoMZBSqzGnXgAyA8NTd27g0Zha3TbRWqHFxUmVagN+wKNbQPCnoRJ+zq7imDPZUSLXjBGNzV+7zNNPXln520kZdzEFdJpATOHvPH28jUuodJD7SL7j5OPVdQvbWB5DSpR/At9QMqKmyb/+oyolrL3Dv2AiLoor6P5ipehR33ULaqaDVqL7zNrfk0LokNtjSyN+ncl8c108wjdgyRdz/T2HqVQVOg6kRmJLRf2ZoYzCbjq1uATS3uH/Wz+x0mmhL0sgqeQV87lBD1qL7adPWZUPCgn8nrujqlFBRVZTX915fE2qnV03+UTrpjB3GzrwSODIBYl1ivzhRUJ5KeLkTpswsXq3QI9oJosJmi3ZAZKKoOS8G+ruX5yAcop/X8lo4VgUVYdioaWzEkgVqOkaoxHvTAuWwvZnInG9uHFrBeC54/QkP+HVFUgcsq2OynH/AY6Ev2xBSqGXV9rqxkEnmtbmsmU/wnHl2vmvCN7ZHhj3xTw7auNUAWBrO2nVO33IZRWwcOb6ymiB3NDQyINZgEun+TfSP77x3gMZ3xcL4SkY5BmsT3QM4/Wei4wFt+WG3BK581ZlxIUOIQUxyow1VKSPh1RgOhfpYgYJbFlEfP5IxXax7Bw1xRJwT1ZdqUCKydVEGHfLfQhAFcsz0nZTpGw++kg5YBJFMnGtSTfNVCKEHVkA7yo+nneK3RY91Rya/oW/UGAUI2DDlxRadaMNzYNGo+GmIMq5mFEqoTy3PsHjjN3KUKQpNVuYeTDsJIzJdqTXh+Cm2lvVI5U7CigT+kF1vE/urX8bI+qasd98VwT2mbI3Of2xxwXXFME72XaAmxig3feMLa9Zvu1IaZSRhbTJQW7945VLzh3vjortfBLbWVd8JALlqm2DmMFGDKVqBZZKxs4GK5Rw3amwV5xp26XNzeLDINdwawfs2z0IhFgg/8MKSNUzIwlPPDhQFjaob5AOk+b7uzr2rEMxP1BzcZ+sCFfrNEe/qlIQ06xlaOkOy1fKJprgLBNrKE3RhpobkTpb+VBO2ChM44NkRu7bry1ko+XBV7vfUbhu1CG6sBdsuC76KHGp6fkW0E+2vlRRTOVHVQVvHmSHtKC7o/vX2i1rX6H77qh/fpvIOA9EA9NsTY7aoRMn97f5b/UkX0ddQMa+deNVgqw29eMIZxLR0abJCBeNs04h+eaxKZS4fMW7LAe71Pw5IdNsmi2XUjyA7NmIuCIlTB8f1H5MA6FG++Ihtg+5xN+g3oR+RXzo/zWeKbTdNNB7vZmcf8NTztPZjVqeGGrxF53b7FXHRKALJeseC7m+OuTa9u0DW01h0IFmdLX3erZuXfYdIYkpStDnQor9QxdMcG++bWP0c9dnRz4pr6/4s6NlT8Qk+FuKULNv6ngJ+2KD+e+Nrr/OJk/pxqFBx6jO/XWYUfv6KmW69k5KRCYSqp9tA0zApCZcD2nTuSv/wI2PF8R4MoiHWJAnrx11voSQ+XbXge2MaWj24ayjexJAQaNrBtvP01OX2PUgW3ZhU04AMRzKyTsVDBGsNfWrHABTvSls6ct3S/wV+B6B3WNnqe7Uhu3Ohmzp59+3wiMlxR03wjlb7Z5SHDyqKA+onii3K0/FJPes4AoZAiSJB5aTMcyYg7boagzamjEMFk2GyHFjeGJn/gNEWbp9aB1GRnx/okn5KDmrQqSbKjLxFbaviVf2LSZNpB7pGdTp2f8oXa3XYgAir061vBD68+aBtak3zQsb+i5ZyW2RwAxhBVwvAhwcynwI7lhgZUcnQCrdhk38j9GxRMUCEvAgvYJAMXHmj87+waq0ZexFL3vQ6CTweXti17SqR1Qt+JnDiK4u/+oZA10Rg90UbJtD1hQAixNj/xnb79Rta4eUq4+IBJ/4+4DjMqHZLNu3bDcWynZaWkIQQFXZAVvj3Ha06Y8kj+ejxxPChHDV02ygXD8Qp1o+GNixN5IiwaoqZrtqTiRxpgueesWRJt8Gk2L1WMiYj5vGwKFGa4KPB8omdqf5gH9vP7yADFg/5IFLnTcqLPj/ba1CTeNv6twXVjNRJnH6RjAILcSbs/QaXDv1PSPMAU6KCBpkRKnX0gXqlPswAHmKpokfbcwcS5osF3PwUbwLXBA3rjaw4Te5MrXa32mUflLnTzdDhElbO7Cisow0AnbjpfITV39PS4HkiqfhmfwHGlrVK2jW2GIbBgmu4KMVujtKOrEBNKr9wSgJ2gDeNtF6JCQbmFA9ewORf2/+PWZHJDN/3Tw5APfQ++8vYHPESXJ8I63LgiVrvokwPWH8VWNYAHvfn5qQbXkEPjc2cdO2HTM/Jcllfn1kJzeTTR4sklHkBX4T6OWV3+/bX4BrskLD1Z2TOfCwpMMOU47C3jHT8nfB/DD4G/BYKk1DWDqcGMEaQpm63KI6uzxN9mpGHN+rPVOyv7LFu4TN0RjZeKKVBUNpIXHefCTd3lJOD0boa06rWHWcR6lwBTeYfxHuil8jwJ6bgFSzpRS9mR1LUaXGRe0u8OA1Dfhh9uTrh2vfpBz9A7Wv6gZW/oK27FLIqSRKVZHIK8PLlq8Qbr5oakgPd7DNIEEG6sg07H35nt20ASf6Xag3fvt5wJBZCgpBFWSUDHWdoSm9cIDQS6CawJswwhNfTogPpa0cf6DZczmoL0Am7weL0Zl0Cbcj/f8f+zu9pp2zFbQa817JlAEpzo0EjxFRUc1HZQ00gH6ad4x/kWo6dywtmTAF0Hlk7PF/+kDXKbm8iNgj38HDzRqzRFNtrqWnXIIacUs44NpASndAiyUyVoSq7pngrI5KR2yQuMjGrKaw480aG0UNhFpSD6sQNzgXqNIqRzBIZJ8XgMEGUew9va9gog5TplDYZQ9eMYLSA6kV8vQpmdqXJFYMLO+RzERzblz5DvsmiM5kVIjrZyCQ2nuQpoNzv6RK2o7gduFaGNVL589mYna4jNUUxr9KpDnhIdUP162YTK47eGSMoaluGAT88au3fRrevJRd7VMTe9VR69+8BUqy6QYbiy1OIE0DvQ/dSnUA6TrLLzv1SbWLVz41tbnhyFHN/ocBkcttEtocWesZbDMUt6w7ctoiFukHsa4bCiFWggyhBPR3AEUIzviwgtQgbkx/IvoHEo7/vH65ocAOrVDiPy1DtbqVOKFwRgYmjbillD37qL4YSJKUu0ztFzRO5DvkYJR69JcAzoWttB7LO/SlUdoB/spVe5DuLF7s991Uhgxy0HdQycqnujZPBavN8wW8CCoZlzLwtIcOTJI/txntcxmfWuenAOdnnaL1zOh0VVMEEEuYwmpp51LsB6pHwjKBTNH05PqJl8HYrAhEwpU26P7Jfi1ZccxFtvz7y8CZrGf2D+GNYzmNio8eEmr7E7amRZzVWb9lTgWsT+6zd+TEn/iGoQ0vdbDjwZ+kpq90aTtidsYmASEpgsdP0g6HlhwShNucqqp0GGkQUBTQiKb5JiqHpF9GM1leNA1/ZW0kFhlOPLUJtJFIS69IWjGeX8my+924Xcs3ycyDHcvKRMTLGrLzkSSQLJ5MV/e+5y2rQkLwgBPTCR+DylWH2TzXRd2V0wxW6VUkTiin4HdgM6fImssz5RtzS29OCzNtq1caO5/kPOo1jcLGb9IGkOlylCKVdq5EVPmHFtn1PY1JeZDZ6gZt90GrjNXmPGLVBcsjAy+GHAszUSSkCyx5hux/oK3d28+b2gx2TkIkgQZ4ZJoxV+zvWvLEUnwPwW2Ej6+s5TeZN3GolBtKYFxi1lJYEPgFnLFQMeMATDagTd7kpyQ9wTXc/m3usnbIfZFoNtA6XlRaJcabYbmIy6/omHE40qqqScu5ezn515t9PFC3ABsJ9bKYdU/IgEQqto4dMHsFr4D6Pltr3imj3xPAVBFwMNdxmP2KBPH6q3ZPXNQfWx+5Oj+ov3Sn3UasyL/pf70CFeUZkX/vjWTVcmzp+98Cf9NNrcBoAmpOVViJiONz/yPWNUhDW2ZJhFM/Ag2disRfNvTAZPj8cvf5oe4xlekSH9MC1cRwRgKEmqof6QGxd5TnAibbMMpLU1LoDylrNPJzw6O3UL4sUf/ujvEB2j3+JK/JOTwbJ944MUdLLaQed+49uLxKqqIzUNIr7jB19bTq6hCmywRdMsGr773Go0A1egmsdP/jiUTOzyJ8g6GtxveG1tfJiMC3wizEQgw4pg396ltPVqCGWgvaMKYQNg1O6uYxNdqSWAHT3xwMps3bMIOkX8gojeUArgF1dunLahHqRXGm+lV8Vc/7WFIjmnoPs3eX3F0d1FASSNLvXT6U/s+zpgE2O20RSGZmTqYT2f5U3gfBV4zGgWmXu206Gu623fRONkJxMh2z4ZqFu/pDc+88oCK9CoqEqaeA5L5Kr5DqfsGGbbAKAT5vhNIlz59Zz/FX0A7mCgnstDV/8kMXTjS/nIKwY1NFWPzqKfGYgkTsg6RiLBmvV+vOJ2rw21h0h1MAF/3h9Aw+Lo7yfhcpHro0/Tjar51tLArSF49qhPuvNFZYzlvT9qn0j3mv6dML6nkPUOiI/B0uPj7iAK+Npx0F1rzwxG+d8N3O6KDb8ZSrmuz9s3HOzPtfsRacj0L7nmgwVvRn1jcSQskgwUtmycjqN5PuatpZloAIzvig0a91H3E/QC6LAS/5dvwQQwrhvo2V2x4HPfjOuKxDAYdGDSoOlfdS8Git78GZsExZKgbHxH4d/mzVXmrIQHhpnKUneKYneMPGkLGZp+I+M0PvhqdqiK/AYsia8So+4G1UMVHEh5n1SMvcy9hPXbFhs1mzh3QD9ZJjHbJnI32IPsr0AR5GZ9qlcIoqAoiRSAbwGMy7JkfGsD7ODTA0v9rj/o31vTzp+DOPLJ4Q7pzs/rfzuxpAmW2h1m3uKAtO01vKkkdruTQpPk8fb6hnBJiizJfoXApzEzdV3orejfGZ1BYaE8TOcNJQLR1UUMA2nK3fpDEBnOhkSmuOb3zkGMD/dRO2niw3IVEZzpTrQ90Nh3H6j8B7TZb1f6Cd6EkwH2hEqS3FjVUDLcTKotfCf3NV1USIW46AHICliEXp245KHCRq/KbQAdFNEy4xiARTsUggIKx/BlxvxX97qJGht3A1CYHR7q5PGz2baPuJcQJvRMEp6z6k5hTVgH7iu2U2o/EDlF4dFetxVkBXPMejFZQCRgaUnm0emwH5cFSTlGIZ+oKwrLcV88ezkLabXPuVAiHkXoaF6fAoZWJ7AYnH0xRkMjd5pQOGlKpsmsyV9TtRNle7WowpWNcrlU86/k4xLMJwLss3QncwC55J7NID5qh/roG499ld+15Pq3QQF/CQKaSo4d+WtiiiO5QVMqvZCZ9UVm4IOI+3ycRr5tl2lHdKsldISIdvyGHzXUlIsWI8XM1XECWBHLEPUnRufy4gF5/nNcCHFVAQdqYKEvxOSuWkC+eF1gQFuy8Ng/d2SrUYwtN6haGMPx+gfBNF+2yqD00i2ZvvdiLlp4YzIE6LUFD2YeYfi9BRUI8okA6XRXMRSktLVqIOtu6huqCNZ2xVYNtL7mrD3bzmjn3OR84XUX6uczQK3upbLDjgVv+M4BysMyWzZKJUt5n8vHK3AjELqFGHoBs6nkw8xKMZlDxA+eCBWX5q8Ua5udq+RGnAdloLysCGE5OGteHqbPHjkLXWhyc6LYAEqEtbeBTO38tgb9/RZwMJL+tw7X8eVgCjOp2Ll5IXfWgnqQiKyfSOfEXP/l5aLx9w+/IeCCci+A4kOS6EKMSocELWtOzIidI4FlsUikuQ8pw+45EaL53q/vJJ5h/VblrpnShqyzDqRpCoycWcZrRuEd9MtPb7Umbb+qwkH22B+cRBNtuqKx1xIIf04LwkTxCa9MfNbIjDXA4sUkMGRJwiwypirRGEh8qpUmY3Wsu2PLzFC2SiIHNWW7f5pYn33qxKWIfoIUOoUOOlpjwB4MQA34SRutuGmg88Uzd3eKaPvmmSTzYpH5AUSqBjUnMM8mYgyUWaRCV/KO2No8G+8JWuPmTKL3XxIhfvjcL+28IjO8frbAtdhd9OQfE5Jlk6UbKbRwhHljdTHKNfbQTlimfdy2oRmOng6WJO8IbegLadv9uL9FB9jU5RJ9U06IDV8ModzXCvcOR4E2CIeW3QuSL3s4fBJfjr/SQ3XqOA2tgkHmreWpaP4f/Oact3mJH43bs1IC1IoQMEUiOYRfX79ywoFCSK+kr5FeFnPjU7PFI+4dv83lggKyuySCZFGVATadHm8PHj2YhDHcEcITLampmm0v7186KyMiP+5AJj035t7PQsq/W6Ln+01csGo/Zt6xH0/vsNv2VRC35LPHv8J1x7DJvmxcwQIZq/iqoiG522v+YAm40prMf+6cjtGvjh6Gfq3P8a+T8QSPgO69S3asn94CcHATU9Y5OqWmhdBjZKHuVaFPNoiZs1VlQzPYcreFKRAS/UaRJdOu6EIjo+o2ElXXJxS0C+02zZ2itdJ/hEhrSH3x9kTwzjdjqOZUzUVmshn4zVFjJrfwekrBjwtTg/2AYdA3xDP5s5a5pi07ZaPQobNJNFwpF6J7kmf+HVut62v/3QG2qI2ms0Ic+KSz+t6Z4xTdkOz1Hz776Ywa+PoEnB5pN3f4BKpp3XCQnolTb9Ttz7hDUxCIetmEr2WFd399WDNsm1oPlGGw4DhPGxm/adcOQIa3cx0kFcsNzcqFoVJHDzwYL+WfIDJuU70XT7mdP55Zc+JyH70oTSJpGldHdfmk7ksJY2qZXMsltmCYeSNws4WOAZUiDWn03KxDJRuKya4FfBsfwWEgsorCocdpE1YMn/xIioAL+gXQfbE8XBHgOODJ7SjdE5mU37JRXMMGFz7HlkYIaSKerFd4ZtYV7y+tZFBHeQ0iwjHSJwii/JVJflLb1jYBGNlTDUhk1ui1UuNLJAzDPveEuVPYdJpNgOnXZVdK3oQUOKPKVWu1uUr8EvQArpuoSEL36VkxQHjCAQErxAlk94jJCmIAx0iSiEbDYVwVZPLEeuHGlSZnQREArUFDDV5/mAndr8ZoMCH6EcHZGzvBRLYWDcMwEQcp1ilb5WQlFWIfhKqvcm147au0J4AUQ9c0ojszIKdNrhNKZgqCBWRwwM9rrMHune415abSBDwcz7le3mXdlsOrgqwXDdvxKrWiYKxOBIM8m1CgQ3lUGMC5lJc2hh03GqkGoKFiDSqmyJuoLVN/eupqVsHX/J+o1YLCWmWzEmZjMfRBcPB+rQ0vfJLxeJCsu058x4o2nKOcLBQkqylM4DHNVuGMk04CGtD1hl1xvYqb90QXtxVNksyi6XhVjisJxspEJKTwKQs65fv5LgwVhVJkpxzLSERYC2Y5aMQqgLqGVSjrkmCg8qVQL+R6pLa1DsLVgnkwD+M43vzUcmxS6UgQGJlfWI3n4SrAd9QiOLJ8bl9DT1HHg8GHmLA4EEhJrO/jhZzbguZmU9rkmHC7Z4bQpo62wEa7na0OThMrujroHueCUuIp/Igx/4HsbQ22la8alTKu1a6W4gLcDSSxrkjIy3FMgnpHEEicEgEd0pIgJ6Yz24VkCdGAzYFIQ0PBIK7wso2RxvA/8qgjVWfVadKdgDPGwzUIzAEhMENpkm3fnxZz/RTMEnSIjKb+9lMq5Dvg98bfLLHb5Nu/+gRWVt6/8bU+Lvjj0V9IKAzEOy5d4mOFlwg5++kPPClinJFFIpGqWwmKJqjGlLdX4NYj40Rcr2yemwjxQjozH6GX1l8DSEFarnvKEi06VOsLuMOXMBSdBRrVZJTsGy3X/dTSievHTQ6wMtRO6aU+CttESmgW55URe20DwAWPL2bd61lIqaslP55bNZl9fH6yrERaUw0yTL/+FMGGBHHpArTd8EZWrBh0Mo7YiOg6PfxE1fyth+T113gvA00y2TkDgjElNkafcNan5YZ8RxetKkgiA/5XyDhfePg2aaz6bc4PmzC8LV1DoGlc+DCBR1YcOHxsBjBOQ7VXvQmytohT6POv8ILLjyBkBjFL5iF3sr9D9RHy5vFhl2ygIjNVA5nRLfsAKqbu+0w+1l1XEZ+aWAZTNTbIgF3JuKddWTDlFMOdaLTDbmpBHR9TuorC8GB/wQVhTaYySwEnx+RyBcfbSBMX3mQpCtfggeKrtZdiX82cKtsuRcHBCFdEN5w58Vj4AixCodCA+SCy/zaWTbjYYRVjSkkmNIGpkiQ7yFS+2Idnz+we9+bbDLJxx4048mfL4b+n2ZDgeg0voi7gXm22Lhy3VuqpomRN9Lw6ZtBlUoJ+SrgsegQEwpX+degNKFz1OnGAZof0PuplVHgfAarOccyAyAksG9/ojGLWzWlr0L0oRF5fgu7tpFlJaoho5xyorwUqkjGlIrl2nNjlcTj02/S4WnBAN62RuU9u7oIsMNNSq+rK5OJmquVTdcCIlhpd9CXQ/99pqZOzIMWLiI//zkaePvxK4r4ijIN0XysMOqPWTCEGHnUbeDtuS7su+47FuQs24S2oN9UtwW43cOaym5LsQIXyUoNECzZbN1b/Azp14uFoXAWw+G9UfwY4V9MekzDPH4tQoZTmfBQEvqL8+OwpE27KWXJ7hWbaSKsuxihPwiKuFqzmr1gX5HAAzUfHlRwyu9uNYLg2mCnGnRBU+WqrKPf1uXFTP9TDfD2mu6TPBwU7M2rg7OWoCFgCIitWamjeCmVkKfEZ76U97Tw/AF9rMsz5LFRvm2ez6gqhMigPhykZyFbiRcSXE0WVXIly+rxpa2Sntm9tA16vB1jvr9+Kt1tvLobCd2Z8yf1JEwXwqK5Bx2w71ZbVv94RTECDcVmqGsI3eu7dtrDpIF1mCyJrNzk2MBVZcux8fGdNjcNd/fBKc5wR8mJzlYQ4L9aH+5YLJuOI+avEs/7V/vq1Z7EtEaaSa3aahHrDBlbG6GuR7mWcbItGIYB6pKrBBfMo0IPwPfhRYh+k6tGO3rox0Lp6bOr3DoeTq+BydBN2ecueXI3OXd4hv2PRXLlE14yrFIORHgQYZsGxRKwAg1KtHOjopu69XA/scGGiob5Qx7Zbb16I7g261M24960MtsyYlqpis5u5qPd8BUAfe1dPjM4+e7g5t3kbz/r83W7U6TXcP/LAXqRe+CkOivQFcpDpTcZCPm9dU8xXTvePuLdR/VuJQJkFHOKBuRxvMxSLNd94w2S1H98qE9YsBhS2MiDUjahrhpKo2FmoXmP6t9o15HdQpyHQYm2jkXMXD8NbQ64VZUu5C/YA/ns44f3i+IyyxlHNVTZDd0Pan6bM8t4LFRTbuOZ3oO95Cuw5BWniCD14kLq9AC1y33swsSKksJzULMWIQtcoD/24olootnyYXVoFCUkQGT68yuyr/7e/rdcILXGutdbB2AjXS1gD1JW64g/ApjaLWgTbD7o58gbqgjJ8gmbOjIbJ3wSlfoL41NzhnvyypBJngDemQ+wwMYlYG8ypspJsgDvaUsipsZ3gINu4iLqAbExKkxn5YFY/7E68iaNlXhs+2Lq9aw9ngCx7WJf1jEnUcRXG4/Xda0hiZEVCMYGXLV0qTbx90eMkSc6Vkg7ggXHYGhpDnvYWOVKu/hdvlhSnIz/Bxv8mxtK2K9ZkucaXV1+e0NApb/pWzYd2zsAqQvTTvxTA0E5e90PMXI/vqglyRfOKDcXpLGh8uncMEV6FJbGi8k/2de2UXeqiyI8vDo0qRsY/soc/0ewonCIpFzn5Oq2kaSmXpKslSZN1FZqqvmFRKFzNRr1nJqWEIjM8O8UdpuBtZyIDSJJ4/AzFS8DFYq2SSWJT0ex1wYWpgk9V9C8386emSIL/tLT2ccilzklQ9AnhZsLKqFPt1U9zdIexZJDGl4AfTjeCNhoatKpesCO7aqDeYMvVk3tnfdvtNvDLucGF3paOVHEsz5eKUfdiLbhL030mj9M3WC+IshYeUDnaVxtoOCW6cOiXtZdkOox9ZTXjy+mCBzECsiDEYHNoC8++EIVHRESXsrVJwEjmDoLvbiCVCvsQ1H9eJd9aQgvKiB0mwAn63wenqz/a9pr9XgHBR15/1sDW902ehHtpDyQwxVIJgGvuT3wzpk4q4zo6CmEJ5PB4aERnpAEIbvJby6l1VEWaTKBArfBrK2iB2mPRryh4ML2MtlxdGx/ShBe2pLkt1Q3qXUtNhvquIoLtCXTbEycfoS+vztT4TM8jeMOvwLm+VI7h558k/dMogTiBRvnTt5eWP52e03f93X4ueMDaXV6EpJbKgGFhLm1p4lHkxK+0MYx71YVIHu9NzILfPA2+3rJ7pJDy49eibtQjVhY8kERJqDMUximGoQwyAjs1ljoqBLBkEC3Hraai1GoBASaraSfp9NdHu40tAORAqYmGWWvOQ5ZV5BoawhhMpKwp4BIJLPmQ+ZIkyzIL1Fqe001pRuWmwkm77EPJJJr+P6027VbQCNOo2d+aHDbW9WamYlSShV6RkzjkI5V5WbD1id5Wuso3wCU3et0EuR/5PT0A64hc3Xj1Ys6SV8F8q6GkXUGOYwIhMdWt9gaeA1Zbw3VErO68eiFn/fnY3ORSglXocI6pbeEbWLp9U1V6InM2G9hbn148a0rt2yv97FOnV5ab53509ua+bCO+OvpzmTDqS5iRcsXWaL2HvP/YZBdN1zGH+CP1dY0Uzg5CMaXZmAGaZEsWAHns6do4moGwj7lCKd1UzRV533S4HESr9Bsg7R33DlLjTKnWjo+AjnNWzQJvjPjcsTs1h52aDRCtYiPahWCkv7MXLqVqx9pnD+yT/ptcLkmZDrLl9JugCrl0Ra/IIAaeExWlWl4p5/LrwyVxYYu7N1QpzVpTsw1bN4EEhGJB6SH/+FvFMWxmsOqgTWfiXBLj33zEOU+An0ikFVQfqzSwMnh1LlNVKCZsaWyytgtz1I7IXS5r87YcglPe5Po8kvml3MPwyxhyVkkmVB0ANXMNiDZg4ts9KW9XV1Suy6RzbecaD1cC252sI/NRnSOKDMvxIbpQDLvn1Y0W0ZkNONYiD3p0o7b/iKA3v2/ESE7X1tDQTgMafGnlxklQqM70+FH6dauAljFBeRj6yk6LOa+5u0FtUeP+A3fHdi7ytu/AvVXjNbojjj9MQg8Rg0zLXn0yCJ+lQx2/+pNsO1H3F/HJ0Bh/xroOjk+HnSl9oQf9Ic0IUMjfTcL+Rm9mOPZPiCRP2YGvA+66AVP2DceG0QGDiDisd8+6rwa3fZtHQZ/RWuhHeoXCocyWjNSa6Ob++q3Zyeiaid+DDB0+/wiehK+uI2unG6DD4wd8St1D3b5WBOjnTs4+G5Z2XbHjo81wHRdWi1zQuRHBbJ6DMDUMTdBx6p/kHDR/Ioj2DLBBAWve0udYF9j2U31lVSi4TxqvOA/9L6V//tkx2qfvtnn/yr163PFDGRyLDMCWwWvK2+G1SXeKlZnxJ4LWbkmT4arzoN+Rm8vF433yY+X7tX173BHfEGUtcpkXiD8qIOXYa7oyNxKW3hDzofr0L6KnAf/M1G2fiPYgZmFsybb9cjAZm989i+7snft6UQDXeGbDMBbwvvvgE6bXiBFi/z5hP63oadAH0cdmkQLR7pLVedJXyqKDWC/1E+K1FkexgjJxAjOazCqcfW3Uqa+GuEZGFy3ahX9E1CH/gTWl9fHHFynN6TeW+8xoEuEQn8Ty+Pa74TdO/ytKBepKMJd1jyBTkX1cHNcJw/sK4ACaCbK2BMQ5N0jmO7kfqNPu5CKw28XG7mJPb7NR6UmPBC1l0qDmG7kQcBiF5bEAR2h+wUOVpXFRWASJhIAFVoxyhGsA+40qj0aINHXsgYVRqa3Hk/C+B1hZM7tSJ1OgioYJIxG3U0S7OYU/XzsT+42XpbWE5FED1clZrVesS03Aeq75IaRuH2PbCBPHGEIUOXTUb0cGHwszgJoCQh0Dc4BVhT0xJjOKPAUNTCx01d8zL+wGJnBK+22b9znjvFKZURRG9xJsuL1bYpY1pnZz+38vmfcn+hsGZR0FZTRxKG7/IHHscuBg9Z59sZ3J0WW9BNGonTpoP63X5kpX7Ak3ISmC75MsPmDDkinEo7/2R94uTGgpiCAt/W9vn2fLE6xUEWvCtYUJki1yqYi5DMOPNaa4IQd0RpZN8l5aiGfqxtfTI/+5kekRIR9OuPbM6ZlkVYpFMO7fnx05YHToteoAvv64HikXMQQdjYikuFsYL9OaWLuOqUDUgmF12KBPzDP3QEh5vdxL+VWPMeq88GLLZLGK5i2DZvvDfC93C5bx2a7VjLqdYT+53pRBDGj6u6TQIy/adrCeOMhLsV8Z3uZM1wbivkfVNkz2qGaILuw3D289EIbi9MAbX1ulmIhdo9T2DgbDE+0byfHJkP5GrY/3255gYLzyAmUIWLvRyeFuvgWLDCdtuWbL7zWGbSvJHujZfHzFRu0o2tHinj1z0lqpTwvVcgx2Uzk9PD2bhu2wg/tHuEEqDQHHS9V878ALGuOinJSfWINbk10kCQ1esA8p6GMYNsc7w3KE5CbSUpxx0dWI1zO+u3rRJNBgwMWgqUUCn9SSuJC8aLGR+PPREHSXEuc7oJu/g3N9sRzDzz5OBbxbmE6wUWYLqVsf/wfi9feuv3+cCx60dpcC0s3NJdGO8li37vn4mfq0Gk5jqHP/S/JZUMN1ERiGu97LjlME+Y3j3EyB6RYL0yPO9L5ELx3j/33VUMA/JcqodCi4ITDu2IhgMKtK2pENjGm36nvI7GgGynXHqdk4ev1GyZMH4kzGXvVXoFGvaJ/FfjmDsyRU+c3aWi0D7PolT3F4WbP/djOvIZtJmIPXbpQpIaDSDDN2FfVFWYKvq0oUtYxP6pN59sLqcbFcsUjeD7pTrmw8upC1ZDEDVkIxVrVKARmsZUBe2dJQd4qV7c8DVgg3sRTjVaogkkk8g3dl+z11pQPViLCjxDL1EaLOw+d1CEitgkgC5jfc99kWqoncsyUllbsB7c+Isz9T077xmv+7Y7z7QJHspsZODXfWNgbf8wwct3bw2ZHdkfbSsw/Zor8OBSxhn5fzs+UJZwBMp39fBEXbuBE/5aCdby6PW5GFrt62gWukPY9/U3UT9XBBFlnuKWtTut+5AgDZ75+UXxzgsQRQNK7A2nRvq7ELlIHk1gNPJDzABS7qapynr04x9ni2s6E3DMACdeKCDw39hQBWJPesiOZA7LAHXyMM2sPWTKK30Zux21214OORSee+TD/o6Xozgni5oOVblyregnuwETaRpNJht8Mg0OJhwNcXooSp5/sQeeMFrMUgustU4YLCdUGuS3fHe3rpm3FflRgHtYvYJL7iCgUHcwcjXZWoY7JkI4YWDALKsjIbnQY7BhMFQliIqUpsG88JWYN2fX/9QPhLv4zzqszI/hObgFykB4BiE7CyRBTaJ/XAxo2vZklfRwugpVZzKKgZet3dzdQfDPszibhrlC+JreqfcO+L9MOevZ4KhsglbAXW3Suhsq94HMRMVnYA6e0w8klqA9Zblbn+Fpr2p0U3HY3RM3mm0UvN9UkU2bN6YL6YRMPC0aijZ3aHhbqhRJ4O83iXEUNfdX33z+IrXyYdXC2PpgvmM/StXbFMJ+OjE4RPO1RGQDZN2jR/AAqtIIVAego/6o5hkHqvFyoz0Djrc+gbg6QUhLrXvmThUfRivwRujzRFrLaLu6N1KNpt443Y6t5PBw5mx7HdngXnGFcj8I4DdcHZRDI+41XcwLwpfMr4UwNYZfPhta3ge/T6ynC3Us7JpYqUKwUuwTB+5uNXqEJ73LnJ46d2DjYOAH7So+iKISuwfCUjDQoxOnJ/vDnLnUbmsTxIbWzYr1aMmqSPZHMX747Ga1V8Nr+wmF0D6IpbJhHBoCx6qGVSK4tnZEkRFNE6snsiB4BNKIXjEtCZ5i1EXPjkdOnMLOpUgOFua3JA3HUyhv0018Pmb1ryZ6Kdui8qIlPAmZI8DEKgv1TGRGD95mQ6G4C54M+RMZNcAyQosB2xjFsreJJ1ZdIBLQinAfQAFkq76IkanpqgkXIiXDRD9cnDGgxcjoSCcOpOn1VaJiE4wEfcMSfDOM8lh9rEuKU7gRV0EIclzMdADS4c/4O7ggtcprbdbQFwchPlvBL/lX4e1Wh+Gy3qMoIxTKz2Hs/n83lsDFbRhKsgw0qN0Glqi1vnswBENOzGYHXw87H5yVzCVyRKDfAcvKBcopyuoIUx4LP6q7/lG5XTQHZCf8Pc4RuXqjZulk7uPOmYbIdInuwumE5zoHuRPVdfinX4r5/bgiuxdjeuJlyETtxx1H0tVK9GwT5zRlboStDKGAXGDnv5G6fYdHZVam08b5vLobyxwnPCjap+/ZiW1MsjODwKOOZuNg819NsjpKALNqy0fTJ+YBLYiEqoiu76hsO3OfRXn6fqr+UyObduZZRS43PjabbSehkjxHF66cMIHG/+p3sZF3M7s8Ct0OU4buCBqHfttN4tNZyvCuv+ql6cMbVFw4YaNW9W0XCHWsMtdcadNVbAN2o0tIfZdeRcQHL8MIFKr9wRV7r0g4rURaVDYnt+BES+Qoqwbq3Ro1C+ab2wIR39jNOUFTvRb8ghG2hG2omYYtcIZ4VOefFo78ZybihWB7vebxr+38zv5LZokeQKrgx05U3bx5Cx4zX6Igvh0CvZyQMo3zRS+BdByrjgBKyAZ1d0jZwSlDqGkQL70nwOCPUC12BEIbkZsyfXCC8pMlEWEOKOZ8j7fTK0V3ZqRRm+yyb65rih8w0ByM2PPtJj0UNT/I+EyampWLt9Z+64PNv12H+J3aoP1n/Ux7159ghUJOmEJ/wx2s3CWgBe1Z57K1ahSyEN2ZXKyxffWtQywaJ6WVkENfrp1RLUZj+otKAzoBkTxNYtjXjhTQD6cAx8gZLq6DTIUKEtLRKa+YB9pJ5rTWkFGVVTUh04pg6tGkrxLdymwo3mZpIDLAjxoxjgDEIsBBQrT53QfSMT5EEls+UeX5LqBA1KezsKCgCLzcA4E1jFRtzbJd0JDrAR3p5Mb4OlbsywcfOKK8B6Hb8Qoj26zVwN4oUru321Tg2WjGmfKGOntwWLCbVVl5wFAMfxqeDRGoU0AfkpJTWQ3jNlLwX6fXtphNAJMpqmoFlQpYqcCgoLKEVpbdvP2gUA+tXGdx6YHPr3W7Fo5/kLH91f0/JqvP60kz7hmLm006Frda1lxFIzF/KPddW+tG6kUhcHFlQoA2VbKMqzMIP018GJssDNp1QFUPmqDTBHD/THCdtNxif/dJrYZev4tW2EQpYvnkTuhLs+S2HipwGtBMtR7hZnkLoliAz+bJEmaHSiRmqylydqPp/rap2y5a/FY+UNbbjdhmdOvPZsEgtPzOB8okWm8/BvXzCw6VKRFtYvweFr/EmMFGQ7wuh4Yc+ELCvLAvLFYoKnuXVlQ85oBtViG9DFRlmTy8sJ1lRkZ7SEa+KHTprLqOdNycR6t26VC2MuKkBHtcdKGK6bdy82g3HUO+WHnuCOlh+TTmarlhYjACIvD0mIxj2Wpn9LzC+3GuWmoLVz7e3aiIjeUh2NT/RNFjxJn43vZUI5K7LYQXAjYbZaZH95tlwMxQa6ztPgOuJHB/slJEOJUEKXWVVuEpOVq8rriszvN38/a+vtN71ev92ZqFs+kFIBiWBxUqg00EKsUU8K7enFq+6USD73TbxPRoi6JDE0vchXY3v5dNBxejSiPC2yrKRRy0imja2kwVuwwA3wkAB7NPZNJJ+9OWzj25PoIP0h9v/zqh9PU1Yx7bTJGlYTI5DinC+ceK/3h2I1QytTSrzYt4sGHdnWY0GTMTkv3yhGVeCisMQ/3RpzoOvOSbs9kW5Z8NOraO63ZxyWuyV2Rm5x3tsy03AXVStDfBOy0FmVOSGbXnTrFil25n5nf5Edp+wK53zNxWLfq5vKrpKdyMEcpqmvAjNrPHnXvddVKipSjNTkW7JMfWFzE+qbUWLkJee549q6caf++mRnD2qb0uiV4Er+iviSW6JcLcsYWHhRQzyNZzURpUYPJd8Kljiv8Z8AeBa0CYDacGjlhmvIqzlupw1YK4kqLIsA40wfIQ9cSjEoIAGmlgF1svijIy/J0gjUwv88Nf0fG1kkg86Nyw34CsN/ZlovmwgRZR9BiFauwNQHRND4MERV5HsKzClrFEvZO6q1IQITyI20rmAb6QCax7S9pZ2PA+caFiP/Jsqx8bsPXT/xWI6cUhWdmNCO8p6GkMFwxyjTZ0QuzZy7bjkfrQYe4I9RP0C1Pa5vBNidOefoIQ8I1Ju5MZwJmYMAcX00whGACaoyIDaVhGR0hKx96uWYyoY9jggsMNXrCgMMNSzAoEHbnbPkgbKz7EKGwyS2y5CfRE8VUhHWNXxZru8KtIb2Ia17a2sIsRBbjFkQVkE4YIdYXtiIQYRhiR/zXrQzDbggph9k+3Bi3OoYSCxzjZIzJB07ou+WGpNlPW7E4xFPu5hxV1IXqht3+2A2uZsg4Ux+42gXlD19mm1WWhdq4Te5208t9thNe/UnsaVKD7yAfR2/3DIkP8AT+C4CwOnk7Yv7ejCwrb12wbIs9GoiTN92R6tWuSMAHlSZ7qjNMNMk1PqghkbYZUMQI+QBfIGfU4BiEt1m6sGAYeu58FNaFUQpp3yNqSFkBTZpurLHpi1xSLK9TRjgaXbZcTDE5FMM9oacJkp7ct3ONbD9Nd2JbFK1mYgF3HduTla6RsdkvosW9G4fKbCqqI+f6GsgxTmU5jmpKgEin9cRwHSraJdE36319KcAbf/t6hGhbb3TR/1veiOSUJ9UPtpI72Uc39dmb9TZgCSEKSDk76uWSt0qVF/Xf6n90nZCTsu33nCKQhEgcrx+grdNN53PyhAOeBvVkbVKjulyrdz9Ue2UdlZmNUR+lymBOYzxe/W6F7H3FThYDuUWrHzOpatPpyxSUno1FF/kC25FGdmHQtsUyHG3WDN7tNF/WdCDgGe/MIKtgWYrRhFHTSdse77Wyk3xkGqVai9BrDVEHZPhpJAd9jTAvvK7csBeY5sUFiOSIGX8BULOP+sbVQOueYy8qfcX5+ClRnbhp6QfKsn6Eg9iI6CXFVZj2rrsAl7B4nRI04RyaKjdaKCyL1QbbijvULUCommHa8PEMxrXgK18st5fmmAIBZgR9kyvpTXagUk6t29n5o6dnPJK0rYrIJHI3ZfZkakpMJU4ttfycbM88p3TiZE2960LtNuJAUWx7ZnFFdltH0vD6TLj4PI5uSgQJqQ4WUDqBt1ULZDWOlYQRz07EA68h1/3IP+x+e1bFdYpvylZuClEDESVTakgEWkDcnDdiqgcrKkTQCzBvt/Aef//ji5GemouQxrRmZwxmUJqhMcLS3lqMC5sIXZoSjHCUIdXGKrDJZM/LsYEMlinyJ8XP+VVPe38SNdoumta88Sf9G/nHx0LgGnjNr6/Q89rMmwXyVcoYfWcek5V4GVyrYoHBHjOXBSJnQy6P6lhbzBT7R7NiqyxmFVBTxffRoUI/vOq1e5RbQlAOy1nIPqubwZsbnzzp1f0NedRef6W0r3nWpaNhMTfMR60p13drjDbYVsE3+1d6zy3U2+QaiJgl0S7pjUiKV6wXlEI9JOeLnYYPUBtv2EKJZ6Nv7Ss3ek/rNhVceglYtDogYb2/804onJFZ0AC3wSsitctB8TSTsmDJWeeSSavF6nJ4N4iWUzEse2tRiwnFRVG44kCm+PLALjTS0jkRb7xei3C72uEoLRLazYY4VJbgsINx37xVTDD0MU/g6rxWGTLStiRijgdWvs3k5oywsNtE/JOW4JtDvzAwh149UrOOEFnIhvbo03Q7lr+lOPoKfI6T9ohwyXCABN8vvHKtVVYpti9UUSsbN1vbRndSYJDIh6cNXir6mFfb6hKCCwuKBvswLRyQK8FNOMzRT5YAZH0GIMtdrbIlAb2JnU4Fq87UUujZV4a5ZQjAlyzpzcYWOaTnEudqG+1Qp6W0cbL14i1eArPE4XXYyw24GP5gUK8gr63n4/B4e7EH6Q8rrJt47G79BHwG5xqqMZW21aaOK5GOT92jH2vcTqmGQO14OU0/o1/2nkhPsgNlIergw10NiQ7E/rLrTAyNj8ClVVgPZPM9FzwxpbhtP4nPnHp+LefRWpJWTPYt+fuOD1swL+4zENhnBki/ABiGLUoXqM4gQHD/NTW3O/PX4CBBMNqgWwvoI8Z4+3mg+6zIAw+JurEsBasAeKjiMUa0hhacv/dF9NJho2C1cLak6FWKEmmCSEsTPzyJ26PHsnBKPglemnPOwNE+jnggX7ftqKnfLT+WN2NdMXSIhoiGw6lIXcO4uRErCXQIg6IEoHhMuTLx/0XEzi7a5VWH6KXyS4c2geSUzSXxEiW+2JrD/dC1ClALo1rnkivvgpH7/zttW024GC3sfTF6w6TIzlo8GEZOTVRplrv4juxnfG+BKLzvzQWCspyVAVRo801YKpHj2WU3zmmVcSpfeagDFfnCNw6WtBU2ZiDln5RVW5nkVRSmu8oosif/oagLDsFcd20S7DgNDrAhRzhm/+nHVV3kTYFq1T93RLafkh5FExg+uHj7i3IxloEFmf4iW4j6DGpBols83q147QljyXoBej/A+Yf/L4D8kY+HuXz8P6rNCA9Ixu8CyfHf8iAGmXnAXL2pLB5yG+DKeViA4NCka+U0bAmL2lLwlCH+5BEZ8t8xaOvFl9tbaksxIfWfdEOeclNdLEvtOZPkdTixwFLJ9O/0L6Y2TB4XcprMCBv4zowNr4PC5GRMq72W6EPeIAYNaeAvwdoTICaDcaT40H0lMB1C0itkwRVWFsULtA1aV7ZDK/JCadU5bkay++94OOD3TbBnF0iHPgPfTSEXU6tAa9T/zpFzmd/+WxvLorQrajjH+lbdd4rls3/5ILKgKUi5xv+Q8sJrsylLf+q95sIXZvSYaev2cRWXP1bZlOrUacXM20zKzMgbNQHdsfYbkGuHO0E6zgYqzZgRsyHPOCz36MUx0vSCZxxEZiCgW1D0aGIFGXWBxduLYHlZy/+GuiNbgDxjffZWsuyI0Juu8YmaLzzeRPvjGibD5xylj7uqJhqfxj5+aXt9XkWkaNiidMaH7Umx19RWk+YRBu+qfRBOKlD3LDf/sHvTgFypDmOFMWipMxtj4eAI+SsBY29o548OY1vyx7dHnkNdgEJXLWurboBfaGdgM/D0E8CTdgHFlRshyRiAeiJ6s5Gk+9fuOk4BQ71Dsm/RUqqp3RzQHAFR781fHHJAProluJ01L9GQjThFqjMRj0Gl8I5sriTNEEq5RoOdCNz52JUzbKepdc90c6CkDEKlZGdbv3QB6UxZwOB/pztGuCdy67SPGvO7+V2g2CW4GIgOEc6A8bgY4gOP0E69U/6dfTOL23BM/EHCwq09PPrkr28afKOjk9AkVmUWwUIBY5aQVlTTZCkf3j/9Mz1EFr4UAJY9w9QPlkiFlw3NTNPjfJ65M7babeTsOlQ0EghARXkqYAM5eN7MLdaaTQFp+h/1O8ikbwQpLaRy9T57/MucJPdsnSj2s/ZOMwX7j0sorNI7cPXZQtJp/669vJjdAq/ucdDjUaWK2ik9ejevU2hX0eLLgCPwczWmixb7chdrSRgb/HD4+cr2Ejuyxa18VuaYA1564Pzh1x0XVX+817aMSNt+bXs5fQp8o7X1R4Pv/ysqBawYk7FMoNlo2tWftYbPwCkeGllowlRkDmf33pi2xFqSeH2qId5LV5Yd7OqS1ruaEk8E/c4VQ+m3ZaloUn/r166+u38DTnu+34QEQDalqrD5Ri+AbuzVe7mEjtj5XLaFWgn1bSabGP3j/6bBfb8+cNbSc50PHRUKyzbqJWx71znB2oOyyQA4/Ab67mDrRYCg8oGuxJ4KwuyTnSC5JKu1KqthdFKOSIUSwTITEVmYDaP61RRhF6tujLowUp2CAoVuM4923Vt1s43jFQt2XL/Pn/JmlJFLMsxBTotd8oJNIO5VrXMPz69/SEDjWQW9KJm5XT9E5RamQIkupgfkbs3D9dCsbfXRPExf617BXtz8IEwgR99c/3xmy9vzGF3MSa3qUbSkHkBSAZATrbNkOUjqoEgJj8OoaPPPL1VmN1A8R9zA29dX0QXek6pa+nvhpPJSgE+b0Fv6Tb5t8gQ6RpQRyzoiAhrP2kDtvutvZksVg1VESoYBNzdVsoUbAMrZZcSj//5aCOuT5dsWA5Gna2bBvvE2B0GEafbru2/Pel8Yt3uw0KsdkVFTwqwlCHjP1T0Y5SFXMnzPVrwMllg+cG8k4v+Wf8rSlGA7LPhLWoNkZxThcFRa5MoKmVchS/IyNNO7xdedCSsGoIC2pLeUEpk46de5tXDc5cnWQNud8gWSC+OHJDPf2np3KLZK0pUP/IjiAAepl9HR7BToBZ1GtBU0PnAsD55/TU9fOt6nWHVjJYfSv1Bl3rxYPSd/J0eaFmheb8686qOXwi0rIf6Wlb36tIV7Wp81Qa/D42KCPkrrdNxMzuE3cscEWbHu9W1U/zMUXWGRUZgVNSCPke3Z1j2jsYttkYwfuzFtAkLfbEE/5xuYmeCF7Y8GtKHpzvVgyz0jV7n1Rjv634HmFT+ZmU+i+wOgYnohD6untiKE8RgwfPM3Rysr7MEAoWmPUlouQpKxoWam/0HWPPwUrO6k4ski2u2MgQ53BP/3nq4qtdO04x/2NKQc58fWelcigEuF0LgmXPzwcm8Hq9W+k4HluLF8wP0v0Bzsm4iS2fVwotK51J118StnXUbmHl5GrRFjcEP4G4Ps+4vZx6vzml1mbZk71gPGDcaSvhjNGm9FJBQrUnFKnpYMfMLdlvjJbmBXfb68eZc7O2Wpvz+7nPTV0pfX80FTshOT83H5/95TV8jZM7czg7VNVRiCBmMYT6+MezCVuAJWNA4L//ZrUVXAetTvjSjRmMFZQPFKZ39iCTj3K8GrMYfhwEceKmILNuxf/C3VlAhWSRPlHwjhIpXli1UOtxaW1906P89a1NxDqUYQ8Lsk12c4f3MlCv+waN9Zbb81RRlm7im3ACdZu/TNXj0kPojXYA75G5p+6BeuLmPwSLr88OTugvssrMcvZCHxB1/pKz2R+zxaaybGzk//tdomHXt+rdwmvv+P3+Vnb1aSdrYL+z5oWuP2cCrtlpNb5pk/OSp2IwHfFyHVlGxrR22aoAHWxF6lohgnXfrCasPu05K8298IKL4OfWIuwWy1KonQk88ICuoWVfaZnJA6P7y/gEbDJEYn/pd2VP08u8k25kNFpjlU0zgioUOC71dZbKPBpfrTyQX7ZGjebE+/bGvP4XM+WsjYrHI18k2zviWkPZtVcxU0y/nlUbPpH18gl3tVMWJxvBq0CmnFPca2oAqDTXk4pyP35LebIRcLSOSFVrx/m5SMXiU83mMw7Xy/VK7CmxH+rpz5oZxoqg+zmOwInwspBirGPdTbWjQhhaJccHlj603yWDFRVHFHAzEnCvnkofTFhfkG+W4Aq6zZ/MF3o4srH9xsw1t5AshS/mluN/Qh7rUxEklWfZqJ8Q7+bCtTUCqSjjoRX1VQIEnpkK2n90ojsrAblC5C1ndX/ozFFl5cuf03NiTBcu698dVpprhihgwesqXnsFGYWT89Toa1tyWD+6/WNFKdopwUaJVjAuNYnulA8YO+zsbzKS6Z/TYxcKGPtQ2tHW9ZY289RL5UVD2WyJGTgiVSqvMVLnabLn/ulJU7A+Lx5D06C8rxoafMpmQp9nIEGioV6wmujM4pqaJazHleMM85zLXEoQK8ZhiAEFScJ2J4Yt+9KtxOCNytILL5SekSw+3YyxhQNZmBAhS6xV/iBLiFFTrtPonF0aQ2B3xW5dRH7bxv5eMnPxnrL1uZYUCFrLY57kGNFSMKg0FskZNYmntbOJJifQNv81ehs1NsfcltJ+87MXznFO49MAHkUxTbEBngKgGWtTHxjoGXELGKTLPlQEbD7sS/SbuJm4bUfrZE02bC2unrTC2LMQJ4IhyPnRAvOhVosQQI2qhXG7o6PH7S8kpcyulmMzwtShfrZUwjhDWykH3uZfAtf+dBnIfPXvNjcgrTmv0z+svo/aCfgqv/mflVzLytHi0xcjZttqGhVGR81Atvqlv9UDTSrn9geOKzaEgy1akEoj2tH0cdnTt37yOcr4bQ1o03pF2R9JJLtnZC11h2nimjUXNHFm/RK+iSNkV40sKO9ymLLmz+F7qYB//YM+ASqmvKcS/fV4NqOJfBxO76T+6qSlXdjZMhr76PV/+QNC6jpIAEwqfVUD+uXULDvOGNdkdxLi3upkbVCVWVmp6bSTsJYe0MFssy+RWZEMDAFrxoYHHNXiKL1KHqMA7qo12XOzRXsP8jL3lVdZeQDgFWjwx1elLh4CoxRYiwgduhJ6KCaHCPwChQt3H79W3H3vttEuTn6iRB04Yowk1kMQ/whiufeWca8AuAcFk+APSJSh3ZklOVhc8G1y0dE9YotGEEdiieCaaCuUOLdiu3ViQcnkAe1cLyLGe0Q17Azq9shCiunZMhl/UC6JQYzRZ9PHCkZX5BtC6AFN/bKledOajOgSDBAfMer5s1/xaXE7NWPdt1RtPbm/0N4AC+cKKS/N8VV5VdIyiSYYB0MDOgPbQkRTD+q4fLorRHYHd3uSGzIzat0tcWH1cijVXU37RtTgfxdW5XVMprm02tlNdU03FBgJcPHVvynnn8OjB3ykIzO2XFIr0WBv/itvObahnSRcGHygbyhbIJR7sZpWR6f/2pUk8utNheU19MQdtRKyShBW2d6ffv3iuY2R2DjstVnIIy1PGgB2/AbGgeuVXmbf4uLmCVIEUUebtqfmjYv1ZnHOLuAurNUmH0ulwOVg8TZr+HkWwS1u92gF3XDE86kPszOmmNIKvvW6/9hKbu/au9a2YsUxeTkYYBsXV5Rp6ruqaeQ3e2TgYXkQC55hJVa1lKJGnTAltQBwSapXw9cGWoqUK6KFasr/XCJRDX+GV0web27g7XUjFG/EkxhjTLsVlUTxvgnxKG7BxsFaK0ZJ2VB4+6BTIN6rzvXmjsa5KVt3sGRdQHUN1mhdiNHa4osTzXQLM23TCa9ugvbHDUobBmJ53cCMdjK3QVCt4i7JVT1AFUBR+7X9fZzs/qpg41uGSS6uSyIpo11cyIcaaR6cZTTBFrd/veh1n7PfsUQKWp776Sh4b84QmRfZws794a/OqTGlUUF9Zaa1kb/xxMhRsWD1n01oDeXEx4eH+yGLGQnmU12s67mUObZ7dXNjaO2G4qMfozg7TELeeVpGH44vr2gYIKMe5LpwoRa0RCtslDkPQ3d+UqyzMQP7iU8uaVqcWy4Mz5uCUmXgMCAmMGZCaAhQKZuGRKVnWNaYwstZPg6+9dIxuLhf4hs0L9VUMxH0ucwHqHtb6SEPyb5mbej8T6OhbhTAg+tz8IbhWMpPqkbckOZx56ue/b08glmWyFYJolGUvP5gFmJFP4Mo5tni09XiWFGe/bh55ZVCXQbc4SvddgbCBWhdxpWF/qfNm8AeZmHXuCTTx7rpNOi/pSQv05IV0n02kIse113p7w9dNtC56ajV+/jkI8kjRC5KTRQ9+Yok1k7iOnifDlXmIdWsd55Nt7Mx3L+H2i+7O7IjHvRE7TNjVsJbQkToyJ/IYICLIQpEQN2oKBHkPe1dJ7K6psnPHroNrxxttU8vM+wo/Kjvlq1cumwwYCuevW7BaR9wI0SDnbsA9B2xMnuywP38HwkR5QSjU2uNC3xfmx1bQSe43zbWpGCFFlWJ9Wl0mUsYN2Qm2yfG2qwrhtPeZL5GgC+kY7hpB2w1cNh2Z4nKe6EE7qDdsBKLkABtmeHy7N/rhxyr1byNcDlylzfpVWoOzVG2welVFr0BEF1/6ZCPmOieAIOnev1wZvAEJfKiDTQlG5ETQe8J/xE7Z/Q8Wwm5j6Mp9kMFaDMulhOsibYq6Q14FuscUrcWUrEp6HnJs9A1UDE2BY1i8lqAMKctMHPY6MPezU36SZmp/Md+hMTZIGhjEMl41hW6/9859o4gte/kIyST8UdLcur9YrJcwYUpRTATkWMO69vWv3bP7eABKtXqCLxSgrqjUf8ttDFhkmAc55zngdPU1Ns0lBfObbxvN2iDXJTBgBSacZ2GDqlURYMQMnlbwnCI+k6pyRczJ1RsLL0DDfhCJsRTJwCIWiOVEykg8JtF6wLOsyKSQRXS0ORgSLX34rrwXskqXAg+DypZcp2roVnwGKrjbYchxfHUCk0+GxxsdxBquLXeXTw+5MEfkLX0jeMLr1HfPKy0gyy+5KsyxarygByDim094qFL2ekVU6xgTe64MWhBCFOg7IsZNMuuqWVUHMt+Hcx5cN3lb2ei4t795OABfTtHW+K68mc748oj09RSbfGHV1d2XrN7pF5fjiPDolJawXealsE4aKhOlg4/K2f/syLbOAL+dCD3xK2L7m/juw+5F5zdl2wu4eKLK06LeX6YoXiKDKsB7gftBZexTejoCdt+1Wn3CcqfxwS3GOH7stTfAxaLs5h42K92tzKbzyPk7652FwWnlajqAH2HaWCiryKDdodmr9kqyVF0B0G5kJlxMSldc8tb70BPlCUGqhezvTD3pSywUcjkA/JSFvUfUzQLJlGtYiMkLSfMT3mr2ArLtlIq5g8GneoiZtzv3wkUAArSmL5V1rj0ty1zYorv0P1Q9SzCTvU45PYgw380+euDRr7fwa/pMWgIdQXz3TqfTtlgw0guj4NG5Uklcc2r/W1aoAHpaGDYY8KmVNwtyUqeMMKiJ1wIJykWSHKNjleacvahOPwVTu3Wg6rPh9hqTzi0LOaoVGIzX4j/2FHCzyeIvms4Qv/YWo16pTQnXAP5j1WRx/cuQfXU6gsvXRfvvvhpvTP56berAAiTgVU/pwUPZ27lSHpAQTZOvAJbftlN+r2Rn3q+GRGsy/E2Cs/RYOErfzcdbxlcxu5i327BTETKNKCiF7GsN357iTir3VjEkSQam2fd2Sp85V6Z3sGnhnOHBxy3mzzhLsi3asvVLX58P5d1y0J2GnaTD0BfU9+SkFKCv8i6wJPGniB9b1mlwHTRY8GFLXK99mZ7OvYOt80y/08n6btB2fPfrRXNvpzfm/tnQUvs0sEBaBhZiQF2QyqOm+IRTiAD+Cu3NxsRHT959vkUn3c0FWP/i2X3MIco0nQAY45HShBWyL6y9gT3Y98pqG872626I7u5ZOTP9Q4wLUyP1ltdNv6M05TkY1u4ZFMa4HRQ59pjJIgVy2dRgufJzbSTKvMtBjpKpLhSW6gWVWDYoG2hxwYURo97WUcaUkCbL39HaessAyIVEHEsZE8FbL6qno4zfwIhlI5zBgJheJlhCX3lzC+Avah7pRJeQ/YVu0X+IMb70pMWkmAJj3dnaO197jw+M43hjkykoApog3ZbuBbkFJLawtpDfpMhD20CYqIw00S1kWyBBm/1eYQATQEIzlZPy9/eanv+qCuQ1zsH88FqmQZwlFEbAmFLKA36/1ELO2N3hXVKj4/YaEchNNtd3GPDqj6/nvhfQOWW3w3BfJNasuhTQd+0NoGZUy2TK/IEoIXtI69mAyI6Q2FcEQ6ZMwbGqSmlJBWSqHtimYth32fXb2HsI80rC6Zih3b5APidjVIzErw/ZcraQ7cwcoPVP1mCWoWoFWQ1CW13mn4mz6iHdZ+VEgDUWTpDJiJmBDlY9KED3x41nitnEwizO8rnAPdKtmDkhigzwTb07pXjr9i/di7fZzS0NTFbFLqyO2oeb6Lmh9yoNuPZy0XuMWvvCg+0bwIVHIsvoQsUKaKVx54rkLWPcB/DEFwwy9YnxMmKRqjQBZWnJF2AoqSzHAmjuWGmTsHFg/iczfWvkz37Hv59rsr7uYEWpAlQ/yHbgJfPuGGWeedHt7YNA5XdfiMJW2QzaqDHC/+/og/QjgA8HadYEv++vvpjJTlXPI5tnJ+v/ONuTdBQqgeNwJnF4o4cIHI956Av5qoQ+BO2oLaP41NE/bpMt1uLrwYz6m0iUOW53MzNprBzH8c9ufOUzszVZTCzdnchjJmnFp6BAKQkOekrN1ym4Uv6X1wfYpcHmaNQA1++JgFbqOi04je+ULmVsbW6NpoVsAG2bWLsICRCKndqPUNzUSZIWadaiA14yP+q6G9fQdfLsKaM9u7miWsyhJxHuSWXNKHk4g77SXmDuONv2AlYJJHtqn8YGw0/kGyZRCw5A05G1AE69l3pIagOBG8dNqNUrcgVVjsYb+wPwjT8pYY/ZsGIqY8/5pk5n/KLQk0WpNgMYVGSKVeqU2n/6RR07E7WGcvDgnrdPo8/9QTMzG69TmD48omvIuMmQ3fKnfqGRmx/5bSxZI+s9xP2tJ+b8W+zoKIHVmpeOMfeEmLsf/ROElQyy9GP6HfUR9vKu67ILa8f3O6jsWUw5qNcTojax25gywLBau5FDUiBaBUiS30X5seUhC+uq8ZqJOrqG+iQBlhGEDvy3XSTp4Z3EBTOrKY6AMHYWauCplufK8iF96EKKGIyrNyKR0hP2Akh0mfrPPeZKfx5WWVeXURYoi1z6PwHs1AclYFoOLr7qzuF/AJcyS8BSrN1aPnk57bYIG/2ILVzFLNQhXmeRyXHXstrWhRk3Tjrf9kXmRuB3HeA4P/LER/5hLkg73U5/pl/r+TEAIEdUwufgBmfpl8ja8rc7Q2uNSJu2wjrRpi83n3CiXouGZ6i9yrthJzgX+tZ3K/tK9INfwRVJbjuRQJYyCYgH2Yw0m1hsoAshASTOBxcjGH31B+2Z8jZ+hAHvCrC/+R+WUjsq6p3ffiA8QuKuUstWc3Fu7uLMyNCbm/890m8pc+i7u5cufXKd9YLcmqdXv7KGXyvZ/dQ3jrT/Zjr4Jzw69SoFqPw2wrVffRBXBF3Cgr3Sa0wIsEek9Q3GoEY4YsNBT3Kh4afERGeQ+OMXluk9oATrXvUY8IXVhIuY0teUq1rNKbCOztWuCiE8s2mkbpVGrkm+8in/3jP/aFlbOLegsSzubGobjfQwkXqg5Te5j0mXF6eD31gEv/ff/QHnoa9JrZsmV6wVH9DOY0ZuK5jST5e6NC/VgMuz0g2g+CAhy7GODuRfJl17olFbY4ADoUcDAI78QNiadb1+Ybn/dvhZ6TXI36M/BP+h73zjOUsRW+rBZz1M5SA4+t8GpfLXachaVDANamqE3Cq04Ti+pbDL2vLPibaCe8OY9z0AKlJ9lkYjs1wLy/+luOEYnGw+fB0Rs4Oi096XP7munwmq7juDtc7fAuPVB5HbPrVzsnN/LXDAfav3umNGgXKoWZqhb03WBsLtG99dOdwkglMtti2h9oEpi8qbFtxuOP951DmHXQxnYp066gKEjwAa7gIu5NzCHiiZrd4YuucELC2jwq5f3r4wBD57CzBD/jZugcXc3q37XUffpR9FK7kXvnff70LB3eMs6rIfCOnBD43LU8oXxUX0LKDXR7NyXmJ5RhH4jGqLG5gokzZh6o6U9Pv8cM9oFV47MIlvYnXkJ5TyAnSfNwhNsEKAwItYhEHUdhzECR+xgIkmDFQEnoo5jQGSpTg/QQY47++TnNigtmqHKhDUJzmjbBFDejIEb5nB8uIk6Bw1THM0mLHmxyv+e5okeKPcgq7JanoWXg8/KNaxL8ZutBkPGwkilCio0fugFGIyak96Li0W60jVQeHoJ3ckD61eVLHmYg8qLiYvT0mWRzWuiO6Tu/Lzpz9rfPA183vbuTB7pvH6Bqa13jr/s9jIfJYJYbpkTuGK7IROd+USN9A2/WFpX3wsqJWffTJc8F8dSzG+ZWrjVhzE7JuIpUIfTvcZl9rKtQ6UpOsD9XIRUnFavsZ+spNp5ytCTQAa3nBSb2gS+cIW0MDjrbflvdsgbXhppa+aDzvB0XS0WAHI5RUg0PYuN7A+RKduLm29hVFdgas/OTe/2kZUogNW4oHYf1AzYSo01kM0m5jcvQPJUSdvubr0qZFwfjGKjyn7ZbtHg9aIMUIcB1OFsm8ql20M9PGsDbpAwq3JvvN9UwP0rOWmjOzObkhV8LDp+WM7BrmjJmLpsaWPXn0aktGSpZx+QNaPQTvocIJDrf3K3R95A8GWC6zCSfWGRtvsTH9V2nrsRpu3DvY0Jk8mxM2Xsr56M0QAQn7Zddp5/gLMs52gey93Hc0WUzdEr3z+IHdDQ+S83s4nn8BDp/7TfGV7NWLyNLgXu9AxdxHVI0zM4ZoBIBa5RYOTaEw/lz3NfyXu7f0KKXE2EUjov7+1ENQqD64cWffaZeDAc30P9loq5Q76DULaROD0BERlfvetUg9euIXIa4fYYIe7MlCI310ODsCLpfsx+8ueiPDvRI+7WHl/Tt9DvLZQtQX/A/pyuo3xS1oaMzWxzb0ivViiRIyV/xtqphUBy0i93WM05O3U5XduN87CGZ8L4RKwRs3uSeT9gZdS/tnCN/kcxHU54P31HRPtqZO/osoIq1Qt/ZSos4adOfxZB5hgFPg8tkEoqA2fRmHkaagicYTrZyAxsQ7ht3I8KfV+RD1UjsGUjH/wcJ+cwpd3OUXyXC0HYKpWZYwFmT2i6f3CrideqREqBuS+W6UEru39QWUY3ZUBb8pZXu4ZvMsB38C78z6c7m+nO/M6o6HH/UzIBOc3LnTPFlGi3C192MnOuIafGnTeybfg6aGvTmrQ5lh6K3Y1+93arMZqYKdwpcIwbfH6zj7gDE1/RH5kJ9ARiM2/KQnVTsG8l23AjHMSBUbpSPpwBbNbdTxL7gjfpQ4Jpmqc3U/eY2t7om5rtZX2EAPYyRYz2uLJRzkpfhaWe+iXdZSznv7Gika0SXdSOgSWGpuXbdQChcEQybAlssh+53obzoWbMUa3EL+yd8Dsjb768i4Mzv0MkTtfxf5sXvoSxIL8UNHpPqzJXf6oFcg9f/F6BiY+xvfB2VzmgJfdeMtKhiw0bfUYGdBSLCTUYsjDyoHEz+XsmCshYoOWw8XL9lPxPYEhTVCW2mFAWnlQ6+sk9Wuod1Begtw/OO+kzAsQB8m/z4zCa02m6ciwyR7k8s9B+6FPvVPowC2VbtS6+Jb0dWm6F+wikjmIdsXvu3eBS7aTyCGH9y/xCQ683ITZZJpNS3UKGS6ee6Q2gyrGfydmR2VtNd9ozxV16pmgGxXjOqD20IWtL33YdLiO/8+sJv8kWqmMxGt2CJ7JOTjHM33RGzkxgfuxo/VjaIckLkBaXW41KSoeUhyFD5V/n4NmF4f5PFJbkmEtuYOrbjm7nwddnsMbW3DdP2gh+g4HQWv+HpgnHftj+ao4GwBzCkgG1Ft85Qamsi6ECpR0ahxKZusXlqZG5Nw5kG2yeXLBofm3Tjhba122sViC6ObbsOt4FM48p5aUi++9Hmvlv6UvW1i6KCUdnsgYtOffT81cAdGDdC4iba7nipcj6uNsIwRjlndCROxRmMQ4Tnz36ubP7/8u7HV8TmPjCkh5kyJ7/NwOvgoed2F2xoU+sklWEJE+HewIm3gbFJibgZzVYhsWkr+wrgy+LaoR7FdbHUaOLK8TAvhd33PPK1H+ilzIH7cZ6NCiq0KzzXb2zAPzCJGjXn/6J6tYzRq/9EJw7lkSIcc0h7xlNU+nFQ1vvJm+F8u+bF/Evjt4+uZX2fdI6hiFUIxTZhKXNchAFQpgHs/+FyhGX3KKeo6cxfc6/3hH4nyn1dlYX/IJVuVWVtJ7gBwlfWaDPBLnKzVunpMVN0UCTjF22sP/TnXkyufyK2PmQf8lI4spowOOLTL7n9OKIBfZRCZPaNWNghdLtl3ErJ/Cx4z3GLlyCbC2UNKu9OFG9tEQwNo/Aa2vf1dev4If6JU64WZuTo45q5lEbW4Pt2/vlIAlH5ZCWGO8mCg45R5MmzVv2h9Y8nXlUWlj4utqzFTVKTFHmzWElJgOXQX4z+2+dO4TkNw8U0QEjhYWJDAPP92yb+zsHD00ZcKzYavNShjlS3oXDr6NclqzFIWoVZ1QgRMkFTQ28sVESVnAho+SyQ7TtI+fYfFvK4aoyyj+un7hfNj5LOD7mD6C1I9b56B/ISQmeTmmYYTr/+jm3ZM2S4CptPnd65BL8a/nAn/Nt7QpSzn+o177Rf/nG1yi+fAxBUEx0n7ODH8ouVcaY1PXhKKRxryY02Lxg5ov+q0X4Gb9D22g2uFnH9jtz/75msL6NqbtbYsft0fGC9K37Ck9D2VV1KZJRKQjcnICTFUJDUZCPjt6BaGuw/pff/zRCF9CqAaDEJvKhaUR3opCQ4wXW/f+3pMRmla/+vid3fmb9dfc2v/aaVwnvRjeDGx8mWf6B4V1ZI3c1w/bgEl9pVavVhnyhuZF7UsvDlfGrVPak1PEfXf2vPnpPxN7gYWU1rnmDQ6wxlbWwYJKIEfuWXVq/e/5HyDwmJW9wP+tH4hntuG233wHRs1j7TaQC34vTZkZmVefC82SizCGqYEiQijDMvHgYAE5+R3+piYL5+6Yo49Nfu/+ToWF+2yhEZ37OmXIfePBMPy3SF1rH9jyRPiepHk1meL3Ec2i4Feavifkne9dYjJ6t24vfu2gopt22eVj+/dB+eoQWQgdArJi063ve6KrYk/06naIFo7cOXROADmzXzohWLyiThpxp7I5nSfKM5M0ZPFtedg8AfNm/dR/MFzO/Y54kvLtYUf2n5WVZ/SpONb4UPAAD89H/nmyDUXi0MqjlvNiXNkELf5Pn7qvS88hu6BRSMB9YhKrI0XdAaPlnS8F0+gVHIEB843gF3cO+fWVSqFam8F7J1xVJjxAhpKJbrCLp6aUM7j+yV/7Utkk1lvBO8igVAm5G1dLuzXWMgjYaGyz7/R+/S2AGj26D/Dzq/cTm75/ZWOphFIMXlw6pR+zHJ4+Pi6PpqDsl8uSzONftnnBRCjGT2Y1Q1kclMoggsbA7t7iREkBT9QJh5ru16t2mPTxuU8WVxdRVtb7r/U5L7ZxPU8JyVdybdjspqaBEHHYYiN03KIllk5l/q4fob2LKS2CffxYhFZjHSPxRJXkd9g3itNrJ8HjjGY5LtOP7N7tyFLRRXL6P+v5IF//xb4yqGM4TI66a2SdZWu/9NL0fJSCTiK8ciUzLSKsL3gb+6gPiH1AvX7jvTBzzLW5gWS46g9ADjA3OQ7+WBO+Z+oICNqbEkTGvP/rFch93we+h++d5sf+PNYaCGgTf3pbENX4RBi64RW3ILZqocwrrpFBUVtXhuyS4+2HtAh5seZ1vohDQqfbZ0XS1kSOLptnoBCQElGWFwwmQnpCx/UYkrHgR0AfrpYhFoklSkUfQYdxDzR8UmES3a7efM2lBKuH//3+SYo5GogQugatafm5Q1zE3RxufBhDhBAqyxdDpvUeeJcsu8ZPAhbBzxAH/cipEp/+i2J/Kpd5p6BuRJvPofhKkz15mXvezOXV00YTLy8z/NCFYwR6NKJ7LhfB7G5zuP9CLWigQDzYVvuA3KchfxYAaPPfyB3eQGLxw5qbglKG2wwC1L4tt2/5EJ1lxKNThZL1kWUw+3TtvYeIFP6l0L1BtphWpyS9vCBkEPAVr8I/lGvL8K8xlDqmD28DVBTLlsX1LSkPqHxrBkaxLkFi0deAZrbmOXuekCrQy+/Dkzp6aY+zzTWsHYGNhPPrDaYtX8vbzgl7vK+7/MoBDzpdH6yM4FLMHfKovV/D+Ai8C2clkpyFIBz14/dvyLP7qJ6gxcCpD+JHFlSFRJuTZU1BWioXttfgrOqfvzXKjgf8WgDvfrxaYDM4F8u1hX++BSrnGxHkUpM64q3LiOl8hRQTwIh1LKKSZ7LegEQTnVBevJZyEjD1HBArfDmJ66EsNld4QQDVdZOBr4brhw+WzMfdhupyQJVgUqNOVskmsCm1hcGMT5+5Ozgh1VMqbgxcDCOpxM40mswGps4ShyUr56Ttqyc9ULK8C/barhgqwg4UqAwcDu3vLdNrnOLnD0jY2e/bN1CZqRHex6VRK6Nh9IipOOAevpXKRSY8rMYBm7KTHTbBKN6lyhAB7eqfaLa4Balimb/zjAiOi1EhB21/WijZ8z87ysXvSlKcycrelsmUxXOkAjvaOofLCFMHesCAFzL7d/lIzC/W0YaykcpjMEtejBrQ7w3sZfT6bfHvsNfQrzkt8zBC63gVtQpRltCYPWj2d4CZprqJpE4Nt8l0+Q9Gls/dYepgsyvt1bfYT9GBMPiV57IzwJFnG+tUj00NlCEYqThF7nLw4SMwoKhptgV0WaPkwiKmkq4dTn21efBwaiYd5Ob/Y8NENiGrXF4iZbq1HXWvPQ06J0yrWwSgO635rF9dlTL1sgcWtnuKV4Av54O5x5d9eOd8ZHdc3NsS0QTn6hj78/9dWnLn8PEXKs+S3WwWklsbDZ5t+wODqu8Mk6c004U3C+dnYWnn8Pnm9+0gcD0d+WWrSjpIn3r1xVY2/kJ87vcLXMQe97/xCA/stWZu9lAWeSI28qnbuMNPG6y8WjyPLwCxmcoHSA8jyHHmPMRHO15iWZQlx0pqedmsjYutHBaI8vxEt7V10eu9wRM879fzm087XouoBbPY//ICZbfv60JcNAFjjLlvz6kBuKvrpBoRlv1o+uG32YQRwu4cx4rCU3L9QYQffMkiF+6OnBzkjfNiA6538FHbBpr/5oCDjY2ecRnVMUJ2hRoVPneMF1Sgljn6RagE89hUCfF0WPrwO3Pe/fRb16nyaK7zWVv56MFar8NOIuRouMyDyDTaGHjbcGgAjL0dTXd5RtSggdK7l0+x5KNyVwi/jHUurChydUV5Ckq/fLNJaBqea07SxXBAf6Gn8+/JMjSM5EXGtUY74oWvRJ2o9fedStGoTuT7jPMc/ttbp1qrO7BgOBp0t9z4Dag+VDy0vZ+MkhHC1bLqnbuMw8mcqbjVECv5lJjdn3b5qOfHZlj2ORdMGNHuwaiWYwmBB4GvwmjU0V71RhCO9y1C81De0gAKGZGPO+/CQB9P/l5bLfmFDQkVLk4TUlpFD3U4CdJXRB5f8UoSURWqTNMIKLwoCrwx2meTqz22h3PyXMpIc4hfgMaZ0nZmQiXGILW2pF/fo4mbWdrz1Xq0aNLK0Z3zYQIMOqgh2wnfbbs5AYn/KuO/me5v4BhalXgasNL7KwTWgD2peyoy8mvrhAEy9agGZBiZMJTPrQOextLFqnwRlqS6IqEgP8aLcl2BT11P8SJqz3KM+e86MM4J1So63rM1vkAip/ZVgk0W8R0dqbriovdNPXNnVp1ChaAHw4UjWHHkrG7t3Dz7sxqp/PRffqnM1B5ua6MwzrzKxzWkGriOVCMMiANEPpJU9rlH13eir4lBa2+0ZQdiWrCBSpdkF+4+1FqLZ9Zp7/LxyilLwvNA9+lYv9OiI2r21m6Nv5OPa2ozFRvAZqpaBY7FXC+PDm4085UiSAiiLLLx2JwiQy6wojWql1AkgmSIRSoD+X5tinJyIsWrVvY0GaULObDH/unIglJPXbpLAEk1BQx1kMn7DaTBgRel1cgKWuLaZz1kQsTK+ZVaiLRdjfNlpvFfdVKtNpYT1+UzdeAaI9XAFueh6OgL9k906t0OAylLwmwf8NH+tdImcI2sjVNOruXTigOw0OBbfhWw9sNNuea75lKcglbisSncOqVnH0MyOAHRzwvpvI0cBnykaOm8rC7+LradeGy72/epXj3tzbG04YguzI8HoYDAQD3gfFCE+tOtxcdMYAzsiPQbSxDe7pbkpizTlyG7WOMue6pPDXiB2ISsYDaWtQXnX3mq2SShXdKqO0sR2NTq2h7VWKl4i4wRGFCSrAogXVVrkOKr5V4Ne6sKzy4RvVy5VeU4g9LACsS4jMl2A/bgVSll7dDUamOAaFrZ/BEWhId+bPUQTpeoGaDm4GDHW0+Prh27h6Dlu/JeCTKMInMoJlx0DF2eWdSmrJI8fIPBlhPdbgStbVhBiNdtmC8rfar4FZqJhRor5ZswxHjaV9ig+cMLGuwWY19DNagqZSumoL8Kk+z5lgljEd8uTbdP/4nRuJpjagb58wc5RlGY+6BgOmxLrAwS+nA3YurQEbjeDXjpnS+zQ46TLIvl9usBTKecXn+pF4VI5FaoBwNk91nkGl9EDATj5xeM/8eBZh6Z/BI3lfjTBf1PU+LH/9iYrrbfmHlrL+nNW3UNu4BR8KmKfTj+/bujvOp/h5zgGb0m1riscpAxux16t19+Q0wxSct37a4Rxo2GGPEPjprLW5njc2bDfVC/Kb/0lJzQsc42X5giz8bWNm63tfuyNt8SLABJKAdtgpQqNbRnXzL7+Wo1XZfaSyAetmpTlRbuiPjwq5RjLdcsnnuKJ0hFL4OQeiS40MiqxESFJzePuiedrWSzu90cFPuD8fI6NhHGdF8aSJ8BFOsJ01UxTaJFIGOiYxUP1g43DlaAGgj5Bch/851LQl93sLDyYP2de93E/agJxpSBybGCpEl1JgPMIZsQZKiW35GqckWpKpWsoNQErqNkeBBmpnhRqSMGHGVtPpFqXIajFFxm/v2hKacSs6Hr1zMwv0+PqWJ7KBzN+WvzYyC5K15MnG2QeHzWidkBRYe7Oul1JGd0z3DB6uOz97mXfPeTYewpf7eziFRlc387vNOasSot2oLJoqwEqDpd61xrAhgt9GzoD8fxPGrFx1DwUNtDa/ac1yKVVzgw75wAKEHH4SclVzr9za65hxFhBrcFNtJvTgyQAULeCRKUKYpGcKTHqmNZQAYH0q99dhtOgdqpbyRlKF/s33UhlKey4j/fX0EP5u6uhLBeDmId0Z9zkdIHgB3L3b3BmPu7PdYL6sjpF0i0/VBPuqYDSRUMyixR7zES1iVXdRXieBKX0azforhLXpg/HejgcJECwaBtNSB7AB1Bqidr7SLAYcQYfCqEJ3KKkbSMrsE2Qu8OMO5kYX24EUq/7Z3YAW52EKBouryx4iwfXVq91gcTQl1W2JH9DqFx8SshIgF70C5zFRI6m8bQB9onM2NQlbepVAEEljjjamJj7rlrFHIj9c0uX1IpoI7cq9kwy3ALUeLDgildRTd5l1M5c23UkN48yJTOee9tYK06UPc7hzgdF3ZxCAnyafNO/iRuVbD+Faikd3VSYQFbBsgvKO77jhJdvZFIz6Dqc+cN2S5tbHXxLrq4WJl1el4nx6IUQgxLpUgArCqF9EkCO6Wdear/QHWEtKhePLJB8rVrqaquqX5ZaCFL06g0tZ2mfnS5MXXqN+okePFJpHyIv1RXYbWWmkM5y7kL+EuE07dmdC3VSDMmXU1cYsB1PAnGrPSlzKofe7KWz8mbg4n612uA7LCEp36WJtWns+kKYNVh3civQJLV022b/fiTRGFnt9pvrwiKo2toFBRmY0yQmvi2YYFFnr1KM14OLQ3dxYDho9geurtF0ubRlp7pHo4unQLkCw0Uiekw8wH9cnm74rOm2orSwQ7Xmz5aC253u7nejOocVmAYjs8Xwu655wF0sxeCoKJJqnrwFNweZ/9nxnnlfrO7NWPx2LIN176ajOooK4vkJdetVlmvoiJ4ZeHWVHe2oxiUX+34hbyiRmSZmdkDR9uT44dXqkDb+TqBsvqiYRLA8Een7vkY7kWxlsSZx224572LFzrcbwimLpOW+kqDLfOP9rJ2S+l/HFkQTVlhwnL9dXIJWgwpkYyzf+KX00fP/xLoGXFaCgsVipVIrYyrTpKUqXd6dSa0R8ltQq6imr3Bj82TJ7AZyIUh/347kBJ9vTE4TLUB0VInqftCIlPBlUhVu6laFEsLm8om4DQ50Jns8ZON9cHvXDs10mqtdzPmQNDMBuHgnMkLsZwZ5+Z+KnIVJXOso4FXaF7HSc4BHF2WKiilrIDeHbi+gWN7Xzm09slaxpZqjVXIYrVYRP6oS3GCeHnlyfhKUIc+xW1iO1PUJGDdActLkJGlAxKrfiaRb+l+oqqRrmLQbQd5IGhC+vB/gb7+T0zOBBfxn7V9PsKJOVYB5HsNfBCDCVMuzdCpyLCtOO3LgThZ2kwnlUfCApeURJlkuqknlODMeuH7uahBpPekYDr6Gu9f14rSxIPbeH8FCXGjJ6Hxwu4fxNVyJ9+3fF4zRG0O2ClalvpC4Yq+kp7ub/ZMZU/PzuQxWwiDpuAmDTps3Nc81OTpCvEAuSS2O6LWLrj0+6CECpz1wkOCVMPGreB2daPSr22qKwsO3CDbKz/9kErBi3YFKrmNsblubFN7zIGxu5+sx+PmHt4mt4w/Tu7nd4rR6NQ27HVMSNdsSQXj2Xm6yzS9X7i9DpsbXfUS+kj37pFBzmZtfnv5NgeYYxNxDKsT1BojoGp1Cqld0hS/Udq6+eILXhBBfteo00dEH6hUlVagP+o5RWSx8AGEw/3dVRTNc5POJv6b80cLUKF8EJCEQFJ+AYxzuyhPwGThl9avHNkKs+zqnyEDBOiNMGjU0LWDcHyq+XTB/WCH3NrgOyyPDIxWZ7ZYZ3ZmGxelM/L27FYT1CD2uOu51FMylx1iVScGcfFb3EEt0v4KT4FCMS4JAEBWVHCJAxxq71VR1DNVOvTlB1kolyVhDr1rXPLjhWmOf7+ft2uRKSE1qZRmeMYbO/Mb4RL21dOpe6rWV2d7gjFxJFDzjo7iwVYytgFU0kDpUFOTY3JOJv6HENPNY5/H71oHUIVzB5Qx/LQFKuHgrbz16g86I7XdjRSkm186qEClopMW0AZLYamBD5oZB3nHvcilFs78R2dH5p9trxJ7j6tflELcsFAEYtDvySjRc+kQg+Fw7K+g9fGlywdT3ouvNC3sKoinNfTHxZnj8QPsZ7PzWUlUJPZEe7uYjSgjVx9uO81u1LKBUAxpNFRQJzguCz4SpQ5e5hX4MoTz4yVgY8fZWA48QNS/Teamjqu4QbEPIemoYduuXrJ8T4z5jC1EZpzQXNj3xFOY/fPB1hvkxd67qwNEIwuqCg/VS5UM6otF/8Bt8wfSQiuLqOyfsrhkCUPPrgQesx7Bv85TopRjHWch25aa+6pa6gybgDhZjsbfT9ahRx3cbETeC9LX6AfUezCTTDBmebG22MuOxZkRz5q1Pphoa0F3xx2GE36v8JdTzsxXBZmriFz0zWh8cKdP29IdnD9WUDw8rtRqb3ZJTDrzetlcvKA5aVUu0TB1z60VKen0TB3/0OcIf9flj3fWGP/sx0cg0VrF4ABfjzc+iU+eCq5/X/U3uBVx5etTx211gFLA9nWOA22udo75ziMUfHTKOxkhZzbdvb/DUs14Z5blldgax3UO5Vesn1LQhJNW90D6N+KJYEM6JX2beuDrcXFMGgMaDKmUyuCdAR2oQ223daqdkqvb7kVeyG4mXf0nxTU5o+QBwwg1DmSsEOUBOVQ4GWyA6uReN2pe+PTZ9ojluqpaYyekdJmlpya/Ruz6t6+7c1Yx8JxWM3nAH8L1F6B2gQoI9mxs843K6xwlZr4+c9KgQ+QCjlH4y23wbbjB/P29BfjYEWtjgpyx5nCL+afK6BTbLzECLXTG8nX7Q7pnvQ+mLzoTB3jjcpoVr/GrMwJe7NhbQ3ZAIwmFd1LaUJv6sH0KdugVw71Z05frSdf4vFCVW0obMDwv1Bj0guyAKObXh12QmRr2kvb8C6t123IcQjmWkH0+hBUgq0qyft0BtSSGQxE+rd9VDmeKVLfeIIA1LcV9YRYVAhD3fa8LrmHs8IQiZY475Y/kd2C11PlGDMDm0+KpwMX7qVhRShHt/EiozS3cnxEz6oLyBL9NJunb+JNStTyih7ftT/6obxyQUXyq2UW5fwfYSxeQR8Twqc8D4gw0SIOcUBuS8bGB3hf9Dz2ioHnfdPQtdWS5lvP9kz/TmB0h4HybjbCnmuy0jKUn9tkp8vUJ1RbIwLyGYk2v8RwvwEGxIBM4CTNlBS+mDvKSVK8zfRvichXdGbjH8lNxCNmEN/GakcpjlZ5XmzJGn9AW/daJm8e8B7UkVU2UrwRtZ3jydrP0phUREyahzvJcw4swSTkv6CzhXdbkAAnb14iWEgIvwLrQmz9wAwlZqSuMHBAkIsLxmCzGKNNjYL7dwdrVgQHihberG7A+YyYFPPZG31G2lLEykHvynZunQUHvHsxO5lUTUiwlNsGZN2VnAxVgWagaVa5S42E/P0OqQjjMPI/wRlaUmNPE86nAi23j0GyV3d1Dsysd6uQ6pWWWPmFBu8GY0nNU86Dqo7rFE10PU7Vv/bP+f9eDESxA9OE/1ZS3kapRS/jouP241I963zy896dviI4VXVeFNzFI6lGYMmdqdI7FY+X5u+nPSOBFc1kxsaSre1873GdWjY+oKAtsdczwzql3niY5pveEXXld2piLCSzwIifUrq4+I+jaSF1lFXvLv0Ce68jlv56T823txp+bcOk4ykG/2hLkA9t1UaS9hs9BCh/V/wF92FVZoU9r+i0AT+3hPeiDoaxg8ZTT2Gs3RSwYiCkWqwXuZnZtAL4U4Ua4c1mZzho39TXZ0fY7VlzVWc4wHHi70bptfSHDMOSX7fMfh+qMlVqDHxu6Z7nci5AIwTkQTVpBXxBz8GnCqnxYbC7UjnzXlx4XkJ2xZQf//1/rK93mz7Wtgv/rT+8l8f+vPxub73/hxxXhoEjL2+C2sCuTZTnG6Hup0MlY5UGeHHUOF3pRpyCjD7Na0MaH9wiep69tr39fR85VxYkRr1t6/er33ggh9Kot93zc133RV33T9zziD/7uf0UkmxQVXZKlUGqlW2bKtbJQ1gpen6jTNWyV/Qt9si+O/8wkmAfHZ4GztsXJyWArZ2JmZ2N25+dQ8FurN3u8GfOm983uNye4037v/P2DBv9usLaB7/88a/CyAc3/wXnGeZJzT8qplFHnb/Rv6KfoW+vH6XfqSXwf/qH+pf4X7n+w4RDMwHZoBg2MgQXi8C+UQN0fowDjBkaxjB2cxm1YgS0oQj2OohS16EAfRjCDxfgCq/AdfoU/DVHZcGfkY5zsGoDMWUUPlnTapneCi81A69x5ay7SGgML8tOLzXaZWuXbAlFZibqXPgfmne2ALIPOhJYalhedfcXg6M98cO+SnxmfTwVqbkRpWmPSrWVqwMnHRlMFQKsTodidS1O12WsE4+gJfm2SYPRVMSDSnCXpgLado3Zva3ftex4Hbv0kor34hxe3eEcXsf6zWxtQqMI1ftgcjjUWTw1jHRFLTmOkT7Qtcofa1/Z200k6bO/SbfXI/cPMbnnbJNfD4j1Mm72i4RZgwUlKy4YUWM19u3cL7PcP9pdRbKA6eNQdbhxu3pYSeV10apukMNppg0bgxVsV2NnuyPPoTv/52XCWLNzeOqEBPWMgX705rZrrQMkS3cVPVzcKmaMXn/cDqVB1Ge4z2SNqCDSqQsvQ04TvFpEr3/mQM/XgaHWQ7zK+y0eruGws7UrEz8y3vUMFMdNMvIbcFoO07T3fcCtOzQK4zVDogfogyUojN2ad8SjL6vRqeH8MKVU6NgBTZf0rPag9MGeQdaSxe8g0LM0XrtoyFyIE0c+0Lnp7WJun27gfx0cA8VHT3yDwL6fQ8JOLPMBWNIVMFWlbmSx7pfN55bCuCTY9DMci4Mz3aAGkzx7Lx/D81RHxq/ete9uL7u+6wYWHndPcsN5m+z9+8mjzrbo782C6yj7CnpFhApRkN4dNUsIgzl5oFupGyGzRiNhg/GXagTqjysm8whk14/2XH7rGToH15sgksypbPo9/YnBMCizui1CpjL0Corn8NGSVlVrrho1rpYsy+W/mFt0p3pPPGx7OYhHrHOAZXEf8GyzYBb29eS7BJP3IZqkngM3lT81jc0AwN20DvqRPS8RwxH10WrpR+VYbDVnNl6Q8zRMgpmXHZQjc+unnxvU1Pmv0e04waf/LneTu+cnfbiR3x2TWsTyKuEXDwaDuwEZG7L933+dEXnpTo8K8KEn0bMhgSYugnpJF26g2r36ZTyt/rQ2JVU35JUBMd7qON/94onSudlKDliIOw3375YCL5fXCcNggA9louHE9EFNF5T2JVo2bpqW3u9h90gYGxtpP0XIzDutcifbePmM6frVh2b7bdXbSTCITx6vNeHQhZnGsZMZFLWd75x7admbVarpC1VutNM1QGnuBriQ313Hq7deP3QMyyQijw28/czpUztdiIZt3aI6vXeBHBhuXdVaWn6A3KC0TX7eaSaCnDjCz6oAmWvWba/nypIh7dFQgBq/4IlBK1s5q5rin0mJDjWzPjwI10QOzT9Sgu/16WbD3uZPp45X10qRwSLheTHwMFDWMqIaNeqwYxSHVETvb764UtgqANguDHSYNl565XExYNIO7UToEm9pgkZFEJZcvrFZLVEuub9JuUFFKpH5EPij66KNkfHVfgzStiDU0zOqX92XqUEGreuWHj0ktW+9juoyDelnhXr5AVIk6yymMTQDalxihAlSRyHWG8Dk1cvwoZ5E0ooH9pJqb1b8+M8Ep82z6JqfAs1e0VyZVhDh/YKf2tf0Z+5FjGx7u7N7uP+T7wPHGiWHI0Ws71YrhRQ4Asko4W4bM9Tyq48s4uw8GmUdUQbgF2HvKepTFeM2AbNFASNqiBRWDaCpSP/fQk3HuTdv5A7P4WndXs+PzkpxpaFUCGm/UVRnAkqrCkiGvCcnc2s6otp09Odr9tB4qmeF4as9DRou37VZdvk4mMpgHxGClHSST491Xzws91oa4beFijVFssb3UUlia0I2CBng0F+swGNq7Z5v3KUrlyHF7miTZWaZ6NqohtCt5iAytagGLi1PaCPwic7R9lrkJS2KiJtOQdYjzoNTykV6HHuhwkyzrRhfUMs90QgyXxZ2Y3k65ySfh4dYYTsuhn0RPzdZUT66xzIPqHxKrxZitc409VkxsbFpkdhouG5K6DotnhKYp06zAScq07/Z4KNPnWCAHPFUptiRb8m9xQS20r1qzrrMos16QNt+Ru/hdvpQomG684wxZ/0JlvbwSUdAtykq5vFDZzi8PWZnReCfUa3rVaNVnBFrZADPxjODJnpKEbTfc+Z0bWAFCqSjvIP/EO6UJbOaw6qKLzvRGAuPffMAxTVRsi7K0mis1WxXwxlymqnIBak2sjw9ggdoXq1cUmrNF+mQ/NRdYaIsnxwFiW9g6DU4D6u6St4osJm7NnamP66nYx4v4hbh9dhT3x7uTdeHnPB8TJ+WS/4/n085y/78RaReNMO/h0lyjXfSs+o8fM8OdqeS0QEi/Aa7/37dOiI4q6eMeQerrobOvvG56vHeLquSz3lpzLW13ZneCQHTqvtaLgnYw6LzfnM9UMfXE2iktPg+gMSExC0sUWwxINt4CfHfnxv3SAF+1L8S0kuE0pbTw5YDNnrbw0JoJRroRdf3S2unoUG3IR4Wv9eaDc6GaGKXIsqTUlGnOyQmktZFv2avsZfUyeRmIzEtyzI73HR1XLwCXtRynmJssksc1z2vlC2Ky04XadX86+8lG8ODZU4mJ+px/z4tCkVNB96UuGAa28wZbPtbWzQwM1peOMREfqAPhfnVlpcIeomPjUr87H3SISI1rMY3PeJMsueViVX3Pl4YS6NC+YSLyDj5pSTacLPf7KFY1A9IERbSHLTpGaAdRH6k7QjYUdRP9IPubjZGZrUdFqpaqFdPIMZ1WzGQxTwqQb8dtctW9LoqUksyr8Wa6kdnPHwvDoUmi9ePqwZAlGpwu8HoDTcOwkJidwUCIt81W+qAR3jhujLsPK1LzS2TL2vIxZ3HLBHnOln08spMobW09WE4200nL/b/jjxHSfwPwMSGqqre3M6JBNIyTce5uRWUl5bonlMtls9J86cz+azeu3t6aYMDQYN8ThkJD6SsBuJqWu4pJFaJxUcXONQxMy2bZvplmlxgpX1vBl0uraVq4lCSrC62yQYF5a51vG0TQuvLJyXkbnH/XH806ezGO5AiXsC1k7Hf6YApyuapRlUEc7Bd0pVXIWBU1u/sktXlWQfMzRq0GHMh6qrdixt8b/yli3XP7M/Df9dofo2myXGMiDRSmauLwnBHitBUZoXN2dMBR3/cdXkNN2bxZ0J6NrX3inrnZmJqKbYA0AZ91FRciqAaNz6RQ2tIk5BlZSiVNAhR8zmb2uKbOpi1rWP+itAfYWh4DwhPKvWVTUNaXQx+s+q2oysjS+ZKiAxrPhyYMR2zLw4cVhHdgnErbXvPQ04uyP8kLZ1tRP/kxhw4anFn9ljb09Ky51J8XN6Xxjo3/uXHHLM7DZmDMUzko9xyY/e8S/Th+GtEIfCU8o7QW5T8+2OBfnKA3/pGsivlOJyTXrEv8TLBgdaRL0oBh7w0hgovHTsei8tAkMa6namxlQ63vgzw4c3LQ7aPWgalnnMc/cZMdU0V/s0dG7Bx+NRkFzuhab8zIe19oERWwW/lUjRRtKNteXtaT3Qx1u1gCi5gebRIs6xODCp4u02swYPRArVYLfDl9ZzMNTtFMrTBrppoEtuwomeVFF5EXHZqKCnN77/iNRc9RUdulm7n6iaAxeSFJKBil5wKb2jTjab1ai8VKJcNoBAqq5INWr3KWKdJFJzV58mcfSHNLfSZ2QDxlCzZUvaqpI2e4McpOzHv+OEvoxqgRt+0ioF1M1mZ0s7aBd6T+wpYU+UMhph6tDbB7vb+wi6cg2fP8fhk2t4T2PFpZU685M4A5uEBkzO1/La/Sk2d2FlTvKMX4Qne4/mi9eSEEAl7ZqcCIVhYLaAuvjAlXAVATHzQRLVFSb9LcX496ZlBsyIfFdnllCbSrMRODZdLBfgHpXee2IhMQVcSBfYv7wYE0XLi917s9i6vvWzi1EhChg6cG0dvZ6sZ7Et+Q/MRaLgGU/TIm5xZSHb7b1DkKcpeiTdp/nx9E9Onodx4ZAntsm9d7KI38JjHKRNbBY52tXdaCRsGFdCSU2wzlazqyjcG8m8j5sVu9a2kKI1b/Y0OQOwZeleXthohhlZE64lKKZroHztCQ/mkwwzg1z2HZfp8Q2LjPy9YOrjhfmzqmhbN/3ZhVAPXhw8X8fX8NzOJnWmAuvBchdE57bpzl/fCjt5A9MDZ+nFARnEL1aDHZ5azQ7nARcePk7nATNPV2MGaB26JMxrt+w0Ba1AMXI2l1NYmE1RI2kG1bRD8GnMXnC1AodeR5YktX2g8FZj57/8ZNSyEfLS/nN/TKUE9HjbDaxc2KxbkF1VJW6jYGmp+Lz8NS8qrxospsHfe+UcKPP8cg+/d8ixw+5ilaOELT7ubxp0jkSGGeDpDDwovwPyLTZTcdsWmmVgiOQlGhW4GHDVMwoVGAhOq5ulYDivZRAK0wY9zfdb6m4G1KeqM23emKiHdDw9x7H7leTAAWMjY0CGIDl6HM8rxjjyFkLPJ0h0jvorxKjoVaEu8hmQquuLZN3yJ9iRlfmEyzGK9ZJtCmnWJN8PtdRUmfm04XEfWynpYeUubMZLtWTKWHLvECw9TD6U6yyRdYv7GrHJjT5GdSLbdwB6efZZFlYtpNw5UNidZMCS9SN7oZPyat8e0JPfHbQuRferR+iB9u2l7ba6pppqzwzeHedJsb+gAloo3k+QbNf5lMFy41BT+9EePOAITdab0NBeZePwc09WuKM31vP2ASbuEH0v7uLWZn044FxNPjKPvdfaCERad5I0MpyJwjYFqPKSGacMItIDhU3n7Qqn8XOL/jhK+ECFGa3sssuTrGPGMGXXEQ9Pnian+POTptoKlN7/S9wMKSczUSTJz/Jc+iNgN3UrLGcT+RJdAzTG9y8Bpesy6mzpbm6bPQgodPZIA2Pk0wAHxwMQAIkjo2AKCejh8AwGA7+O0ikH3wHQBM9CZyCpO0fIBvx5GC/yZsy1hJAQA4V/8oAD29XwBQgzFEtsPnQCqvKJohkPDoeTP0sX/eTzDM20YxAgoGDgEJDToMmLBgw4ELDz4UAoSIECNBigw5il9tGEExnCApmmE5XhAlWVE13TAt23E9PwijOEmzvCirumm7fhineVm3/XA8nS/X2/3xfL0/398fSI+qNYGafhKnzUar0+v2B6PheG19c2NrZ/tg//Do/gOAfFZJQfoEOuVq/k+cmgJl2gEwnViTNJGa1HAAQPsAyFJ4AAwYtEjyYve7INu8/ADAYsVawoRY0Pt8yJd8zafU2w2Af6MDoP9RYwdenpPP+RayRJUi1SrUqlOvRpNmADRaZrkO7/TLeHkZm4AEmpsJGRd/aGcAEMcHhgIAWBQZsNYmg3HpH0xF+QvLz+VO3/l0L1dmMlPZm+N5OH9CJhrGHIuscJdfcWRHuL7x/xmglqmVaiO1Rm2ndldvZsJQY4QTc/77t+N//58YxlHTZ431ttgbtwTn17mIsU/d7Za3GtXQ9g504wOhbDTimSZQTc/t5/nPYv1JLYwuxj8F5bVZt7pv3a6v6xDczd+1tbpW1tI+wnEOhd8nJHVsypGjYytG4dFrpB4+pkvn2+Uvyuh6J4ZvdFwnyhff+h7rFwVSVTqJAgUUKKJApc7sBQXG1c3dm8VHn1FQqJgRjmJQLDcnMwdOKrWG8/tWf0n7p9uKzJWrsmbz9RLWJn51K8qqbtputz8cr641noylfxqdwWSxOf/vH3+TUn9ESv//94jEkk/1aWhqaevo6ukbGBoZWzOxbsOmLdt27Nqz78ChI7ccO3H7Wemiu+6BEIygGE6QFM2wHC+Ikqyomm6xBnfsDqd5Bt3Pg2PD3KPyMI+Onyd5egyYCT3L84zkRV7mVV7nTd7mXRSwETAtn2Oh67fwiNieE/cjnNIMAQL4+Dhk889zj38AmRfi8neu9fzNePf5FPA37Z+NlqdQqxmG2RAgfT6UeyabsyZtz1LhFeEjCZgBfZVevMDL8WzGAEBOs51NrLvMP3SQkh50J+GGArKZ8gMBjasI5BubL3vXdnZyAIDvCb0NOBWjl82sddszGZN+i7skhnwCEwMD7+IhZVH7jMJr6GMq5ppwGppaGMEwAb9+4rz2+EJkZtQZEraApDTWReP2R/S9nFu3CGun6CiitaQloo/J7nWOXZiEXh2jopChgDsG++RbYne9+tkv2Kmxxn01Doq7VSh/EfsNOzdoxmQ+X61fsX1HyjCk42d2ZP9fECD45tN6nShLQt8SZ4TuVoFWYee04lyZxQC8kiCpI/AmDSwPwjz5W0/gB5yZgM9KHE9SDQvUH/p3SDqIvmm6rgbXf0lqx2g3drn4/YTaybMgLNdAzrWE4sDE9zupq6gM6I2blFThVoLRJUR3kGP7+ZA9WvXwYQGoJpnTNhk05gTxcLyHUQWxEMdgkkgkRyDXmGGCYQFet2jvGFo7XrZ7iAQzzcArDqepxGAFBVw3YzaHprhh7bDCvA+g53Mk5HbW4pSJUSd+BTDug+suxPBKg54u+RKaH4F/YBzE97HNMAEwv2OmcDNFDwJCjIQ8IWZCnK/8Kt0Ibo3i/OZs7w+M76vP7T2EAscLqWJfcCMSnv1LHG857uwv5DjN8Qors1X7bhuTz87wik2OQCpMdYNvxL5bquIwK+6TiVKPbBL3OH7h9Rb/3oqJagkQghHv0B7bLBjlHljo2I65fMtl7Dlvx0JotXkSr1/acoVwidDYcmaksjZSjgtwGWqQiM77iXGBPZuW3YzaXRHdGqZ2oqkE6ySn8sLG+yauHAeSeLqOAm6EBRWzmCiuxgFvbzht8SruT26xOwk3eCxb6v54fiHznH/yOhzYyaXxdScuTxcTpAcH467LwvxEK2M7+w/Ly8jYWDa5DMZrXwzelRoIUlfxb4CetniasVNW8J4pcrjWgEN7XGoUdhGZuoLrBBj+Cs8Lka7CPXChvy3H+zmI7Tc8d1KpmMUtqwkeU308ue8N4d3XYO4Aaf+62QPQKfQuE/oenB7O0BeYuzE9gmgDui24Lsb3P3WvT5FuIawKM2qnK0ZVSmrHyEsifbuPH2/0Gde/f69kOlu/AkVIuX9hOX4YMIrjhZbuDV/C3Y6b1gOKLDRcn8PSapQf9q0wVEYpPQE8ZBzpa+ojxc8DKyy2mSAABb/LhXGBNL1m3hHKydyX7hJFA3w/Eo529nz2LaXP9xHNB2SeZC/ajTXEqPXamyOmz7GiASgmwEKByENh5Omcx0Qg4u5HMA+VOgIJiqtns8d1B6F/sAcB5jXvwliy8CqRH4e2j0JLg/jZ0b8+Fwp7EeEZwDAkw+s85zrSAGQHcYi6lYFMxdFR/B63tdMod2Gu7ejTiO47ycPXKq+vQxlhEwTC4obFfvg/l1NnIs+2Bm/PhLVdMFUE3Vr4hgJW5+FxQiMfHnfcje0XXe0apeCGA1nWZHPo3Npf0fJi4jB4mnHKoGJ3InmPT/B8ddJ32xftb9UY4QHCwUcAoDT/N46ML7AvgmCVAhgMQShD0PKP5VgexcAwOCxlOAw6GYEj68pIhCDAoEU/MDq+wRgDmYEGM5rIWLRnwdZsZHEAArTRAAClZyEkdC66icGwO8kQou56DYpodBmGcrwlDtF4RpDZ2YzEcoYZDcsNY3RKH1vGwHHbIZkUfcZYWr6kD9m8svO/jVCv+dDs4tveUJ29kdsblexxWt/fbLTC/dG/XNZj8K/eadcz9tqJb4uUrT3t/yOpic9MnM3sAvRqUup243FfKT0MKOc+VMVHasvt2TUKIeqNECoWdF735WA8cqSRnZSgt74lse54Tu8shUjoPJmmQ81IjFGfIeZrjGFCidYcCQbl3RrJcRjhcYI1AYczHSLabJvWuq4PehMSXDI42U+xoLrGOgPh/ihbOMGFdQezf0tcfIqDAT5jmcBLvQAFFvT5VuZy7pIYaFzy7zqMUlT2QiIiTkhJf+BEUHOaNLchX9LGhTdILfMM8WGAjERslyaOObp19wMe3xvT5+e4L1yG94Vn1vtJ6E3BzqT0x/iQOGNgrggGEnmWJVLrza95BFYFdZr0pqn4xxnAz7ZPOrvxu9ln9GCHuHCq4qq2L77Vp8JuQPrgG8R7iwRQrDcvyLhL2T1n5FNGYDi1WhnLFfWeY4GC7lB0PGKUoMbrzuuEsy8uljTmTmNQAmvoaLLZQOmok1jKn+jxKpDVzuCgR3FYa2Y1qrBICINYO3L1m0h4Nbwz8Pagv0tjAAAA") format("woff2");}</style></defs> ++ <rect x="0" y="0" width="1339.8517879909941" height="1669.333441840278" fill="#ffffff"/><g stroke-linecap="round" transform="translate(174.05543178982202 183.77775743272582) rotate(0 107.5 36.5)"><path d="M18.25 0 M18.25 0 C53 -1.5, 90.87 -2.3, 196.75 0 M18.25 0 C62.45 1.19, 105.88 0.54, 196.75 0 M196.75 0 C207.71 0.99, 215.09 4.48, 215 18.25 M196.75 0 C208.37 -1.99, 217.23 8, 215 18.25 M215 18.25 C216.3 28.51, 213.15 40.3, 215 54.75 M215 18.25 C213.85 27.93, 213.77 35.5, 215 54.75 M215 54.75 C214.62 68.28, 207.35 74.34, 196.75 73 M215 54.75 C215.37 68.77, 208.59 72.07, 196.75 73 M196.75 73 C141.79 73.81, 90.16 72.82, 18.25 73 M196.75 73 C155.72 72.13, 114.23 72.98, 18.25 73 M18.25 73 C5.95 71.03, -1.38 66.71, 0 54.75 M18.25 73 C5.89 71.93, 1.51 66.94, 0 54.75 M0 54.75 C-0.8 40.57, -2.19 28.17, 0 18.25 M0 54.75 C-0.05 47.77, 0.3 39.75, 0 18.25 M0 18.25 C-0.39 5.34, 7.37 0.73, 18.25 0 M0 18.25 C1.35 7.7, 6.44 2.04, 18.25 0" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(199.90550655788843 207.77775743272582) rotate(0 81.6499252319336 12.5)"><text x="81.6499252319336" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">rt_isula_create</text></g><g stroke-linecap="round"><g transform="translate(390.055431789822 220.95750009135367) rotate(0 220.39825439453125 -3.209137645198979)"><path d="M-0.01 -1.17 C73.54 -1.86, 367.34 -4.59, 440.8 -5.38 M-1.48 0.84 C71.97 -0.07, 366.29 -5.49, 439.86 -6.96" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(390.055431789822 220.95750009135367) rotate(0 220.39825439453125 -3.209137645198979)"><path d="M409.92 3.85 C420.83 0.99, 427.08 -1.06, 439.48 -7.35 M412.46 4.54 C419.31 1.78, 427.28 -2.89, 439.76 -6.39" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(390.055431789822 220.95750009135367) rotate(0 220.39825439453125 -3.209137645198979)"><path d="M409.54 -16.67 C420.34 -14.3, 426.69 -11.12, 439.48 -7.35 M412.07 -15.98 C419.16 -13.34, 427.23 -12.62, 439.76 -6.39" stroke="#1e1e1e" stroke-width="1" fill="none"/></g></g><mask/><g transform="translate(577.1851009792751 179.99996948242188) rotate(0 98.7699203491211 12.5)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">attach socket path</text></g><g stroke-linecap="round" transform="translate(835.3518795437283 169.83329010009766) rotate(0 147 42.5)"><path d="M21.25 0 M21.25 0 C82.49 -1.78, 141.93 -0.47, 272.75 0 M21.25 0 C92.49 2.14, 163.07 2.39, 272.75 0 M272.75 0 C285.28 1.47, 292.6 8.31, 294 21.25 M272.75 0 C284.94 2.16, 292.99 5.01, 294 21.25 M294 21.25 C294.43 29.9, 291.88 41.25, 294 63.75 M294 21.25 C293.92 35.61, 293.91 51.62, 294 63.75 M294 63.75 C295.59 78.77, 287.85 83.3, 272.75 85 M294 63.75 C295.57 79.39, 286.48 82.82, 272.75 85 M272.75 85 C202.88 87.1, 130.72 85.83, 21.25 85 M272.75 85 C175.41 86.87, 77.57 86.3, 21.25 85 M21.25 85 C8.79 85, -0.61 79.28, 0 63.75 M21.25 85 C7.17 84.43, -0.79 79.56, 0 63.75 M0 63.75 C1.7 46.22, 1.62 31.99, 0 21.25 M0 63.75 C0.57 51.27, -0.75 39.44, 0 21.25 M0 21.25 C1.47 8.86, 6.33 1.07, 21.25 0 M0 21.25 C-2.18 4.83, 5.08 1.8, 21.25 0" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(842.4419979519314 187.33329010009766) rotate(0 139.90988159179688 25)"><text x="139.90988159179688" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">create local unix socket for</text><text x="139.90988159179688" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">attach: attach_socket_fd</text></g><g stroke-linecap="round" transform="translate(10 10) rotate(0 659.9258939954971 210.0000152587891)"><path d="M32 0 M32 0 C362.6 -0.29, 692.78 -0.17, 1287.85 0 M32 0 C352.2 -2.87, 673.05 -2.9, 1287.85 0 M1287.85 0 C1310.24 -0.4, 1319.35 12.27, 1319.85 32 M1287.85 0 C1311.44 -0.13, 1320.26 8.46, 1319.85 32 M1319.85 32 C1317.97 156.81, 1319.24 279.87, 1319.85 388 M1319.85 32 C1318.57 150.52, 1318.76 270.58, 1319.85 388 M1319.85 388 C1319.38 410.21, 1307.35 419.45, 1287.85 420 M1319.85 388 C1319.89 411.06, 1310.79 420.26, 1287.85 420 M1287.85 420 C877.53 416.36, 466.94 416.41, 32 420 M1287.85 420 C875.87 419.82, 464.61 419.4, 32 420 M32 420 C9.98 418.93, -1.59 410.58, 0 388 M32 420 C8.51 420.77, 0.37 411.4, 0 388 M0 388 C-0.76 309.45, -2.75 232.33, 0 32 M0 388 C-0.49 279.35, -0.56 169.84, 0 32 M0 32 C0.78 8.74, 10.9 -0.14, 32 0 M0 32 C-1.17 11.57, 10.26 0.7, 32 0" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(63.55543178982202 66.44442918565551) rotate(0 75.12992858886719 12.5)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">runtime_create</text></g><g stroke-linecap="round" transform="translate(230.8887752956814 50.277780320909415) rotate(0 62 39.5)"><path d="M56.8 -0.87 C66.27 -1.89, 78.88 1.18, 88.36 4.06 C97.84 6.94, 107.91 11.09, 113.7 16.4 C119.49 21.71, 122.52 29.23, 123.1 35.94 C123.67 42.64, 121.68 50.76, 117.16 56.64 C112.63 62.51, 104.49 67.53, 95.95 71.19 C87.42 74.86, 76.14 78.06, 65.94 78.62 C55.75 79.17, 44.02 77.33, 34.78 74.5 C25.54 71.68, 16.37 66.92, 10.52 61.68 C4.67 56.45, 0.29 49.54, -0.33 43.09 C-0.94 36.64, 2.4 29.05, 6.81 22.98 C11.23 16.92, 17.08 10.65, 26.16 6.69 C35.23 2.74, 54.71 0.21, 61.27 -0.76 C67.83 -1.73, 65.44 0.12, 65.51 0.85 M47.6 1.38 C56.98 -0.73, 69.64 -1.02, 79.53 1.05 C89.42 3.12, 99.62 8.61, 106.92 13.81 C114.21 19, 121.14 26.07, 123.33 32.22 C125.52 38.38, 123.3 44.62, 120.07 50.76 C116.84 56.9, 111.23 64.36, 103.94 69.06 C96.66 73.77, 86.54 77.69, 76.36 78.97 C66.17 80.26, 52.7 78.7, 42.84 76.76 C32.98 74.81, 24.35 72.09, 17.2 67.3 C10.06 62.51, 2.22 54.81, -0.03 48.04 C-2.28 41.27, 0.26 33.04, 3.7 26.7 C7.14 20.36, 12.91 14.49, 20.64 9.98 C28.37 5.47, 45.43 0.96, 50.09 -0.38 C54.75 -1.72, 48.36 0.93, 48.6 1.93" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(264.7581844641662 77.34706246404079) rotate(0 28.28997039794922 12.5)"><text x="28.28997039794922" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">isulad</text></g><g stroke-linecap="round"><g transform="translate(295.1511322352708 133.749971177843) rotate(0 -0.4685050245399225 22.33593741452205)"><path d="M0.77 0.35 C0.64 7.8, -1.18 37.36, -1.53 44.73 M-0.28 -0.5 C0.05 7.15, 1.2 38.58, 0.87 46.29" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(295.1511322352708 133.749971177843) rotate(0 -0.4685050245399225 22.33593741452205)"><path d="M-6.25 24.33 C-4.9 29.65, -3.35 34.63, 1.84 46.01 M-6.66 25.81 C-5.31 31.98, -2.78 36.35, 0.58 46.25" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(295.1511322352708 133.749971177843) rotate(0 -0.4685050245399225 22.33593741452205)"><path d="M9.03 24.28 C6.73 29.72, 4.62 34.7, 1.84 46.01 M8.62 25.76 C5.7 31.88, 3.94 36.26, 0.58 46.25" stroke="#1e1e1e" stroke-width="1" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(886.518566555447 50.666648864746094) rotate(0 81.5 35)"><path d="M48.54 2.48 C59.22 -0.17, 75.4 -0.82, 88.08 -0.35 C100.76 0.12, 113.88 2.47, 124.65 5.29 C135.41 8.1, 146.32 11.77, 152.66 16.54 C159.01 21.3, 162.14 28.46, 162.73 33.88 C163.32 39.31, 161.33 44.34, 156.18 49.11 C151.03 53.88, 142.08 59.18, 131.85 62.5 C121.62 65.83, 107.57 68.23, 94.8 69.06 C82.03 69.89, 67.07 69.01, 55.22 67.49 C43.37 65.97, 32.35 63.81, 23.71 59.94 C15.07 56.07, 7.11 49.44, 3.36 44.27 C-0.39 39.1, -1.13 34.21, 1.2 28.91 C3.52 23.61, 7.26 17.21, 17.3 12.46 C27.33 7.71, 52.05 2.47, 61.41 0.43 C70.78 -1.61, 73.12 -0.36, 73.5 0.22 M60.37 0.25 C71.37 -1.76, 85.04 0.45, 97.05 1.55 C109.06 2.65, 122.62 3.67, 132.44 6.84 C142.26 10.01, 150.97 15.15, 155.94 20.55 C160.92 25.95, 163.24 33.65, 162.29 39.25 C161.35 44.84, 156.98 49.98, 150.28 54.1 C143.59 58.22, 133.29 61.49, 122.12 63.97 C110.95 66.45, 95.95 68.75, 83.26 68.98 C70.57 69.22, 57.2 67.46, 45.96 65.36 C34.72 63.27, 23.35 60.54, 15.81 56.41 C8.28 52.28, 2.73 45.88, 0.77 40.57 C-1.18 35.27, -0.04 29.53, 4.08 24.59 C8.19 19.65, 16.33 14.76, 25.47 10.95 C34.6 7.13, 53.12 3.15, 58.89 1.71 C64.65 0.27, 59.78 1.89, 60.06 2.31" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(916.5994240083724 72.91791152321696) rotate(0 51.289939880371094 12.5)"><text x="51.289939880371094" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">isulad-shim</text></g><g stroke-linecap="round"><g transform="translate(968.1061197051379 127.23003834612894) rotate(0 0.6368326392942549 20.80162587698436)"><path d="M-0.91 -0.54 C-0.93 6.46, 0.04 34.24, 0.58 41.27 M0.81 1.8 C1.13 9.03, 2.71 35.76, 2.92 42.63" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(968.1061197051379 127.23003834612894) rotate(0 0.6368326392942549 20.80162587698436)"><path d="M-5.99 22.27 C-2.92 32, 0.59 37.44, 3.15 40.75 M-5.89 23.22 C-3.11 27.55, -1.6 31.81, 3.47 42.75" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(968.1061197051379 127.23003834612894) rotate(0 0.6368326392942549 20.80162587698436)"><path d="M8.23 21.61 C6.26 31.46, 4.74 37.14, 3.15 40.75 M8.33 22.55 C7.95 27.01, 6.31 31.41, 3.47 42.75" stroke="#1e1e1e" stroke-width="1" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(1050.037889287901 89.54711413213465) rotate(0 56.38128423578627 -0.771309788893916)"><path d="M0.66 0.8 C19.42 0.58, 94.21 -1.45, 112.76 -1.81 M-0.45 0.18 C18.14 0.17, 92.66 -0.39, 111.81 -0.41" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(1050.037889287901 89.54711413213465) rotate(0 56.38128423578627 -0.771309788893916)"><path d="M85 9.97 C92.14 5.19, 101.76 3.82, 112.82 -0.61 M84.34 10.67 C92.08 6.81, 101.18 2.71, 112.09 -0.82" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(1050.037889287901 89.54711413213465) rotate(0 56.38128423578627 -0.771309788893916)"><path d="M84.91 -10.55 C92 -8.87, 101.65 -3.76, 112.82 -0.61 M84.24 -9.85 C91.99 -7.1, 101.11 -4.59, 112.09 -0.82" stroke="#1e1e1e" stroke-width="1" fill="none"/></g></g><mask/><g transform="translate(1047.518536037869 58.83331298828125) rotate(0 34.819976806640625 12.5)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">manage</text></g><g stroke-linecap="round" transform="translate(1163.6851009792751 24.666709899902344) rotate(0 76 60)"><path d="M70.74 0.59 C81.01 -0.84, 94.62 1.79, 105.04 4.93 C115.46 8.06, 126.04 13.06, 133.28 19.4 C140.52 25.74, 145.47 34.47, 148.47 42.97 C151.47 51.48, 153.05 61.82, 151.26 70.43 C149.46 79.04, 143.99 87.72, 137.7 94.64 C131.4 101.55, 123 107.62, 113.5 111.91 C104 116.2, 91.82 119.86, 80.7 120.37 C69.58 120.88, 57.13 118.17, 46.78 114.95 C36.43 111.73, 25.87 107.42, 18.61 101.05 C11.36 94.67, 6.36 85.32, 3.27 76.7 C0.18 68.08, -1.73 57.91, 0.1 49.32 C1.92 40.72, 7.63 32.11, 14.23 25.13 C20.83 18.15, 29.41 11.64, 39.69 7.45 C49.97 3.26, 69.39 1.09, 75.9 -0.03 C82.41 -1.15, 78.72 -0.16, 78.74 0.72 M41.29 6.22 C50.27 1.61, 63.12 1.69, 73.89 1.34 C84.65 0.99, 95.74 0.63, 105.9 4.14 C116.05 7.64, 127.76 15.7, 134.82 22.38 C141.87 29.07, 145.55 35.96, 148.21 44.26 C150.88 52.57, 152.91 63.66, 150.81 72.21 C148.71 80.76, 142.42 88.78, 135.61 95.56 C128.81 102.34, 119.65 108.88, 109.98 112.89 C100.31 116.9, 88.35 119.27, 77.6 119.63 C66.86 119.98, 55.56 118.61, 45.52 115.03 C35.47 111.46, 24.42 105.02, 17.32 98.18 C10.23 91.33, 5.82 82.29, 2.92 73.98 C0.02 65.67, -2.26 56.59, -0.06 48.32 C2.14 40.05, 9.48 31.37, 16.14 24.35 C22.8 17.33, 35.8 8.73, 39.9 6.19 C44 3.65, 39.99 7.95, 40.74 9.1" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(1195.2650234508944 47.24030302870949) rotate(0 44.179962158203125 37.5)"><text x="44.179962158203125" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">container</text><text x="44.179962158203125" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">first </text><text x="44.179962158203125" y="50" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">process</text></g><g stroke-linecap="round"><g transform="translate(970.1851925320095 257.66666412353516) rotate(0 -0.33331298828125 26.333343505859375)"><path d="M-1.09 -0.47 C-1.01 8.05, -0.4 43.33, -0.41 52.26 M0.53 -1.76 C0.57 7.44, -0.77 44.8, -1.23 53.58" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(970.1851925320095 257.66666412353516) rotate(0 -0.33331298828125 26.333343505859375)"><path d="M-9.97 28.9 C-7.87 33.29, -4.75 39.45, -0.23 52.78 M-8.65 29.47 C-6.57 37.52, -2.55 48.6, -1.06 52.69" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(970.1851925320095 257.66666412353516) rotate(0 -0.33331298828125 26.333343505859375)"><path d="M8.03 29.65 C6.12 33.71, 5.24 39.7, -0.23 52.78 M9.35 30.22 C4.28 37.95, 1.15 48.74, -1.06 52.69" stroke="#1e1e1e" stroke-width="1" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(844.1851925320095 312.00000762939453) rotate(0 147 39)"><path d="M19.5 0 M19.5 0 C85.94 1.49, 155.91 -1.04, 274.5 0 M19.5 0 C82.57 -0.57, 146.02 -0.4, 274.5 0 M274.5 0 C286.61 -0.93, 292.32 4.52, 294 19.5 M274.5 0 C287.73 -0.61, 292.32 7.37, 294 19.5 M294 19.5 C292.87 33.12, 292.46 46.81, 294 58.5 M294 19.5 C293.56 35.2, 294.17 49.94, 294 58.5 M294 58.5 C294.34 70.75, 286.68 78.52, 274.5 78 M294 58.5 C291.79 69.73, 287.84 77.96, 274.5 78 M274.5 78 C194.25 76.89, 115 76.57, 19.5 78 M274.5 78 C202.9 77.47, 132.2 76.08, 19.5 78 M19.5 78 C8.16 76.13, 1.18 71.96, 0 58.5 M19.5 78 C4.28 79.63, 0.26 72.88, 0 58.5 M0 58.5 C1.27 50.37, -1.85 41.03, 0 19.5 M0 58.5 C-0.05 44.04, -0.23 29.01, 0 19.5 M0 19.5 C-0.72 8.12, 7.27 0.78, 19.5 0 M0 19.5 C2.19 6.72, 6.95 -0.9, 19.5 0" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(853.2952999538845 326.00000762939453) rotate(0 137.889892578125 25)"><text x="137.889892578125" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">add attach_socket_fd to </text><text x="137.889892578125" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">epoll loop</text></g><g stroke-linecap="round" transform="translate(128.8887752956814 151.999971177843) rotate(0 151.66665649414062 753.6667353312175)"><path d="M32 0 M32 0 C104.94 -1.16, 178.95 0.83, 271.33 0 M32 0 C114.64 -0.09, 197.29 -1.28, 271.33 0 M271.33 0 C292.25 -0.85, 304.84 10.82, 303.33 32 M271.33 0 C294.58 0.7, 301.5 10.75, 303.33 32 M303.33 32 C300.45 333.21, 300.24 634.28, 303.33 1475.33 M303.33 32 C299.14 321.56, 299.5 610.9, 303.33 1475.33 M303.33 1475.33 C301.83 1497.97, 294.62 1507.21, 271.33 1507.33 M303.33 1475.33 C303.99 1495.31, 294.55 1507.01, 271.33 1507.33 M271.33 1507.33 C221.32 1506.39, 173.8 1507.63, 32 1507.33 M271.33 1507.33 C205.77 1506.79, 140.37 1506.15, 32 1507.33 M32 1507.33 C12.14 1506.21, -1.29 1496.02, 0 1475.33 M32 1507.33 C8.86 1505.47, 0.07 1495.95, 0 1475.33 M0 1475.33 C1.51 1022.96, 2.19 571.86, 0 32 M0 1475.33 C-4.7 1002.55, -4.49 529.38, 0 32 M0 32 C1.14 12.33, 10.78 1.87, 32 0 M0 32 C0.41 10.67, 9.07 -1.01, 32 0" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g stroke-linecap="round" transform="translate(176.22211880154077 485.4443834092883) rotate(0 108 30)"><path d="M15 0 M15 0 C76.86 1.65, 135.45 -1.03, 201 0 M15 0 C69.18 -0.43, 121.65 -0.98, 201 0 M201 0 C212.13 -0.28, 215.61 6.99, 216 15 M201 0 C208.74 -1.02, 215.39 5.95, 216 15 M216 15 C216.86 22.24, 217.5 31.26, 216 45 M216 15 C217.05 27.42, 215.87 39.23, 216 45 M216 45 C214.83 56.15, 210.05 59.7, 201 60 M216 45 C215.96 54.04, 210.02 58.55, 201 60 M201 60 C158.83 62.68, 114.77 60.2, 15 60 M201 60 C146.3 57.76, 90.53 57.6, 15 60 M15 60 C4.1 61.77, -0.67 55.56, 0 45 M15 60 C3.38 59.3, -2.13 55.84, 0 45 M0 45 C1.24 32.59, 0.38 22.05, 0 15 M0 45 C-0.67 35.72, 0 24.81, 0 15 M0 15 C-1.85 3.99, 6.44 -0.6, 15 0 M0 15 C-1.45 2.72, 2.95 0.42, 15 0" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(200.55218929714624 502.9443834092883) rotate(0 83.66992950439453 12.5)"><text x="83.66992950439453" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">rt_isula_attach</text></g><g mask="url(#mask-8aC1eihLyJ15DJRIEQ4pM)" stroke-linecap="round"><g transform="translate(394.1851721869573 635.5245647006168) rotate(0 225.36507879105667 -76.71889222750684)"><path d="M1.17 1.16 C76.42 -24.56, 376.67 -128.13, 451.53 -153.71 M0.32 0.72 C75.49 -24.88, 376.16 -126.51, 451 -152.31" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(394.1851721869573 635.5245647006168) rotate(0 225.36507879105667 -76.71889222750684)"><path d="M429.55 -132.17 C434.05 -140.03, 443.34 -146.9, 451.5 -151.31 M427.6 -133.24 C433.99 -139.46, 440.02 -144.36, 450.82 -152.98" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(394.1851721869573 635.5245647006168) rotate(0 225.36507879105667 -76.71889222750684)"><path d="M422.93 -151.6 C429.5 -153.73, 440.76 -154.82, 451.5 -151.31 M420.98 -152.67 C429.05 -153.77, 436.84 -153.52, 450.82 -152.98" stroke="#1e1e1e" stroke-width="1" fill="none"/></g></g><mask id="mask-8aC1eihLyJ15DJRIEQ4pM"><rect x="0" y="0" fill="#fff" width="944.9153297690707" height="888.9623491556305"/><rect x="583.5102805800648" y="546.30567247311" fill="#000" width="72.07994079589844" height="25" opacity="1"/></mask><g transform="translate(583.5102805800648 546.30567247311) rotate(0 36.59855265921911 12.943649178370833)"><text x="36.03997039794922" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">connect</text></g><g stroke-linecap="round" transform="translate(784.5184750027126 138) rotate(0 208.66668701171875 756.7499618530275)"><path d="M32 0 M32 0 C147.41 1.69, 261.6 0.18, 385.33 0 M32 0 C153.11 -0.85, 273.91 -0.32, 385.33 0 M385.33 0 C407.01 -0.02, 419.01 9.65, 417.33 32 M385.33 0 C406.57 0.33, 418.23 11.77, 417.33 32 M417.33 32 C414.86 592.5, 414.53 1152.89, 417.33 1481.5 M417.33 32 C417.57 542.33, 417.78 1051.61, 417.33 1481.5 M417.33 1481.5 C416.87 1502.15, 408.47 1511.69, 385.33 1513.5 M417.33 1481.5 C418.72 1502.28, 406.42 1515.3, 385.33 1513.5 M385.33 1513.5 C265.38 1512.7, 146.56 1511.46, 32 1513.5 M385.33 1513.5 C252.68 1515.54, 119.96 1515.65, 32 1513.5 M32 1513.5 C9.02 1513.33, -0.6 1503.05, 0 1481.5 M32 1513.5 C8.94 1513.93, -0.18 1501.63, 0 1481.5 M0 1481.5 C-1.5 1028.74, -1.88 576.32, 0 32 M0 1481.5 C0.75 1010.9, 1.07 541.2, 0 32 M0 32 C-1.68 11.6, 11.98 -1.54, 32 0 M0 32 C2.2 9.83, 8.67 -0.44, 32 0" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g stroke-linecap="round"><g transform="translate(272.22208828396265 543.777787950304) rotate(0 -1.0796190936341077 25.999999999999996)"><path d="M-1.08 -0.25 C-1.67 8.4, -2.73 44.34, -2.79 52.96 M0.56 -1.42 C0.3 7.41, -0.01 42.35, -0.41 51.34" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(272.22208828396265 543.777787950304) rotate(0 -1.0796190936341077 25.999999999999996)"><path d="M-9.12 25.63 C-3.88 36.91, -3.36 42.8, 1.47 52.43 M-9.5 27.25 C-7.83 32.83, -4.27 38.51, -0.02 51.1" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(272.22208828396265 543.777787950304) rotate(0 -1.0796190936341077 25.999999999999996)"><path d="M8.68 26.06 C7.46 37.02, 1.54 42.76, 1.47 52.43 M8.29 27.68 C6.01 33.04, 5.6 38.61, -0.02 51.1" stroke="#1e1e1e" stroke-width="1" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(156.22208828396265 609.777787950304) rotate(0 117 30)"><path d="M15 0 M15 0 C88.38 1.43, 159.49 -0.3, 219 0 M15 0 C75.48 -0.34, 137.72 -0.54, 219 0 M219 0 C230.73 1.77, 234.07 6.05, 234 15 M219 0 C226.88 1.95, 232.17 6.46, 234 15 M234 15 C231.89 24.62, 233.06 33.82, 234 45 M234 15 C233.25 21.14, 234.54 28.53, 234 45 M234 45 C234.71 53.96, 230.88 61.13, 219 60 M234 45 C233.24 56.03, 230.78 58.12, 219 60 M219 60 C146.78 60.13, 73.54 59.45, 15 60 M219 60 C161.34 60.72, 102.87 60.75, 15 60 M15 60 C5.36 59.77, 0.58 56.59, 0 45 M15 60 C3.71 58.68, 1.8 54.24, 0 45 M0 45 C1.3 34.91, 0.8 24.33, 0 15 M0 45 C0.27 36.87, -0.89 29.99, 0 15 M0 15 C1.11 4.92, 3.07 -1.07, 15 0 M0 15 C0.13 4.85, 6.36 1.62, 15 0" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(163.44218868679468 627.277787950304) rotate(0 109.77989959716797 12.5)"><text x="109.77989959716797" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">get_attach_socketfd</text></g><g stroke-linecap="round" transform="translate(848.8519405788845 454.3333206176758) rotate(0 140.5 30)"><path d="M15 0 M15 0 C97.88 -0.32, 183.62 0.75, 266 0 M15 0 C67 -2.23, 119.01 -1.37, 266 0 M266 0 C274.41 -1.7, 282.79 5.02, 281 15 M266 0 C278.24 -2.02, 281.28 4.74, 281 15 M281 15 C281.8 24, 279.5 35.63, 281 45 M281 15 C280.78 23.04, 279.9 31.39, 281 45 M281 45 C280.45 56.49, 275.25 58.11, 266 60 M281 45 C279.78 54.59, 275.95 58.08, 266 60 M266 60 C187.34 60.09, 111.99 60.18, 15 60 M266 60 C204.58 57.27, 142.22 58.22, 15 60 M15 60 C4.78 61.86, -0.19 54.45, 0 45 M15 60 C5.9 57.99, -2.27 54.03, 0 45 M0 45 C0.89 36.6, -1.85 31.25, 0 15 M0 45 C0.29 34.19, 0.73 23.1, 0 15 M0 15 C-0.74 3.63, 3.39 -0.53, 15 0 M0 15 C-0.69 3.61, 4.22 -0.79, 15 0" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(854.0020565456814 471.8333206176758) rotate(0 135.34988403320312 12.5)"><text x="135.34988403320312" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">do_attach_socket_accept</text></g><g stroke-linecap="round"><g transform="translate(970.6741785083344 517.1666641235352) rotate(0 0.6059008275641418 32.83332824707031)"><path d="M1.11 0.37 C1.61 11.49, 1.91 54.88, 1.83 65.87 M0.24 -0.49 C0.73 10.32, 1.32 53.05, 1.21 63.85" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(970.6741785083344 517.1666641235352) rotate(0 0.6059008275641418 32.83332824707031)"><path d="M-8.56 36.74 C-6.91 40.47, -4.02 47.42, 2.59 64.13 M-9.58 35.44 C-7.12 43.72, -4.41 49.9, 0.73 63.49" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(970.6741785083344 517.1666641235352) rotate(0 0.6059008275641418 32.83332824707031)"><path d="M11.96 36.65 C9.41 40.59, 8.09 47.55, 2.59 64.13 M10.94 35.35 C8.17 43.84, 5.67 50.05, 0.73 63.49" stroke="#1e1e1e" stroke-width="1" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(839.1852535671658 582.3333206176758) rotate(0 138.5 29)"><path d="M14.5 0 M14.5 0 C83.44 1.58, 150.53 1.07, 262.5 0 M14.5 0 C109.35 1.14, 205.29 0.35, 262.5 0 M262.5 0 C271.23 0.73, 277.09 6.41, 277 14.5 M262.5 0 C269.91 -1.85, 278.21 3.94, 277 14.5 M277 14.5 C279.08 21.89, 277.71 31.86, 277 43.5 M277 14.5 C276.97 23.44, 277.15 32.25, 277 43.5 M277 43.5 C277.87 52.09, 271.53 59.49, 262.5 58 M277 43.5 C277.17 51.07, 271.52 56.26, 262.5 58 M262.5 58 C163.94 57.21, 69.3 56, 14.5 58 M262.5 58 C198.73 57.65, 135.61 56.09, 14.5 58 M14.5 58 C6.15 59.55, 1.84 52.32, 0 43.5 M14.5 58 C4.8 59.08, 2.05 54.36, 0 43.5 M0 43.5 C-1 33.18, -0.42 22.18, 0 14.5 M0 43.5 C1.02 36.06, 0.2 28.45, 0 14.5 M0 14.5 C0.51 4.44, 3.53 1.78, 14.5 0 M0 14.5 C1.91 3.87, 6.23 -0.36, 14.5 0" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(849.8953594631619 598.8333206176758) rotate(0 127.7898941040039 12.5)"><text x="127.7898941040039" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">add conn_fd to epoll loop</text></g><g stroke-linecap="round"><g transform="translate(975.3577402210192 644.666633605957) rotate(0 -0.24791416638805686 26.65081008761784)"><path d="M0.29 -0.24 C0.23 8.79, 0.5 44.73, 0.47 53.63 M-1.02 -1.41 C-1.22 7.3, -0.08 42.38, 0.03 51.68" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(975.3577402210192 644.666633605957) rotate(0 -0.24791416638805686 26.65081008761784)"><path d="M-10.05 28.46 C-5.65 36.97, -2.86 46.58, -1.77 50.37 M-10.49 27.79 C-6.69 31.9, -4.9 38.82, 0.43 51.54" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(975.3577402210192 644.666633605957) rotate(0 -0.24791416638805686 26.65081008761784)"><path d="M8.18 28.05 C5.86 36.75, 1.94 46.51, -1.77 50.37 M7.73 27.38 C7.46 31.73, 5.19 38.74, 0.43 51.54" stroke="#1e1e1e" stroke-width="1" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(822.8519405788845 542.3333053588867) rotate(0 159 61.333351135253906)"><path d="M30.67 0 M30.67 0 C131.77 -1.28, 233.02 -0.59, 287.33 0 M287.33 0 C305.84 -0.5, 319.05 11.21, 318 30.67 M318 30.67 C318.63 49.92, 317.11 68.21, 318 92 M318 92 C316.21 112.04, 306.22 120.8, 287.33 122.67 M287.33 122.67 C219.78 121.68, 155.64 122.72, 30.67 122.67 M30.67 122.67 C10 120.79, -1.86 111.32, 0 92 M0 92 C1.86 66.25, 0.83 43.03, 0 30.67 M0 30.67 C-0.8 9.92, 8.55 -1.75, 30.67 0" stroke="#1e1e1e" stroke-width="1.5" fill="none" stroke-dasharray="8 9"/></g><g stroke-linecap="round"><g transform="translate(268.22208828396265 671.1111009385852) rotate(0 0.7777913411458286 58.55559285481773)"><path d="M0.62 0.41 C0.69 19.91, 1 96.73, 1 116.36 M-0.52 -0.42 C-0.7 19.28, -0.83 98.02, -0.24 117.5" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(268.22208828396265 671.1111009385852) rotate(0 0.7777913411458286 58.55559285481773)"><path d="M-10.15 88.51 C-8.58 97.52, -1.86 110.44, 1.12 118.61 M-9.96 89.21 C-7.15 97.58, -4.25 106.03, -0.13 117.85" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(268.22208828396265 671.1111009385852) rotate(0 0.7777913411458286 58.55559285481773)"><path d="M10.37 88.27 C4.84 97.47, 4.46 110.48, 1.12 118.61 M10.56 88.97 C7.27 97.56, 4.07 106.08, -0.13 117.85" stroke="#1e1e1e" stroke-width="1" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(151.3887651231554 798.6666429307727) rotate(0 119.5 30)"><path d="M15 0 M15 0 C87.05 -0.03, 156.36 1.86, 224 0 M15 0 C93.42 2.1, 171.09 1.83, 224 0 M224 0 C234.89 1.86, 237.63 5.7, 239 15 M224 0 C233.2 0.9, 239.54 5.31, 239 15 M239 15 C238.33 24.23, 239.77 34.82, 239 45 M239 15 C239.61 24.18, 237.82 34, 239 45 M239 45 C239.93 54.07, 232.73 58.88, 224 60 M239 45 C238.78 55.34, 232.18 60.35, 224 60 M224 60 C169.59 59.73, 117.73 59.9, 15 60 M224 60 C163.1 58.92, 103.43 59.4, 15 60 M15 60 C4.11 60.75, 0.69 53.76, 0 45 M15 60 C3.51 62.02, 2.23 55.85, 0 45 M0 45 C1.44 35.39, 0.43 24.46, 0 15 M0 45 C0.86 34.94, 0.51 23.05, 0 15 M0 15 C0.43 5.68, 5.2 0.7, 15 0 M0 15 C0.37 6.34, 4.89 0.36, 15 0" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(158.74885728624133 803.6666429307727) rotate(0 112.13990783691406 25)"><text x="112.13990783691406" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">write attach isulad fd</text><text x="112.13990783691406" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">to socket fd</text></g><g stroke-linecap="round"><g transform="translate(262.60041869976146 864.6666429307727) rotate(0 -0.6948600605093844 34.22223917643228)"><path d="M0.11 0.66 C-0.28 11.91, -1.81 56.5, -1.92 67.91 M-1.29 -0.05 C-1.92 11.33, -3.22 57.96, -3.14 69.16" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(262.60041869976146 864.6666429307727) rotate(0 -0.6948600605093844 34.22223917643228)"><path d="M-11.92 39.96 C-10.48 51.85, -5.1 60.7, -4.49 69.59 M-12.85 41.42 C-9.73 50.42, -5.76 60.11, -3.08 69.27" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(262.60041869976146 864.6666429307727) rotate(0 -0.6948600605093844 34.22223917643228)"><path d="M8.6 40.24 C3.11 52.06, 1.56 60.81, -4.49 69.59 M7.67 41.7 C3.93 50.68, 1.04 60.28, -3.08 69.27" stroke="#1e1e1e" stroke-width="1" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(155.99979824490003 935.1111212836372) rotate(0 113 30)"><path d="M15 0 M15 0 C76.06 -1.59, 140.21 -3.21, 211 0 M15 0 C70.45 0.01, 126.07 -1.42, 211 0 M211 0 C220.43 1.54, 224.88 3.28, 226 15 M211 0 C221.97 2.02, 226.91 4.22, 226 15 M226 15 C224.28 23.18, 225.95 26.42, 226 45 M226 15 C226.97 23.14, 225.46 32.12, 226 45 M226 45 C227.83 55.38, 220.72 58.85, 211 60 M226 45 C227.76 55.46, 223.13 59.75, 211 60 M211 60 C161.54 56.89, 114.06 58.28, 15 60 M211 60 C158.44 58.99, 104.21 60.07, 15 60 M15 60 C5.56 59.93, 0.23 53.1, 0 45 M15 60 C3.66 58.73, 2.22 56.92, 0 45 M0 45 C0.08 37.2, 1.58 28.39, 0 15 M0 45 C1.23 35.85, 0.19 25.19, 0 15 M0 15 C-1.09 3.44, 6.3 0.93, 15 0 M0 15 C-0.9 4.87, 3.82 -1.84, 15 0" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(162.50988430447035 940.1111212836372) rotate(0 106.48991394042969 25)"><text x="106.48991394042969" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">get_container_attac</text><text x="106.48991394042969" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">h_exitcode</text></g><g stroke-linecap="round" transform="translate(147.55543178982202 580.4444444444446) rotate(0 131.00001525878906 231.888910929362)"><path d="M32 0 M32 0 C100.07 -3.5, 171.55 -3.46, 230 0 M230 0 C252.31 -0.48, 260.32 8.83, 262 32 M262 32 C261.08 126.87, 262.01 222.71, 262 431.78 M262 431.78 C263.74 451.45, 250.49 463.76, 230 463.78 M230 463.78 C181.25 460.54, 132.68 461.56, 32 463.78 M32 463.78 C11.13 464.53, 0.31 452.88, 0 431.78 M0 431.78 C0.04 350.19, -1.85 269.55, 0 32 M0 32 C-0.37 12.12, 10.46 -0.86, 32 0" stroke="#1e1e1e" stroke-width="1.5" fill="none" stroke-dasharray="8 9"/></g><g stroke-linecap="round" transform="translate(869.5184444851345 700.3333511352539) rotate(0 103 28)"><path d="M14 0 M14 0 C83.47 -1.47, 150.3 -1.31, 192 0 M14 0 C58.74 -2.02, 102.89 -0.72, 192 0 M192 0 C201.82 1.76, 205.78 5.14, 206 14 M192 0 C200.84 -0.22, 206.46 6.7, 206 14 M206 14 C204.85 22.79, 204.45 32.08, 206 42 M206 14 C205.99 21.6, 205.4 32.01, 206 42 M206 42 C205.18 51.71, 199.72 54.98, 192 56 M206 42 C204.45 50.97, 201.96 56.53, 192 56 M192 56 C125.9 57.09, 58 54.61, 14 56 M192 56 C139.15 54.69, 85.95 54.71, 14 56 M14 56 C2.95 54.6, 0.27 51.52, 0 42 M14 56 C6.94 58.23, 1.35 50.86, 0 42 M0 42 C-0.78 36.15, 1.41 30, 0 14 M0 42 C-1.14 35.4, -0.71 30.09, 0 14 M0 14 C0.19 5.57, 2.78 0.52, 14 0 M0 14 C-0.5 5.67, 3.19 -0.86, 14 0" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(919.5584835476345 715.8333511352539) rotate(0 52.9599609375 12.5)"><text x="52.9599609375" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">attach_cb</text></g><g stroke-linecap="round"><g transform="translate(971.518566555447 759.6666641235352) rotate(0 0.666656494140625 24.666656494140625)"><path d="M-0.53 0.19 C-0.56 8.13, 0.54 40.16, 0.8 48.44 M1.39 -0.75 C1.08 7.23, -0.22 41.41, -0.44 49.51" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(971.518566555447 759.6666641235352) rotate(0 0.666656494140625 24.666656494140625)"><path d="M-7.78 25.16 C-6.51 34.45, -1.19 43.66, 0.34 49.51 M-8.2 26.55 C-5.15 33.02, -2.75 39.42, 0 50.41" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(971.518566555447 759.6666641235352) rotate(0 0.666656494140625 24.666656494140625)"><path d="M9.09 25.72 C3.95 34.72, 2.88 43.72, 0.34 49.51 M8.67 27.11 C6.83 33.3, 4.34 39.54, 0 50.41" stroke="#1e1e1e" stroke-width="1" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(868.5185360378689 814.9999771118164) rotate(0 110 30)"><path d="M15 0 M15 0 C74.25 -1.64, 133.85 -0.27, 205 0 M15 0 C73.46 0.34, 132.1 0.19, 205 0 M205 0 C214.27 1.1, 219.69 5.97, 220 15 M205 0 C214.49 -0.98, 218.27 3.87, 220 15 M220 15 C217.82 22.37, 218.42 28.04, 220 45 M220 15 C220.16 23.65, 220.57 31.27, 220 45 M220 45 C221.22 54.74, 215.63 58.56, 205 60 M220 45 C218.96 57.15, 215.3 61.81, 205 60 M205 60 C164.96 62.69, 127.97 60.62, 15 60 M205 60 C139.13 62.21, 71.2 62.17, 15 60 M15 60 C4.99 59.28, -1.72 56.13, 0 45 M15 60 C3.26 61.43, 0.12 56.39, 0 45 M0 45 C1.61 37.74, 1.36 33.55, 0 15 M0 45 C0.04 37.09, 0.56 28.99, 0 15 M0 15 C-1.46 6.86, 4.1 1.6, 15 0 M0 15 C-0.4 5.01, 6.51 -0.46, 15 0" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(876.5186199612087 819.9999771118164) rotate(0 101.99991607666016 25)"><text x="101.99991607666016" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">read isulad fd from </text><text x="101.99991607666016" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">attach conn fd</text></g><g mask="url(#mask-kyQXnKVifMgMx2IeOowxQ)" stroke-linecap="round"><g transform="translate(396.5185156928168 821.9271925032234) rotate(0 231.7777608235678 16.400393930815937)"><path d="M-1.19 -0.05 C76.13 5.07, 386.39 26.29, 464.04 31.71 M0.39 -1.13 C77.58 4.03, 385.99 26.89, 463.35 32.67" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(396.5185156928168 821.9271925032234) rotate(0 231.7777608235678 16.400393930815937)"><path d="M434.38 41.63 C441.63 40.47, 451.91 35.85, 464.38 33.23 M434.24 41.48 C445.35 37.16, 455.44 35, 462.59 33.29" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(396.5185156928168 821.9271925032234) rotate(0 231.7777608235678 16.400393930815937)"><path d="M435.9 21.16 C442.84 26.41, 452.64 28.19, 464.38 33.23 M435.76 21.01 C446.48 24.47, 456 30.07, 462.59 33.29" stroke="#1e1e1e" stroke-width="1" fill="none"/></g></g><mask id="mask-kyQXnKVifMgMx2IeOowxQ"><rect x="0" y="0" fill="#fff" width="960.0740373399524" height="954.7279803648553"/><rect x="601.8063015407986" y="825.8275864340393" fill="#000" width="52.979949951171875" height="25" opacity="1"/></mask><g transform="translate(601.8063015407986 825.8275864340394) rotate(0 26.13931778706609 11.87186141544953)"><text x="26.489974975585938" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">event</text></g><g stroke-linecap="round"><g transform="translate(983.7272698787762 878.4999771118164) rotate(0 1.7944079541773021 20.69311233120615)"><path d="M-0.82 -0.01 C0.1 6.61, 3.86 33.59, 4.53 40.31 M0.94 -1.06 C1.89 5.62, 3.87 34.27, 4.08 41.29" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(983.7272698787762 878.4999771118164) rotate(0 1.7944079541773021 20.69311233120615)"><path d="M-4.08 23.76 C-4.02 25.67, -2.29 30.9, 3.15 39.9 M-4.27 22.71 C-1.68 26.16, -0.21 31.43, 3.97 40.36" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(983.7272698787762 878.4999771118164) rotate(0 1.7944079541773021 20.69311233120615)"><path d="M10.1 22.99 C7.13 24.97, 5.83 30.36, 3.15 39.9 M9.92 21.94 C9.1 25.48, 7.16 30.93, 3.97 40.36" stroke="#1e1e1e" stroke-width="1" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(790.8520321316189 918.3332901000977) rotate(0 194 85)"><path d="M243.75 21.5 M243.75 21.5 C268.2 31.29, 292.97 42.38, 339.25 64.5 M243.75 21.5 C267.03 31.13, 288.27 41.41, 339.25 64.5 M339.25 64.5 C389.9 84.76, 386.79 87.68, 339.25 107.5 M339.25 64.5 C387.42 85.15, 386.3 84.4, 339.25 107.5 M339.25 107.5 C304.45 124.45, 268.4 139.95, 243.75 148.5 M339.25 107.5 C319.17 116.33, 297.94 125.93, 243.75 148.5 M243.75 148.5 C196.51 170.56, 193.66 170.85, 146.25 148.5 M243.75 148.5 C197.24 168.64, 194.23 169.58, 146.25 148.5 M146.25 148.5 C118.64 137.86, 95.23 125.96, 48.75 107.5 M146.25 148.5 C126.54 140.27, 106.21 131.77, 48.75 107.5 M48.75 107.5 C1.97 87.17, -1.91 85.94, 48.75 64.5 M48.75 107.5 C-0.29 83.88, -2.14 84.03, 48.75 64.5 M48.75 64.5 C82.27 53.06, 113.91 37.13, 146.25 21.5 M48.75 64.5 C80.08 50.93, 112.06 36.62, 146.25 21.5 M146.25 21.5 C193.93 -0.69, 193.18 1.5, 243.75 21.5 M146.25 21.5 C196.56 0.89, 193.56 -2.27, 243.75 21.5" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(896.9721188015408 965.8332901000977) rotate(0 87.87991333007812 37.5)"><text x="87.87991333007812" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">attach </text><text x="87.87991333007812" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">connections nums </text><text x="87.87991333007812" y="50" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">exceeds the limit?</text></g><g mask="url(#mask-kbnyolq-Z74LBQP3fwup-)" stroke-linecap="round"><g transform="translate(797.692819152215 1001.4357817134754) rotate(0 -77.34841863156252 -18.162576430413935)"><path d="M0.04 -0.92 C-25.9 -6.91, -128.96 -29.61, -154.63 -35.4 M-1.4 1.21 C-27.58 -5.15, -130.04 -30.57, -155.54 -37.03" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(797.692819152215 1001.4357817134754) rotate(0 -77.34841863156252 -18.162576430413935)"><path d="M-127.24 -40.02 C-135.61 -37.73, -147.44 -36.11, -154.4 -36.1 M-125.49 -39.83 C-136 -39.1, -145.66 -37.59, -156.14 -36.12" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(797.692819152215 1001.4357817134754) rotate(0 -77.34841863156252 -18.162576430413935)"><path d="M-132.22 -20.12 C-138.76 -25.48, -148.68 -31.52, -154.4 -36.1 M-130.47 -19.93 C-139.39 -25.59, -147.44 -30.52, -156.14 -36.12" stroke="#1e1e1e" stroke-width="1" fill="none"/></g></g><mask id="mask-kbnyolq-Z74LBQP3fwup-"><rect x="0" y="0" fill="#fff" width="1052.38965641534" height="1137.7609345743033"/><rect x="704.7544194415509" y="970.7732052830614" fill="#000" width="31.179962158203125" height="25" opacity="1"/></mask><g transform="translate(704.7544194415509 970.7732052830615) rotate(0 15.189490863000998 12.751875128094099)"><text x="15.589981079101562" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">yes</text></g><g stroke-linecap="round" transform="translate(493.56632101656123 894.3450039406299) rotate(0 71.5 70.5)"><path d="M32 0 M32 0 C49.53 -0.22, 66.97 -0.63, 111 0 M32 0 C50.76 0.63, 70 0.09, 111 0 M111 0 C130.72 0.79, 142.76 11.26, 143 32 M111 0 C134.59 -1.18, 141.44 10.81, 143 32 M143 32 C144.82 61.9, 144.93 92.55, 143 109 M143 32 C142.87 54.83, 142.08 77.15, 143 109 M143 109 C144.84 129.45, 133.68 142.68, 111 141 M143 109 C144.32 128.17, 132.35 143.28, 111 141 M111 141 C93.67 139.44, 75.27 139.47, 32 141 M111 141 C80.59 139.94, 48.54 140.23, 32 141 M32 141 C12.27 142.08, -0.74 129.02, 0 109 M32 141 C11 139.46, 0.48 131.63, 0 109 M0 109 C1.25 90.42, 0.94 75, 0 32 M0 109 C-0.65 92.15, -1.38 75.67, 0 32 M0 32 C-1.47 11.04, 12.63 1.96, 32 0 M0 32 C-0.66 9.15, 12.29 0.5, 32 0" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(508.35636770845576 927.3450039406299) rotate(0 56.70995330810547 37.5)"><text x="56.70995330810547" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">error exit</text><text x="56.70995330810547" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">write -1 to </text><text x="56.70995330810547" y="50" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">conn fd</text></g><g stroke-linecap="round"><g transform="translate(495.9705765849401 963.8297920068389) rotate(0 -55.00005086263026 -1.1772148664429096)"><path d="M0.13 0.1 C-18.51 -0.44, -92.65 -2.47, -111.13 -2.74 M-1.27 -0.9 C-19.6 -1.31, -91.18 -0.98, -109 -1.4" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(495.9705765849401 963.8297920068389) rotate(0 -55.00005086263026 -1.1772148664429096)"><path d="M-80.57 -13.32 C-91.39 -9.08, -99.4 -4.14, -108.92 -2.36 M-80.36 -12.1 C-86.13 -8.69, -93.79 -6.03, -109.68 -2.01" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(495.9705765849401 963.8297920068389) rotate(0 -55.00005086263026 -1.1772148664429096)"><path d="M-80.73 7.2 C-91.57 4.14, -99.53 1.79, -108.92 -2.36 M-80.52 8.42 C-86.44 7.38, -94.06 5.59, -109.68 -2.01" stroke="#1e1e1e" stroke-width="1" fill="none"/></g></g><mask/><g mask="url(#mask-wjyMIAmeLzfRwQ2MfyjOP)" stroke-linecap="round"><g transform="translate(979.3039099182738 1086.994185804675) rotate(0 -2.842170943040401e-14 44.52889725798309)"><path d="M-0.52 -0.66 C-0.3 14.15, 1.07 74.05, 1.12 88.96 M1.4 1.61 C1.59 16.61, 1.18 75.72, 0.76 90.45" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(979.3039099182738 1086.994185804675) rotate(0 -2.842170943040401e-14 44.52889725798309)"><path d="M-10.15 63.96 C-6.44 69.65, -3.93 76.7, -1.18 89.42 M-9.48 62.72 C-6.27 68.05, -4 75.07, 0.78 90.44" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(979.3039099182738 1086.994185804675) rotate(0 -2.842170943040401e-14 44.52889725798309)"><path d="M10.36 64.29 C8.33 69.79, 5.1 76.75, -1.18 89.42 M11.04 63.04 C9.53 68.15, 7.07 75.09, 0.78 90.44" stroke="#1e1e1e" stroke-width="1" fill="none"/></g></g><mask id="mask-wjyMIAmeLzfRwQ2MfyjOP"><rect x="0" y="0" fill="#fff" width="1079.3039099182738" height="1276.0519803206412"/><rect x="969.0939184631957" y="1119.023083062658" fill="#000" width="20.41998291015625" height="25" opacity="1"/></mask><g transform="translate(969.0939184631957 1119.023083062658) rotate(0 10.672516626681528 12.866832216288913)"><text x="10.209991455078125" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">no</text></g><g stroke-linecap="round" transform="translate(831.415054937805 1180.8297920068392) rotate(0 149.5 36.5)"><path d="M18.25 0 M18.25 0 C107.74 -2.42, 198.81 0.14, 280.75 0 M18.25 0 C120.89 0.19, 225.12 -0.12, 280.75 0 M280.75 0 C294.31 -0.34, 298.52 4.8, 299 18.25 M280.75 0 C292.44 -1.36, 301.14 6.75, 299 18.25 M299 18.25 C298.85 28.99, 298.97 43.61, 299 54.75 M299 18.25 C299.34 32.9, 297.83 45.53, 299 54.75 M299 54.75 C300.29 66.2, 291.03 74.86, 280.75 73 M299 54.75 C300.25 67.02, 294.6 71.4, 280.75 73 M280.75 73 C225.4 73.69, 172.12 73.63, 18.25 73 M280.75 73 C207.24 74.32, 135.57 73.61, 18.25 73 M18.25 73 C4.14 71.56, 0.24 68.47, 0 54.75 M18.25 73 C7.49 70.77, -1.88 67.56, 0 54.75 M0 54.75 C-2.06 42.36, -0.38 33.31, 0 18.25 M0 54.75 C-0.36 44.48, 0.91 35.81, 0 18.25 M0 18.25 C1.13 5.56, 7.16 -0.73, 18.25 0 M0 18.25 C2.19 7.97, 6.46 0.83, 18.25 0" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(842.6951757874144 1204.8297920068392) rotate(0 138.21987915039062 12.5)"><text x="138.21987915039062" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">add_attach_terminal_fifos</text></g><g stroke-linecap="round"><g transform="translate(987.1213272778964 1263.416803044929) rotate(0 3.430411218631434 25.34542575536915)"><path d="M-0.06 0.19 C1.18 8.66, 5.38 42.74, 6.6 51.02 M-1.55 -0.75 C-0.42 7.32, 4.14 40.49, 5.51 49.07" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(987.1213272778964 1263.416803044929) rotate(0 3.430411218631434 25.34542575536915)"><path d="M-6.32 26.12 C-3.23 32.39, -2.34 38.8, 7.35 49.06 M-5.87 27.17 C-5.31 30.3, -1.59 35.82, 6.12 48.8" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(987.1213272778964 1263.416803044929) rotate(0 3.430411218631434 25.34542575536915)"><path d="M10.99 23.57 C10.06 30.43, 6.92 37.43, 7.35 49.06 M11.44 24.63 C8.47 28.29, 8.67 34.33, 6.12 48.8" stroke="#1e1e1e" stroke-width="1" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(833.5262508199664 1330.1076545556675) rotate(0 153 31.5)"><path d="M15.75 0 M15.75 0 C103.92 -2.44, 190.45 -3.43, 290.25 0 M15.75 0 C124.08 0.13, 234.16 0.16, 290.25 0 M290.25 0 C300.45 0.7, 305.24 6.32, 306 15.75 M290.25 0 C301.61 -1.29, 308.03 6.94, 306 15.75 M306 15.75 C305.87 28, 304.75 39.95, 306 47.25 M306 15.75 C306.44 24.7, 306.15 31.31, 306 47.25 M306 47.25 C307.12 59.38, 301.79 62.93, 290.25 63 M306 47.25 C304.84 59.01, 301.52 61.04, 290.25 63 M290.25 63 C194.62 64.88, 97.79 65.09, 15.75 63 M290.25 63 C207.93 62.82, 125.66 63.72, 15.75 63 M15.75 63 C6.33 61.78, 1.6 58.33, 0 47.25 M15.75 63 C4.88 62.86, -1.92 59.46, 0 47.25 M0 47.25 C0.82 38.66, 0.88 29.4, 0 15.75 M0 47.25 C0.72 40.18, 0.75 33.55, 0 15.75 M0 15.75 C0.29 3.52, 4.26 -0.03, 15.75 0 M0 15.75 C1.55 6.32, 3.92 1.62, 15.75 0" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(838.7763728902789 1349.1076545556675) rotate(0 147.7498779296875 12.5)"><text x="147.7498779296875" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">add fifofd_in fd to epoll loop</text></g><g stroke-linecap="round" transform="translate(803.7483882711383 789.3853136539743) rotate(0 186.66666666666674 323.3333333333335)"><path d="M32 0 M32 0 C115.81 1.39, 195.87 2.52, 341.33 0 M341.33 0 C361.4 0.65, 373.68 12.25, 373.33 32 M373.33 32 C374.85 244.08, 374.91 455.74, 373.33 614.67 M373.33 614.67 C372.36 637.53, 362.96 646.28, 341.33 646.67 M341.33 646.67 C228.1 647.69, 115.18 648.85, 32 646.67 M32 646.67 C8.92 648.59, 1.6 635.92, 0 614.67 M0 614.67 C-2.16 384.08, -2.46 154.01, 0 32 M0 32 C-0.93 10.69, 12.08 1.72, 32 0" stroke="#1e1e1e" stroke-width="1.5" fill="none" stroke-dasharray="8 9"/></g><g stroke-linecap="round" transform="translate(506.6927648987423 1144.996458673506) rotate(0 71.5 70.5)"><path d="M32 0 M32 0 C59.59 -0.06, 89.98 1.81, 111 0 M32 0 C49.54 -0.22, 69.13 -0.97, 111 0 M111 0 C132.32 -1.03, 141.16 9.61, 143 32 M111 0 C131.32 0.79, 143.52 12.66, 143 32 M143 32 C143.11 54.71, 141.38 78.95, 143 109 M143 32 C143.2 57.34, 143.91 80.82, 143 109 M143 109 C144.32 128.94, 131.86 142.53, 111 141 M143 109 C142.86 131.15, 132.81 141.84, 111 141 M111 141 C85.09 139.74, 56.57 139.14, 32 141 M111 141 C83.22 140.87, 55.48 141.24, 32 141 M32 141 C11.45 139.2, -1.9 128.67, 0 109 M32 141 C9.49 139.82, -0.34 129.17, 0 109 M0 109 C-1.07 78.64, -0.34 49.14, 0 32 M0 109 C-0.26 83.5, 1.23 59.21, 0 32 M0 32 C0.85 8.86, 11.43 -1.35, 32 0 M0 32 C1.9 9.17, 12.86 0.76, 32 0" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(516.7828299011837 1177.996458673506) rotate(0 61.409934997558594 37.5)"><text x="61.409934997558594" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">success exit</text><text x="61.409934997558594" y="25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">write 0 to </text><text x="61.409934997558594" y="50" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">conn fd</text></g><g stroke-linecap="round"><g transform="translate(832.5137502681974 1361.0474734333848) rotate(0 -89.35733223356846 -68.54912174280759)"><path d="M0.17 -0.33 C-29.32 -23.32, -148.14 -114.58, -177.77 -137.4 M-1.19 -1.54 C-30.69 -24.42, -148.79 -113.61, -178.23 -136.02" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(832.5137502681974 1361.0474734333848) rotate(0 -89.35733223356846 -68.54912174280759)"><path d="M-150.11 -125.59 C-156.8 -127.36, -163.22 -130.49, -177.1 -137.26 M-149.25 -127.91 C-157.53 -129.69, -164.06 -130.97, -179.11 -136.1" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(832.5137502681974 1361.0474734333848) rotate(0 -89.35733223356846 -68.54912174280759)"><path d="M-162.51 -109.24 C-166.38 -114.84, -169.91 -121.79, -177.1 -137.26 M-161.65 -111.56 C-166.52 -117.75, -169.71 -123.44, -179.11 -136.1" stroke="#1e1e1e" stroke-width="1" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(509.3039099182736 1219.3853136539747) rotate(0 -62.777760823567746 -123.33333333333343)"><path d="M1.17 -0.84 C-19.65 -41.99, -104.36 -206.54, -125.44 -247.6 M0.33 1.33 C-20.6 -39.63, -105.28 -205.31, -126.33 -246.55" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(509.3039099182736 1219.3853136539747) rotate(0 -62.777760823567746 -123.33333333333343)"><path d="M-105.77 -225.93 C-111.51 -230.83, -116.52 -236.13, -127.32 -248.47 M-104.14 -226.6 C-111.42 -231.84, -118.19 -238.66, -126.54 -247.14" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(509.3039099182736 1219.3853136539747) rotate(0 -62.777760823567746 -123.33333333333343)"><path d="M-124.05 -216.59 C-124.98 -224.1, -125.25 -231.83, -127.32 -248.47 M-122.42 -217.26 C-123.87 -225.55, -124.91 -235.3, -126.54 -247.14" stroke="#1e1e1e" stroke-width="1" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(166.85948242803897 1519.385415379235) rotate(0 108 30)"><path d="M15 0 M15 0 C81.49 0.24, 150.63 2.26, 201 0 M15 0 C85.57 0.17, 153.91 -0.69, 201 0 M201 0 C212.88 1.5, 214.38 3.88, 216 15 M201 0 C212.75 -0.66, 217.92 3.42, 216 15 M216 15 C215.53 23.08, 217.88 35.51, 216 45 M216 15 C217.16 21.56, 216.88 28.36, 216 45 M216 45 C214.85 53.75, 209.39 60.15, 201 60 M216 45 C217.91 53.6, 209.13 59.74, 201 60 M201 60 C155.71 61.45, 106.72 61, 15 60 M201 60 C151 59.73, 103.65 58.65, 15 60 M15 60 C6.95 59.09, 1.34 55.72, 0 45 M15 60 C2.79 58.61, 0.43 54.84, 0 45 M0 45 C-1.48 33.01, -0.87 22.85, 0 15 M0 45 C-0.66 35.91, 1.09 27.55, 0 15 M0 15 C-1.17 5.09, 6.95 -1.7, 15 0 M0 15 C1.88 6.38, 7.05 -1.57, 15 0" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(204.54954590460147 1536.885415379235) rotate(0 70.3099365234375 12.5)"><text x="70.3099365234375" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">IoCopy thread</text></g><g stroke-linecap="round" transform="translate(819.0261999573364 1518.0520057519568) rotate(0 177.5 33)"><path d="M16.5 0 M16.5 0 C138.75 0.49, 262.45 -0.64, 338.5 0 M16.5 0 C143.85 2.04, 271.72 2.07, 338.5 0 M338.5 0 C350.27 1.18, 356.62 4.03, 355 16.5 M338.5 0 C351.22 0.37, 353.7 4.79, 355 16.5 M355 16.5 C356.67 26.2, 353.28 36.43, 355 49.5 M355 16.5 C355.69 29.76, 354.87 42.65, 355 49.5 M355 49.5 C353.03 60.93, 350.36 64.71, 338.5 66 M355 49.5 C357.22 61.11, 348.61 65.05, 338.5 66 M338.5 66 C271.28 66.5, 206.69 66.17, 16.5 66 M338.5 66 C230.24 66.08, 121.45 65.52, 16.5 66 M16.5 66 C6.74 67.79, -1.53 61.8, 0 49.5 M16.5 66 C7.5 67.75, 2.01 62.8, 0 49.5 M0 49.5 C-0.64 43.7, -0.56 37.45, 0 16.5 M0 49.5 C-1.25 38.98, -1.19 25.81, 0 16.5 M0 16.5 C-0.69 6.95, 6.65 -1.06, 16.5 0 M0 16.5 C0.55 3.56, 5.37 1.38, 16.5 0" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(874.4163214172974 1538.5520057519568) rotate(0 122.10987854003906 12.5)"><text x="122.10987854003906" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">isula_epoll_loop threads</text></g><g stroke-linecap="round"><g transform="translate(383.859482428039 1532.719538116982) rotate(0 207.72221374511741 1.6662211018298194)"><path d="M-0.77 1.16 C68.48 1.55, 346.47 2.7, 415.76 2.87 M1.02 0.72 C70.12 1.23, 346.16 3.8, 414.98 4.16" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(383.859482428039 1532.719538116982) rotate(0 207.72221374511741 1.6662211018298194)"><path d="M388.64 14.74 C394.98 10.33, 402.94 9.41, 414.91 2.34 M387.08 14.27 C396.45 11, 405.89 7.07, 414.03 4.78" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(383.859482428039 1532.719538116982) rotate(0 207.72221374511741 1.6662211018298194)"><path d="M388.8 -5.78 C395.18 -3.59, 403.09 2.09, 414.91 2.34 M387.23 -6.25 C396.57 -2.5, 405.95 0.59, 414.03 4.78" stroke="#1e1e1e" stroke-width="1" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(803.3652211967894 1569.4944890586517) rotate(0 -206.44947200293785 -1.1489090407252434)"><path d="M0.23 0.69 C-68.35 0.42, -343.26 -0.79, -412.07 -1.48 M-1.11 0 C-69.73 -0.61, -343.89 -2.71, -412.58 -3.18" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(803.3652211967894 1569.4944890586517) rotate(0 -206.44947200293785 -1.1489090407252434)"><path d="M-383.17 -11.85 C-390.1 -10.2, -400.58 -8.89, -411.58 -1.87 M-385.24 -13.52 C-394.99 -10.32, -404.07 -6.36, -413.27 -3.08" stroke="#1e1e1e" stroke-width="1" fill="none"/></g><g transform="translate(803.3652211967894 1569.4944890586517) rotate(0 -206.44947200293785 -1.1489090407252434)"><path d="M-383.32 8.67 C-390.23 5.13, -400.67 1.24, -411.58 -1.87 M-385.39 7 C-395.19 3.05, -404.22 -0.15, -413.27 -3.08" stroke="#1e1e1e" stroke-width="1" fill="none"/></g></g><mask/><g transform="translate(547.0816198792113 1496.0519803206416) rotate(0 46.5799560546875 12.5)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">io stream</text></g></svg> +\ No newline at end of file +-- +2.42.0 + diff --git a/0009-2243-Refactor-capbilities-specs.patch b/0009-2243-Refactor-capbilities-specs.patch new file mode 100644 index 0000000..116a30f --- /dev/null +++ b/0009-2243-Refactor-capbilities-specs.patch @@ -0,0 +1,1056 @@ +From e84112fd7128e05a1a4a380d8242c672d1f539f9 Mon Sep 17 00:00:00 2001 +From: xuxuepeng <xuxuepeng1@huawei.com> +Date: Mon, 13 Nov 2023 08:44:15 +0000 +Subject: [PATCH 09/14] !2243 Refactor capbilities specs * Refactor capbilities + specs + +--- + src/cmd/isula/isula_host_spec.c | 1 + + src/daemon/modules/spec/specs.c | 12 +- + src/daemon/modules/spec/specs_security.c | 140 ++++---- + src/utils/cutils/utils_cap.c | 119 +++++++ + src/utils/cutils/utils_cap.h | 39 +++ + src/utils/cutils/utils_verify.c | 81 ----- + src/utils/cutils/utils_verify.h | 6 - + test/cutils/utils_verify/utils_verify_ut.cc | 14 - + .../image/oci/oci_config_merge/CMakeLists.txt | 1 + + test/image/oci/storage/images/CMakeLists.txt | 1 + + test/specs/specs/CMakeLists.txt | 1 + + test/specs/specs/oci_runtime_spec.json | 32 ++ + test/specs/specs/specs_ut.cc | 315 ++++++++++++++++++ + test/specs/specs_extend/CMakeLists.txt | 1 + + 14 files changed, 573 insertions(+), 190 deletions(-) + create mode 100644 src/utils/cutils/utils_cap.c + create mode 100644 src/utils/cutils/utils_cap.h + +diff --git a/src/cmd/isula/isula_host_spec.c b/src/cmd/isula/isula_host_spec.c +index 6f39588d..09dea271 100644 +--- a/src/cmd/isula/isula_host_spec.c ++++ b/src/cmd/isula/isula_host_spec.c +@@ -36,6 +36,7 @@ + #include "utils_file.h" + #include "utils_string.h" + #include "utils_verify.h" ++#include "utils_cap.h" + #include "opt_ulimit.h" + + static bool parse_restart_policy(const char *policy, host_config_restart_policy **rp) +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index 95346603..cc49d85f 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -49,6 +49,7 @@ + #include "utils_file.h" + #include "utils_string.h" + #include "utils_verify.h" ++#include "utils_cap.h" + + #ifndef CLONE_NEWUTS + #define CLONE_NEWUTS 0x04000000 +@@ -814,15 +815,16 @@ static int adapt_settings_for_privileged(oci_runtime_spec *oci_spec, bool privil + { + int ret = 0; + size_t all_caps_len = 0; ++ const char **all_caps = NULL; + + if (!privileged) { + return 0; + } + +- all_caps_len = util_get_all_caps_len(); +- if (oci_spec == NULL) { +- ret = -1; +- goto out; ++ all_caps = util_get_all_caps(&all_caps_len); ++ if (all_caps == NULL) { ++ ERROR("Failed to get all capabilities"); ++ return -1; + } + + clean_correlated_items(oci_spec); +@@ -838,7 +840,7 @@ static int adapt_settings_for_privileged(oci_runtime_spec *oci_spec, bool privil + goto out; + } + +- ret = refill_oci_process_capabilities(&oci_spec->process->capabilities, g_all_caps, all_caps_len); ++ ret = refill_oci_process_capabilities(&oci_spec->process->capabilities, all_caps, all_caps_len); + if (ret != 0) { + ERROR("Failed to copy all capabilities"); + ret = -1; +diff --git a/src/daemon/modules/spec/specs_security.c b/src/daemon/modules/spec/specs_security.c +index e78cc744..b34aec7c 100644 +--- a/src/daemon/modules/spec/specs_security.c ++++ b/src/daemon/modules/spec/specs_security.c +@@ -37,6 +37,7 @@ + #include "utils_array.h" + #include "utils_string.h" + #include "utils_verify.h" ++#include "utils_cap.h" + + #define MAX_CAP_LEN 32 + +@@ -104,41 +105,31 @@ static int tweak_drops_capabilities(char ***new_caps, size_t *new_caps_len, char + size_t i = 0; + int ret = 0; + +- if (util_strings_in_slice((const char **)drops, drops_len, "all")) { +- goto out; ++ if (basic_caps == NULL || basic_caps_len == 0) { ++ *new_caps = NULL; ++ *new_caps_len = 0; ++ return 0; + } + +- for (i = 0; (basic_caps != NULL && i < basic_caps_len); i++) { +- // skip `all` already handled above +- if (!basic_caps[i] || !strcasecmp(basic_caps[i], "all")) { +- continue; +- } +- +- // if we don't drop `all`, add back all the non-dropped caps ++ for (i = 0; i < basic_caps_len; i++) { + if (!util_strings_in_slice((const char **)drops, drops_len, basic_caps[i] + strlen("CAP_"))) { + ret = append_capability(new_caps, new_caps_len, basic_caps[i]); + if (ret != 0) { + ERROR("Failed to append capabilities"); +- ret = -1; +- goto out; ++ return -1; + } + } + } + +-out: +- return ret; ++ return 0; + } + + static int tweak_adds_capabilities(char ***new_caps, size_t *new_caps_len, const char **adds, size_t adds_len) + { + size_t i = 0; +- int ret = 0; + int nret = 0; +- size_t all_caps_len = 0; + char tmpcap[MAX_CAP_LEN] = { 0 }; + +- all_caps_len = util_get_all_caps_len(); +- + for (i = 0; i < adds_len; i++) { + // skip `all` already handled above + if (strcasecmp(adds[i], "all") == 0) { +@@ -148,111 +139,92 @@ static int tweak_adds_capabilities(char ***new_caps, size_t *new_caps_len, const + nret = snprintf(tmpcap, sizeof(tmpcap), "CAP_%s", adds[i]); + if (nret < 0 || (size_t)nret >= sizeof(tmpcap)) { + ERROR("Failed to print string"); +- ret = -1; +- goto out; +- } +- if (!util_strings_in_slice(g_all_caps, all_caps_len, tmpcap)) { +- ERROR("Unknown capability to add: '%s'", tmpcap); +- ret = -1; +- goto out; ++ return -1; + } + + // add cap if not already in the list + if (!util_strings_in_slice((const char **)*new_caps, *new_caps_len, tmpcap)) { +- ret = append_capability(new_caps, new_caps_len, tmpcap); +- if (ret != 0) { ++ nret = append_capability(new_caps, new_caps_len, tmpcap); ++ if (nret != 0) { + ERROR("Failed to append capabilities"); +- ret = -1; +- goto out; ++ return -1; + } + } + } + +-out: +- return ret; +-} +- +-static bool valid_drops_cap(const char **drops, size_t drops_len) +-{ +- int nret = 0; +- size_t i; +- size_t all_caps_len = 0; +- char tmpcap[MAX_CAP_LEN] = { 0 }; +- +- all_caps_len = util_get_all_caps_len(); +- // look for invalid cap in the drop list +- for (i = 0; i < drops_len; i++) { +- if (strcasecmp(drops[i], "all") == 0) { +- continue; +- } +- +- nret = snprintf(tmpcap, sizeof(tmpcap), "CAP_%s", drops[i]); +- if (nret < 0 || (size_t)nret >= sizeof(tmpcap)) { +- ERROR("Failed to print string"); +- return false; +- } +- if (!util_strings_in_slice(g_all_caps, all_caps_len, tmpcap)) { +- ERROR("Unknown capability to drop: '%s'", drops[i]); +- return false; +- } +- } +- +- return true; ++ return 0; + } + + // tweak_capabilities can tweak capabilities by adding or dropping capabilities +-// based on the basic capabilities. ++// based on the basic capabilities. The following are the priorities of the tweaks: ++// 1. if adds contains "all", then the basic capabilities will be ignored, and all capabilities will be added. ++// 2. if drops contains "all", all capabilities will be dropped. ++// 3. add individual capabilities in adds ++// 4. drop individual capabilities in drops. ++// The reason why we handle "all" first is that we can avoid the case that the individual capabilities are ++// not included by "all". + static int tweak_capabilities(char ***caps, size_t *caps_len, const char **adds, size_t adds_len, const char **drops, + size_t drops_len) + { +- size_t i; +- size_t all_caps_len = 0; + int ret = 0; + char **new_caps = NULL; + char **basic_caps = NULL; ++ const char **all_caps = NULL; + size_t new_caps_len = 0; + size_t basic_caps_len = 0; ++ size_t all_caps_len = 0; ++ bool add_all = false; ++ bool drop_all = false; + +- all_caps_len = util_get_all_caps_len(); +- if (!valid_drops_cap(drops, drops_len)) { ++ all_caps = util_get_all_caps(&all_caps_len); ++ if (all_caps == NULL) { ++ ERROR("Failed to get all capabilities"); + return -1; + } + +- if (util_strings_in_slice((const char **)adds, adds_len, "all")) { +- ret = copy_capabilities(&basic_caps, &basic_caps_len, g_all_caps, all_caps_len); +- } else { ++ add_all = util_strings_in_slice((const char **)adds, adds_len, "all"); ++ drop_all = util_strings_in_slice((const char **)drops, drops_len, "all"); ++ ++ ++ if (!add_all && !drop_all) { ++ // if neither add_all nor drop_all, we start with the default capabilities + ret = copy_capabilities(&basic_caps, &basic_caps_len, (const char **)*caps, *caps_len); +- } +- if (ret != 0) { +- ERROR("Failed to copy capabilities"); +- ret = -1; +- goto free_out; ++ if (ret != 0) { ++ ERROR("Failed to copy capabilities"); ++ ret = -1; ++ goto free_out; ++ } ++ } else if (drop_all) { ++ // if drop_all, we start with an empty set ++ basic_caps = NULL; ++ basic_caps_len = 0; ++ } else { ++ // if not drop_all but add_all, we start with all capabilities ++ ret = copy_capabilities(&basic_caps, &basic_caps_len, all_caps, all_caps_len); ++ if (ret != 0) { ++ ERROR("Failed to copy all capabilities"); ++ ret = -1; ++ goto free_out; ++ } + } + +- ret = tweak_drops_capabilities(&new_caps, &new_caps_len, basic_caps, basic_caps_len, drops, drops_len); ++ // Add capabilities to the basic capabilities ++ ret = tweak_adds_capabilities(&basic_caps, &basic_caps_len, adds, adds_len); + if (ret != 0) { + ret = -1; + goto free_out; + } + +- ret = tweak_adds_capabilities(&new_caps, &new_caps_len, adds, adds_len); ++ // Drop capabilities from the basic capabilities ++ ret = tweak_drops_capabilities(&new_caps, &new_caps_len, basic_caps, basic_caps_len, drops, drops_len); + if (ret != 0) { + ret = -1; + goto free_out; + } + + free_out: +- for (i = 0; i < basic_caps_len; i++) { +- free(basic_caps[i]); +- } +- free(basic_caps); +- +- // free old caps +- for (i = 0; i < *caps_len; i++) { +- free((*caps)[i]); +- (*caps)[i] = NULL; +- } +- free(*caps); ++ util_free_array_by_len(basic_caps, basic_caps_len); ++ util_free_array_by_len(*caps, *caps_len); + + // set new caps + *caps = new_caps; +diff --git a/src/utils/cutils/utils_cap.c b/src/utils/cutils/utils_cap.c +new file mode 100644 +index 00000000..6473df45 +--- /dev/null ++++ b/src/utils/cutils/utils_cap.c +@@ -0,0 +1,119 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2022. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: xuxuepeng ++ * Create: 2023-11-08 ++ * Description: provide capbilities utils functions ++ *******************************************************************************/ ++ ++#define _GNU_SOURCE ++ ++#include "utils_cap.h" ++ ++#include <stdint.h> ++#include <stdio.h> ++#include <isula_libutils/log.h> ++ ++#include "utils_string.h" ++ ++const char *g_all_caps[] = { ++ "CAP_CHOWN", ++ "CAP_DAC_OVERRIDE", ++ "CAP_DAC_READ_SEARCH", ++ "CAP_FOWNER", ++ "CAP_FSETID", ++ "CAP_KILL", ++ "CAP_SETGID", ++ "CAP_SETUID", ++ "CAP_SETPCAP", ++ "CAP_LINUX_IMMUTABLE", ++ "CAP_NET_BIND_SERVICE", ++ "CAP_NET_BROADCAST", ++ "CAP_NET_ADMIN", ++ "CAP_NET_RAW", ++ "CAP_IPC_LOCK", ++ "CAP_IPC_OWNER", ++ "CAP_SYS_MODULE", ++ "CAP_SYS_RAWIO", ++ "CAP_SYS_CHROOT", ++ "CAP_SYS_PTRACE", ++ "CAP_SYS_PACCT", ++ "CAP_SYS_ADMIN", ++ "CAP_SYS_BOOT", ++ "CAP_SYS_NICE", ++ "CAP_SYS_RESOURCE", ++ "CAP_SYS_TIME", ++ "CAP_SYS_TTY_CONFIG", ++ "CAP_MKNOD", ++ "CAP_LEASE", ++#ifdef CAP_AUDIT_WRITE ++ "CAP_AUDIT_WRITE", ++#endif ++#ifdef CAP_AUDIT_CONTROL ++ "CAP_AUDIT_CONTROL", ++#endif ++ "CAP_SETFCAP", ++ "CAP_MAC_OVERRIDE", ++ "CAP_MAC_ADMIN", ++#ifdef CAP_SYSLOG ++ "CAP_SYSLOG", ++#endif ++#ifdef CAP_WAKE_ALARM ++ "CAP_WAKE_ALARM", ++#endif ++#ifdef CAP_BLOCK_SUSPEND ++ "CAP_BLOCK_SUSPEND", ++#endif ++#ifdef CAP_AUDIT_READ ++ "CAP_AUDIT_READ", ++#endif ++#ifdef CAP_PERFMON ++ "CAP_PERFMON", ++#endif ++#ifdef CAP_BPF ++ "CAP_BPF", ++#endif ++#ifdef CAP_CHECKPOINT_RESTORE ++ "CAP_CHECKPOINT_RESTORE", ++#endif ++}; ++ ++static inline size_t util_get_all_caps_len() ++{ ++ return sizeof(g_all_caps) / sizeof(char *); ++} ++ ++bool util_valid_cap(const char *cap) ++{ ++ int nret = 0; ++ char tmpcap[32] = { 0 }; ++ size_t all_caps_len = util_get_all_caps_len(); ++ ++ if (cap == NULL) { ++ return false; ++ } ++ ++ nret = snprintf(tmpcap, sizeof(tmpcap), "CAP_%s", cap); ++ if (nret < 0 || (size_t)nret >= sizeof(tmpcap)) { ++ ERROR("Failed to print string"); ++ return false; ++ } ++ if (!util_strings_in_slice(g_all_caps, all_caps_len, tmpcap)) { ++ return false; ++ } ++ ++ return true; ++} ++ ++const char **util_get_all_caps(size_t *cap_len) ++{ ++ *cap_len = util_get_all_caps_len(); ++ return g_all_caps; ++} +diff --git a/src/utils/cutils/utils_cap.h b/src/utils/cutils/utils_cap.h +new file mode 100644 +index 00000000..de63d070 +--- /dev/null ++++ b/src/utils/cutils/utils_cap.h +@@ -0,0 +1,39 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2022. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: xuxuepeng ++ * Create: 2023-11-08 ++ * Description: provide capbilities utils functions ++ *******************************************************************************/ ++ ++#ifndef UTILS_CUTILS_UTILS_CAP_H ++#define UTILS_CUTILS_UTILS_CAP_H ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++#include <stdbool.h> ++#include <stddef.h> ++#include <linux/capability.h> ++ ++bool util_valid_cap(const char *cap); ++ ++/** ++ * Get all supported capabilities for linux, ++ * note that the returned strings are unmutable ++ */ ++const char **util_get_all_caps(size_t *cap_len); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif // UTILS_CUTILS_UTILS_CAP_H +diff --git a/src/utils/cutils/utils_verify.c b/src/utils/cutils/utils_verify.c +index 2f10f278..f4ce3199 100644 +--- a/src/utils/cutils/utils_verify.c ++++ b/src/utils/cutils/utils_verify.c +@@ -22,7 +22,6 @@ + #include <sys/stat.h> + #include <errno.h> + #include <fcntl.h> +-#include <linux/capability.h> + #include <stdio.h> + #include <strings.h> + +@@ -32,59 +31,6 @@ + #include "utils_array.h" + #include "utils_string.h" + +-const char *g_all_caps[] = { +- "CAP_CHOWN", +- "CAP_DAC_OVERRIDE", +- "CAP_DAC_READ_SEARCH", +- "CAP_FOWNER", +- "CAP_FSETID", +- "CAP_KILL", +- "CAP_SETGID", +- "CAP_SETUID", +- "CAP_SETPCAP", +- "CAP_LINUX_IMMUTABLE", +- "CAP_NET_BIND_SERVICE", +- "CAP_NET_BROADCAST", +- "CAP_NET_ADMIN", +- "CAP_NET_RAW", +- "CAP_IPC_LOCK", +- "CAP_IPC_OWNER", +- "CAP_SYS_MODULE", +- "CAP_SYS_RAWIO", +- "CAP_SYS_CHROOT", +- "CAP_SYS_PTRACE", +- "CAP_SYS_PACCT", +- "CAP_SYS_ADMIN", +- "CAP_SYS_BOOT", +- "CAP_SYS_NICE", +- "CAP_SYS_RESOURCE", +- "CAP_SYS_TIME", +- "CAP_SYS_TTY_CONFIG", +- "CAP_MKNOD", +- "CAP_LEASE", +-#ifdef CAP_AUDIT_WRITE +- "CAP_AUDIT_WRITE", +-#endif +-#ifdef CAP_AUDIT_CONTROL +- "CAP_AUDIT_CONTROL", +-#endif +- "CAP_SETFCAP", +- "CAP_MAC_OVERRIDE", +- "CAP_MAC_ADMIN", +-#ifdef CAP_SYSLOG +- "CAP_SYSLOG", +-#endif +-#ifdef CAP_WAKE_ALARM +- "CAP_WAKE_ALARM", +-#endif +-#ifdef CAP_BLOCK_SUSPEND +- "CAP_BLOCK_SUSPEND", +-#endif +-#ifdef CAP_AUDIT_READ +- "CAP_AUDIT_READ", +-#endif +-}; +- + bool util_valid_cmd_arg(const char *arg) + { + return (arg != NULL) && (strchr(arg, '|') == NULL) && (strchr(arg, '`') == NULL) && (strchr(arg, '&')) == NULL && +@@ -215,33 +161,6 @@ bool util_valid_str(const char *str) + return (str != NULL && str[0] != '\0') ? true : false; + } + +-size_t util_get_all_caps_len() +-{ +- return sizeof(g_all_caps) / sizeof(char *); +-} +- +-bool util_valid_cap(const char *cap) +-{ +- int nret = 0; +- char tmpcap[32] = { 0 }; +- size_t all_caps_len = util_get_all_caps_len(); +- +- if (cap == NULL) { +- return false; +- } +- +- nret = snprintf(tmpcap, sizeof(tmpcap), "CAP_%s", cap); +- if (nret < 0 || (size_t)nret >= sizeof(tmpcap)) { +- ERROR("Failed to print string"); +- return false; +- } +- if (!util_strings_in_slice(g_all_caps, all_caps_len, tmpcap)) { +- return false; +- } +- +- return true; +-} +- + bool util_valid_container_id(const char *id) + { + char *patten = "^[a-f0-9]{1,64}$"; +diff --git a/src/utils/cutils/utils_verify.h b/src/utils/cutils/utils_verify.h +index ad4466ef..54d1ce71 100644 +--- a/src/utils/cutils/utils_verify.h ++++ b/src/utils/cutils/utils_verify.h +@@ -38,8 +38,6 @@ extern "C" { + + #define VALID_VOLUME_NAME "[a-zA-Z0-9][a-zA-Z0-9_.-]{1,63}" + +-extern const char *g_all_caps[]; +- + bool util_valid_cmd_arg(const char *arg); + + bool util_valid_signal(int sig); +@@ -54,10 +52,6 @@ bool util_valid_device_mode(const char *mode); + + bool util_valid_str(const char *str); + +-size_t util_get_all_caps_len(); +- +-bool util_valid_cap(const char *cap); +- + bool util_valid_time_tz(const char *time); + + bool util_valid_embedded_image_name(const char *name); +diff --git a/test/cutils/utils_verify/utils_verify_ut.cc b/test/cutils/utils_verify/utils_verify_ut.cc +index 99775d09..79670ec1 100644 +--- a/test/cutils/utils_verify/utils_verify_ut.cc ++++ b/test/cutils/utils_verify/utils_verify_ut.cc +@@ -98,20 +98,6 @@ TEST(utils_verify, test_util_valid_str) + ASSERT_EQ(util_valid_str(nullptr), false); + } + +-TEST(utils_verify, test_util_get_all_caps_len) +-{ +- ASSERT_NE(util_get_all_caps_len(), 0); +-} +- +-TEST(utils_verify, test_util_valid_cap) +-{ +- ASSERT_EQ(util_valid_cap("DAC_READ_SEARCH"), true); +- +- ASSERT_EQ(util_valid_cap(nullptr), false); +- ASSERT_EQ(util_valid_cap(""), false); +- ASSERT_EQ(util_valid_cap("DA_READ_SEARCH"), false); +-} +- + TEST(utils_verify, test_util_valid_time_tz) + { + ASSERT_EQ(util_valid_time_tz("2022-10-04T18:22:45.289257759Z"), true); +diff --git a/test/image/oci/oci_config_merge/CMakeLists.txt b/test/image/oci/oci_config_merge/CMakeLists.txt +index ce4df5ba..90809080 100644 +--- a/test/image/oci/oci_config_merge/CMakeLists.txt ++++ b/test/image/oci/oci_config_merge/CMakeLists.txt +@@ -14,6 +14,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/utils/cutils/utils_file.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/utils/cutils/utils_timestamp.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/utils/cutils/utils_fs.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/utils/cutils/utils_cap.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/utils/cutils/map/map.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/utils/cutils/map/rb_tree.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/utils/cutils/util_atomic.c +diff --git a/test/image/oci/storage/images/CMakeLists.txt b/test/image/oci/storage/images/CMakeLists.txt +index 8446ebba..28e0b505 100644 +--- a/test/image/oci/storage/images/CMakeLists.txt ++++ b/test/image/oci/storage/images/CMakeLists.txt +@@ -11,6 +11,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/utils_convert.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/utils_file.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/utils_base64.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/utils_cap.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/util_atomic.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/sha256/sha256.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/path.c +diff --git a/test/specs/specs/CMakeLists.txt b/test/specs/specs/CMakeLists.txt +index 1d627e37..a9dbc52c 100644 +--- a/test/specs/specs/CMakeLists.txt ++++ b/test/specs/specs/CMakeLists.txt +@@ -14,6 +14,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/util_atomic.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_mount_spec.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_fs.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_cap.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/sha256/sha256.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/path.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map/map.c +diff --git a/test/specs/specs/oci_runtime_spec.json b/test/specs/specs/oci_runtime_spec.json +index 0223fd6f..efd5da35 100644 +--- a/test/specs/specs/oci_runtime_spec.json ++++ b/test/specs/specs/oci_runtime_spec.json +@@ -154,6 +154,38 @@ + "CAP_SYS_CHROOT", + "CAP_KILL", + "CAP_AUDIT_WRITE" ++ ], ++ "effective": [ ++ "CAP_CHOWN", ++ "CAP_DAC_OVERRIDE", ++ "CAP_FSETID", ++ "CAP_FOWNER", ++ "CAP_MKNOD", ++ "CAP_NET_RAW", ++ "CAP_SETGID", ++ "CAP_SETUID", ++ "CAP_SETFCAP", ++ "CAP_SETPCAP", ++ "CAP_NET_BIND_SERVICE", ++ "CAP_SYS_CHROOT", ++ "CAP_KILL", ++ "CAP_AUDIT_WRITE" ++ ], ++ "permitted": [ ++ "CAP_CHOWN", ++ "CAP_DAC_OVERRIDE", ++ "CAP_FSETID", ++ "CAP_FOWNER", ++ "CAP_MKNOD", ++ "CAP_NET_RAW", ++ "CAP_SETGID", ++ "CAP_SETUID", ++ "CAP_SETFCAP", ++ "CAP_SETPCAP", ++ "CAP_NET_BIND_SERVICE", ++ "CAP_SYS_CHROOT", ++ "CAP_KILL", ++ "CAP_AUDIT_WRITE" + ] + } + }, +diff --git a/test/specs/specs/specs_ut.cc b/test/specs/specs/specs_ut.cc +index 96aa1c63..ad903a3f 100644 +--- a/test/specs/specs/specs_ut.cc ++++ b/test/specs/specs/specs_ut.cc +@@ -20,6 +20,7 @@ + #include "isula_libutils/oci_runtime_spec.h" + #include "specs_api.h" + #include "specs_namespace.h" ++#include "specs_security.h" + #include "isula_libutils/host_config.h" + #include "isula_libutils/container_config.h" + #include "oci_ut_common.h" +@@ -27,6 +28,7 @@ + #include <gmock/gmock.h> + #include "isulad_config_mock.h" + #include "utils.h" ++#include "utils_cap.h" + + using ::testing::Args; + using ::testing::ByRef; +@@ -344,3 +346,316 @@ TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_5) + + testing::Mock::VerifyAndClearExpectations(&m_isulad_conf); + } ++ ++/********************************* UT for merge caps *******************************************/ ++struct capabilities_lens { ++ size_t bounding_len; ++ size_t effective_len; ++ size_t inheritable_len; ++ size_t permitted_len; ++ size_t ambient_len; ++}; ++ ++void check_capabilities_len(defs_process_capabilities *cap, struct capabilities_lens *lens) ++{ ++ lens->bounding_len = cap->bounding_len; ++ lens->effective_len = cap->effective_len; ++ lens->inheritable_len = cap->inheritable_len; ++ lens->permitted_len = cap->permitted_len; ++ lens->ambient_len = cap->ambient_len; ++} ++ ++void validate_capabilities_len(defs_process_capabilities *cap, struct capabilities_lens *lens, ssize_t len_diff) ++{ ++ ASSERT_EQ((ssize_t)cap->bounding_len, (ssize_t)lens->bounding_len + len_diff); ++ ASSERT_EQ((ssize_t)cap->effective_len, (ssize_t)lens->effective_len + len_diff); ++ ASSERT_EQ((ssize_t)cap->permitted_len, (ssize_t)lens->permitted_len + len_diff); ++ // Currently we don't support inheritable and ambient capabilities ++} ++ ++TEST(merge_capability_ut, test_merge_caps_without_adds_drops) ++{ ++ oci_runtime_spec *oci_spec = nullptr; ++ int ret = 0; ++ char *err = nullptr; ++ struct capabilities_lens old_lens = { 0 }; ++ char *oci_config_file = nullptr; ++ ++ oci_config_file = json_path(OCI_RUNTIME_SPEC_FILE); ++ ASSERT_TRUE(oci_config_file != nullptr); ++ ++ ++ oci_spec = oci_runtime_spec_parse_file(oci_config_file, nullptr, &err); ++ ASSERT_TRUE(oci_spec != nullptr); ++ free(err); ++ ++ check_capabilities_len(oci_spec->process->capabilities, &old_lens); ++ ++ ret = merge_caps(oci_spec, nullptr, 0, nullptr, 0); ++ ASSERT_EQ(ret, 0); ++ ++ validate_capabilities_len(oci_spec->process->capabilities, &old_lens, 0); ++ ++ free_oci_runtime_spec(oci_spec); ++} ++ ++TEST(merge_capability_ut, test_merge_caps_adds_without_drops) ++{ ++ oci_runtime_spec *oci_spec = nullptr; ++ int ret = 0; ++ char *err = nullptr; ++ struct capabilities_lens old_lens = { 0 }; ++ char *oci_config_file = nullptr; ++ /* All of below capabilities are not in oci_config_file */ ++ const char *adds[] = { "NET_ADMIN", "SYS_ADMIN", "SYS_TTY_CONFIG", "SYS_PTRACE" }; ++ const char *drops[] = {}; ++ size_t adds_len = sizeof(adds) / sizeof(adds[0]); ++ size_t drops_len = sizeof(drops) / sizeof(drops[0]); ++ ++ oci_config_file = json_path(OCI_RUNTIME_SPEC_FILE); ++ ASSERT_TRUE(oci_config_file != nullptr); ++ ++ ++ oci_spec = oci_runtime_spec_parse_file(oci_config_file, nullptr, &err); ++ ASSERT_TRUE(oci_spec != nullptr); ++ free(err); ++ ++ check_capabilities_len(oci_spec->process->capabilities, &old_lens); ++ ++ ret = merge_caps(oci_spec, adds, adds_len, drops, drops_len); ++ ASSERT_EQ(ret, 0); ++ ++ /* All of capabilities in adds are added */ ++ validate_capabilities_len(oci_spec->process->capabilities, &old_lens, adds_len); ++ ++ free_oci_runtime_spec(oci_spec); ++} ++ ++TEST(merge_capability_ut, test_merge_caps_adds_existing_without_drops) ++{ ++ oci_runtime_spec *oci_spec = nullptr; ++ int ret = 0; ++ char *err = nullptr; ++ struct capabilities_lens old_lens = { 0 }; ++ char *oci_config_file = nullptr; ++ /* CHOWN already exits in oci_config_file */ ++ const char *adds[] = { "CHOWN", "SYS_ADMIN", "SYS_TTY_CONFIG", "SYS_PTRACE" }; ++ const char *drops[] = {}; ++ size_t adds_len = sizeof(adds) / sizeof(adds[0]); ++ size_t drops_len = sizeof(drops) / sizeof(drops[0]); ++ ++ oci_config_file = json_path(OCI_RUNTIME_SPEC_FILE); ++ ASSERT_TRUE(oci_config_file != nullptr); ++ ++ ++ oci_spec = oci_runtime_spec_parse_file(oci_config_file, nullptr, &err); ++ ASSERT_TRUE(oci_spec != nullptr); ++ free(err); ++ ++ check_capabilities_len(oci_spec->process->capabilities, &old_lens); ++ ++ ret = merge_caps(oci_spec, adds, adds_len, drops, drops_len); ++ ASSERT_EQ(ret, 0); ++ ++ /* CHOWN is not added, since it already exits in the default list */ ++ validate_capabilities_len(oci_spec->process->capabilities, &old_lens, adds_len - 1); ++ ++ free_oci_runtime_spec(oci_spec); ++} ++ ++TEST(merge_capability_ut, test_merge_caps_drops_without_adds) ++{ ++ oci_runtime_spec *oci_spec = nullptr; ++ int ret = 0; ++ char *err = nullptr; ++ struct capabilities_lens old_lens = { 0 }; ++ char *oci_config_file = nullptr; ++ const char *adds[] = {}; ++ /* Below capabilities are not in the oci_config_file */ ++ const char *drops[] = { "SYS_TTY_CONFIG", "SYS_PTRACE" }; ++ size_t adds_len = sizeof(adds) / sizeof(adds[0]); ++ size_t drops_len = sizeof(drops) / sizeof(drops[0]); ++ ++ oci_config_file = json_path(OCI_RUNTIME_SPEC_FILE); ++ ASSERT_TRUE(oci_config_file != nullptr); ++ ++ ++ oci_spec = oci_runtime_spec_parse_file(oci_config_file, nullptr, &err); ++ ASSERT_TRUE(oci_spec != nullptr); ++ free(err); ++ ++ check_capabilities_len(oci_spec->process->capabilities, &old_lens); ++ ++ ret = merge_caps(oci_spec, adds, adds_len, drops, drops_len); ++ ASSERT_EQ(ret, 0); ++ ++ /* Nothing dropped */ ++ validate_capabilities_len(oci_spec->process->capabilities, &old_lens, 0); ++ ++ free_oci_runtime_spec(oci_spec); ++} ++ ++TEST(merge_capability_ut, test_merge_caps_drops_existing_without_adds) ++{ ++ oci_runtime_spec *oci_spec = nullptr; ++ int ret = 0; ++ char *err = nullptr; ++ struct capabilities_lens old_lens = { 0 }; ++ char *oci_config_file = nullptr; ++ const char *adds[] = {}; ++ /* Below capabilities are in the oci_config_file */ ++ const char *drops[] = { "CHOWN", "MKNOD" }; ++ size_t adds_len = sizeof(adds) / sizeof(adds[0]); ++ size_t drops_len = sizeof(drops) / sizeof(drops[0]); ++ ++ oci_config_file = json_path(OCI_RUNTIME_SPEC_FILE); ++ ASSERT_TRUE(oci_config_file != nullptr); ++ ++ ++ oci_spec = oci_runtime_spec_parse_file(oci_config_file, nullptr, &err); ++ ASSERT_TRUE(oci_spec != nullptr); ++ free(err); ++ ++ check_capabilities_len(oci_spec->process->capabilities, &old_lens); ++ ++ ret = merge_caps(oci_spec, adds, adds_len, drops, drops_len); ++ ASSERT_EQ(ret, 0); ++ ++ /* All dropped */ ++ validate_capabilities_len(oci_spec->process->capabilities, &old_lens, adds_len - drops_len); ++ ++ free_oci_runtime_spec(oci_spec); ++} ++ ++TEST(merge_capability_ut, test_merge_caps_adds_drops) ++{ ++ oci_runtime_spec *oci_spec = nullptr; ++ int ret = 0; ++ char *err = nullptr; ++ struct capabilities_lens old_lens = { 0 }; ++ char *oci_config_file = nullptr; ++ /* All of below capabilities are not in oci_config_file */ ++ const char *adds[] = { "NET_ADMIN", "SYS_ADMIN", "SYS_TTY_CONFIG", "SYS_PTRACE" }; ++ const char *drops[] = { "SYS_TTY_CONFIG", "SYS_PTRACE" }; ++ size_t adds_len = sizeof(adds) / sizeof(adds[0]); ++ size_t drops_len = sizeof(drops) / sizeof(drops[0]); ++ ++ oci_config_file = json_path(OCI_RUNTIME_SPEC_FILE); ++ ASSERT_TRUE(oci_config_file != nullptr); ++ ++ ++ oci_spec = oci_runtime_spec_parse_file(oci_config_file, nullptr, &err); ++ ASSERT_TRUE(oci_spec != nullptr); ++ free(err); ++ ++ check_capabilities_len(oci_spec->process->capabilities, &old_lens); ++ ++ ret = merge_caps(oci_spec, adds, adds_len, drops, drops_len); ++ ASSERT_EQ(ret, 0); ++ ++ validate_capabilities_len(oci_spec->process->capabilities, &old_lens, adds_len - drops_len); ++ ++ free_oci_runtime_spec(oci_spec); ++} ++ ++TEST(merge_capability_ut, test_merge_caps_adds_all_without_drops) ++{ ++ oci_runtime_spec *oci_spec = nullptr; ++ int ret = 0; ++ char *err = nullptr; ++ struct capabilities_lens old_lens = { 0 }; ++ char *oci_config_file = nullptr; ++ /* NET_ADMIN is in all */ ++ const char *adds[] = { "ALL", "NET_ADMIN" }; ++ const char *drops[] = {}; ++ size_t adds_len = sizeof(adds) / sizeof(adds[0]); ++ size_t drops_len = sizeof(drops) / sizeof(drops[0]); ++ size_t all_caps_len = 0; ++ util_get_all_caps(&all_caps_len); ++ ++ oci_config_file = json_path(OCI_RUNTIME_SPEC_FILE); ++ ASSERT_TRUE(oci_config_file != nullptr); ++ ++ oci_spec = oci_runtime_spec_parse_file(oci_config_file, nullptr, &err); ++ ASSERT_TRUE(oci_spec != nullptr); ++ free(err); ++ ++ check_capabilities_len(oci_spec->process->capabilities, &old_lens); ++ ++ ret = merge_caps(oci_spec, adds, adds_len, drops, drops_len); ++ ASSERT_EQ(ret, 0); ++ ++ ASSERT_EQ(oci_spec->process->capabilities->bounding_len, all_caps_len); ++ ASSERT_EQ(oci_spec->process->capabilities->effective_len, all_caps_len); ++ ASSERT_EQ(oci_spec->process->capabilities->permitted_len, all_caps_len); ++ ++ free_oci_runtime_spec(oci_spec); ++} ++ ++TEST(merge_capability_ut, test_merge_caps_adds_all_and_extra_without_drops) ++{ ++ oci_runtime_spec *oci_spec = nullptr; ++ int ret = 0; ++ char *err = nullptr; ++ struct capabilities_lens old_lens = { 0 }; ++ char *oci_config_file = nullptr; ++ /* ABC is not in all */ ++ const char *adds[] = { "ALL", "ABC" }; ++ const char *drops[] = {}; ++ size_t adds_len = sizeof(adds) / sizeof(adds[0]); ++ size_t drops_len = sizeof(drops) / sizeof(drops[0]); ++ size_t all_caps_len = 0; ++ util_get_all_caps(&all_caps_len); ++ ++ oci_config_file = json_path(OCI_RUNTIME_SPEC_FILE); ++ ASSERT_TRUE(oci_config_file != nullptr); ++ ++ oci_spec = oci_runtime_spec_parse_file(oci_config_file, nullptr, &err); ++ ASSERT_TRUE(oci_spec != nullptr); ++ free(err); ++ ++ check_capabilities_len(oci_spec->process->capabilities, &old_lens); ++ ++ ret = merge_caps(oci_spec, adds, adds_len, drops, drops_len); ++ ASSERT_EQ(ret, 0); ++ ++ ASSERT_EQ(oci_spec->process->capabilities->bounding_len, all_caps_len + 1); ++ ASSERT_EQ(oci_spec->process->capabilities->effective_len, all_caps_len + 1); ++ ASSERT_EQ(oci_spec->process->capabilities->permitted_len, all_caps_len + 1); ++ ++ free_oci_runtime_spec(oci_spec); ++} ++ ++TEST(merge_capability_ut, test_merge_caps_adds_all_drops_all) ++{ ++ oci_runtime_spec *oci_spec = nullptr; ++ int ret = 0; ++ char *err = nullptr; ++ struct capabilities_lens old_lens = { 0 }; ++ char *oci_config_file = nullptr; ++ /* ABC, EFG is not in all */ ++ const char *adds[] = { "ALL", "ABC", "EFG"}; ++ const char *drops[] = { "ALL", "ABC" }; ++ size_t adds_len = sizeof(adds) / sizeof(adds[0]); ++ size_t drops_len = sizeof(drops) / sizeof(drops[0]); ++ size_t all_caps_len = 0; ++ util_get_all_caps(&all_caps_len); ++ ++ oci_config_file = json_path(OCI_RUNTIME_SPEC_FILE); ++ ASSERT_TRUE(oci_config_file != nullptr); ++ ++ oci_spec = oci_runtime_spec_parse_file(oci_config_file, nullptr, &err); ++ ASSERT_TRUE(oci_spec != nullptr); ++ free(err); ++ ++ check_capabilities_len(oci_spec->process->capabilities, &old_lens); ++ ++ ret = merge_caps(oci_spec, adds, adds_len, drops, drops_len); ++ ASSERT_EQ(ret, 0); ++ ++ ASSERT_EQ(oci_spec->process->capabilities->bounding_len, 1); ++ ASSERT_EQ(oci_spec->process->capabilities->effective_len, 1); ++ ASSERT_EQ(oci_spec->process->capabilities->permitted_len, 1); ++ ++ free_oci_runtime_spec(oci_spec); ++} +diff --git a/test/specs/specs_extend/CMakeLists.txt b/test/specs/specs_extend/CMakeLists.txt +index 294690e8..bf4b378e 100644 +--- a/test/specs/specs_extend/CMakeLists.txt ++++ b/test/specs/specs_extend/CMakeLists.txt +@@ -14,6 +14,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/util_atomic.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_mount_spec.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_fs.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_cap.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/sha256/sha256.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/path.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map/map.c +-- +2.42.0 + diff --git a/0010-2245-fix-utils_verify_ut-failure-when-remote-grpc-fu.patch b/0010-2245-fix-utils_verify_ut-failure-when-remote-grpc-fu.patch new file mode 100644 index 0000000..8a7253e --- /dev/null +++ b/0010-2245-fix-utils_verify_ut-failure-when-remote-grpc-fu.patch @@ -0,0 +1,33 @@ +From c8415e904333c99a2fcd4f8d070942b6923d44ed Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Mon, 13 Nov 2023 08:52:42 +0000 +Subject: [PATCH 10/14] !2245 fix utils_verify_ut failure when remote grpc + function is turned off * fix utils_verify_ut failure when remote grpc + function is turned off + +--- + test/cutils/utils_verify/utils_verify_ut.cc | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/test/cutils/utils_verify/utils_verify_ut.cc b/test/cutils/utils_verify/utils_verify_ut.cc +index 79670ec1..f2ff57c7 100644 +--- a/test/cutils/utils_verify/utils_verify_ut.cc ++++ b/test/cutils/utils_verify/utils_verify_ut.cc +@@ -70,12 +70,14 @@ TEST(utils_verify, test_util_validate_socket) + ASSERT_EQ(util_validate_socket("unix://./isulad"), false); + ASSERT_EQ(util_validate_socket("unix://isulad"), false); + ++#ifdef ENABLE_GRPC_REMOTE_CONNECT + ASSERT_EQ(util_validate_socket("tcp://localhost:2375"), true); + ASSERT_EQ(util_validate_socket("tcp://127.0.0.1:2375"), true); + + ASSERT_EQ(util_validate_socket("tcp://"), false); + ASSERT_EQ(util_validate_socket("tcp://127.0.0.1"), false); + ASSERT_EQ(util_validate_socket("tcp://127.0.0.1,2375"), false); ++#endif + } + + TEST(utils_verify, test_util_valid_device_mode) +-- +2.42.0 + diff --git a/0011-add-runc-attach-implement.patch b/0011-add-runc-attach-implement.patch new file mode 100644 index 0000000..088e339 --- /dev/null +++ b/0011-add-runc-attach-implement.patch @@ -0,0 +1,1312 @@ +From aaf1b46c66aa596ec718c11c4f4270e41e7b570e Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Tue, 7 Nov 2023 16:39:35 +0800 +Subject: [PATCH 11/14] add runc attach implement + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + .../connect/grpc/grpc_containers_client.cc | 9 + + src/cmd/isula/stream/attach.c | 12 +- + src/cmd/isulad-shim/common.c | 116 ++++- + src/cmd/isulad-shim/common.h | 30 ++ + src/cmd/isulad-shim/main.c | 14 + + src/cmd/isulad-shim/process.c | 453 ++++++++++++++++-- + src/cmd/isulad-shim/process.h | 7 +- + src/cmd/isulad-shim/terminal.c | 2 +- + .../executor/container_cb/execution_stream.c | 1 + + src/daemon/modules/api/runtime_api.h | 1 + + .../modules/runtime/isula/isula_rt_ops.c | 168 ++++++- + src/utils/cutils/error.h | 4 +- + 12 files changed, 755 insertions(+), 62 deletions(-) + +diff --git a/src/client/connect/grpc/grpc_containers_client.cc b/src/client/connect/grpc/grpc_containers_client.cc +index 2dd73100..bcb1e8da 100644 +--- a/src/client/connect/grpc/grpc_containers_client.cc ++++ b/src/client/connect/grpc/grpc_containers_client.cc +@@ -1394,6 +1394,8 @@ public: + auto run(const struct isula_attach_request *request, struct isula_attach_response *response) -> int override + { + ClientContext context; ++ bool detach = false; ++ std::string attach_detach_msg = "read escape sequence"; + + if (set_custom_header_metadata(context, request) != 0) { + ERROR("Failed to translate request to grpc"); +@@ -1415,6 +1417,9 @@ public: + break; + } + if (!stream_response.stdout().empty()) { ++ if (strcmp(stream_response.stdout().c_str(), attach_detach_msg.c_str()) == 0) { ++ detach = true; ++ } + std::cout << stream_response.stdout() << std::flush; + } + if (!stream_response.stderr().empty()) { +@@ -1437,6 +1442,10 @@ public: + response->cc = ISULAD_ERR_EXEC; + } + ++ if (detach) { ++ response->server_errono = ISULAD_INFO_DETACH; ++ } ++ + out: + if (request->attach_stdin) { + pthread_cancel(writer.native_handle()); +diff --git a/src/cmd/isula/stream/attach.c b/src/cmd/isula/stream/attach.c +index ff49af92..b61c9350 100644 +--- a/src/cmd/isula/stream/attach.c ++++ b/src/cmd/isula/stream/attach.c +@@ -37,6 +37,7 @@ + #include "connect.h" + #include "constants.h" + #include "client_helpers.h" ++#include "error.h" + #ifndef GRPC_CONNECTOR + #include "client_console.h" + #endif +@@ -70,8 +71,10 @@ static int check_tty(bool tty, struct termios *oldtios, bool *reset_tty) + } + *reset_tty = true; + } else { +- INFO("the input device is not a TTY"); +- return 0; ++ // if it is trying to attach to a container TTY ++ // from a non-TTY client input stream, returns -1. ++ COMMAND_ERROR("the input device is not a TTY"); ++ return -1; + } + + return 0; +@@ -353,6 +356,7 @@ static int client_attach(struct client_arguments *args, uint32_t *exit_code) + #endif + + config = get_connect_config(args); ++ // Obtain the container's real exit code by waiting for the container to stop. + container_wait_thread(args, exit_code, &sem_exited); + ret = ops->container.attach(&request, response, &config); + if (ret != 0) { +@@ -374,7 +378,9 @@ static int client_attach(struct client_arguments *args, uint32_t *exit_code) + + if (sem_timedwait(&sem_exited, &ts) != 0) { + if (errno == ETIMEDOUT) { +- COMMAND_ERROR("Wait container status timeout."); ++ if (response->server_errono != ISULAD_INFO_DETACH) { ++ INFO("Wait container stopped status timeout."); ++ } + } else { + CMD_SYSERROR("Failed to wait sem"); + } +diff --git a/src/cmd/isulad-shim/common.c b/src/cmd/isulad-shim/common.c +index 781dc004..48d266dc 100644 +--- a/src/cmd/isulad-shim/common.c ++++ b/src/cmd/isulad-shim/common.c +@@ -33,16 +33,26 @@ + #include <isula_libutils/utils_file.h> + + int g_log_fd = -1; ++int g_attach_log_fd = -1; + + int init_shim_log(void) + { +- g_log_fd = open_no_inherit(SHIM_LOG_NAME, O_CREAT | O_WRONLY | O_APPEND | O_SYNC, 0640); ++ g_log_fd = open_no_inherit(SHIM_LOG_NAME, O_CREAT | O_WRONLY | O_APPEND | O_SYNC, LOG_FILE_MODE); + if (g_log_fd < 0) { + return SHIM_ERR; + } + return SHIM_OK; + } + ++int init_attach_log(void) ++{ ++ g_attach_log_fd = open_no_inherit(ATTACH_LOG_NAME, O_CREAT | O_WRONLY | O_APPEND | O_SYNC, LOG_FILE_MODE); ++ if (g_attach_log_fd < 0) { ++ return SHIM_ERR; ++ } ++ return SHIM_OK; ++} ++ + void signal_routine(int sig) + { + switch (sig) { +@@ -162,11 +172,24 @@ int generate_random_str(char *id, size_t len) + return SHIM_OK; + } + +-void write_message(const char *level, const char *fmt, ...) +-{ + #define MAX_MSG_JSON_TEMPLATE 32 + #define MAX_MESSAGE_CONTENT_LEN 128 + #define MAX_MESSAGE_LEN (MAX_MSG_JSON_TEMPLATE + MAX_MESSAGE_CONTENT_LEN) ++ ++static void format_log_msg(const char *level, const char *buf, char *msg, int max_message_len) ++{ ++ time_t current_time = time(NULL); ++ struct tm *local_time = localtime(¤t_time); ++ char time_str[20]; ++ ++ strftime(time_str, sizeof(time_str), "%Y-%m-%d %H:%M:%S", local_time); ++ ++ (void)snprintf(msg, max_message_len - 1, "{\"time\": \"%s\", \"level\": \"%s\", \"msg\": \"%s\"}\n", time_str, level, ++ buf); ++} ++ ++void write_message(const char *level, const char *fmt, ...) ++{ + if (g_log_fd < 0) { + return; + } +@@ -183,15 +206,31 @@ void write_message(const char *level, const char *fmt, ...) + return; + } + +- nwrite = snprintf(msg, MAX_MESSAGE_LEN - 1, "{\"level\": \"%s\", \"msg\": \"%s\"}\n", level, buf); +- if (nwrite < 0 || (size_t)nwrite >= (MAX_MESSAGE_LEN - 1)) { ++ format_log_msg(level, buf, msg, MAX_MESSAGE_CONTENT_LEN); ++ ++ (void)isula_file_total_write_nointr(g_log_fd, msg, strlen(msg)); ++} ++ ++void write_attach_message(const char *level, const char *fmt, ...) ++{ ++ char buf[MAX_MESSAGE_CONTENT_LEN] = { 0 }; ++ char msg[MAX_MESSAGE_LEN] = { 0 }; ++ int nwrite = -1; ++ ++ if (g_attach_log_fd < 0) { + return; + } +- +- nwrite = isula_file_total_write_nointr(g_log_fd, msg, strlen(msg)); +- if (nwrite < 0 || (size_t)nwrite != strlen(msg)) { ++ va_list arg_list; ++ va_start(arg_list, fmt); ++ nwrite = vsnprintf(buf, MAX_MESSAGE_CONTENT_LEN, fmt, arg_list); ++ va_end(arg_list); ++ if (nwrite < 0) { + return; + } ++ ++ format_log_msg(level, buf, msg, MAX_MESSAGE_CONTENT_LEN); ++ ++ (void)isula_file_total_write_nointr(g_attach_log_fd, msg, strlen(msg)); + } + + /* note: This function can only read small text file. */ +@@ -272,3 +311,64 @@ int open_no_inherit(const char *path, int flag, mode_t mode) + + return fd; + } ++ ++/* judge the fd whether is attach fifo */ ++struct isula_linked_list *get_attach_fifo_item(int fd, struct isula_linked_list *list) ++{ ++ struct isula_linked_list *it = NULL; ++ struct isula_linked_list *next = NULL; ++ ++ if (fd <= 0 || list == NULL || isula_linked_list_empty(list)) { ++ return it; ++ } ++ ++ isula_linked_list_for_each_safe(it, list, next) { ++ struct shim_fifos_fd *elem = (struct shim_fifos_fd *)it->elem; ++ if (elem == NULL) { ++ continue; ++ } ++ if (elem->in_fd == fd) { ++ return it; ++ } ++ if (elem->out_fd == fd) { ++ return it; ++ } ++ if (elem->err_fd == fd) { ++ return it; ++ } ++ } ++ ++ return it; ++} ++ ++void free_shim_fifos_fd(struct shim_fifos_fd *item) ++{ ++ if (item == NULL) { ++ return; ++ } ++ if (item->in_fifo != NULL) { ++ free(item->in_fifo); ++ item->in_fifo = NULL; ++ } ++ if (item->out_fifo != NULL) { ++ free(item->out_fifo); ++ item->out_fifo = NULL; ++ } ++ if (item->err_fifo != NULL) { ++ free(item->err_fifo); ++ item->err_fifo = NULL; ++ } ++ if (item->in_fd >= 0) { ++ close(item->in_fd); ++ item->in_fd = -1; ++ } ++ if (item->out_fd >= 0) { ++ close(item->out_fd); ++ item->out_fd = -1; ++ } ++ if (item->err_fd >= 0) { ++ close(item->err_fd); ++ item->err_fd = -1; ++ } ++ free(item); ++} +\ No newline at end of file +diff --git a/src/cmd/isulad-shim/common.h b/src/cmd/isulad-shim/common.h +index 55efdc28..2020a799 100644 +--- a/src/cmd/isulad-shim/common.h ++++ b/src/cmd/isulad-shim/common.h +@@ -21,6 +21,7 @@ + #include <sys/types.h> + #include <stdint.h> + #include <isula_libutils/utils.h> ++#include <isula_libutils/utils_linked_list.h> + + #ifdef __cplusplus + extern "C" { +@@ -59,8 +60,22 @@ extern "C" { + #define CONTAINER_ACTION_REBOOT 129 + #define CONTAINER_ACTION_SHUTDOWN 130 + ++#define ATTACH_SOCKET "attach_socket.sock" ++#define ATTACH_LOG_NAME "attach-log.json" ++#define ATTACH_DETACH_MSG "read escape sequence" ++#define MAX_ATTACH_NUM 16 ++ ++#define CTRL_Q 0x11 // ASCII code control character ctrl + Q ++ ++#define LOG_FILE_MODE 0600 ++ ++#define SOCKET_DIRECTORY_MODE 0600 ++#define ATTACH_FIFOPATH_MODE 0600 ++ + int init_shim_log(void); + ++int init_attach_log(void); ++ + void signal_routine(int sig); + + /** +@@ -90,18 +105,33 @@ void signal_routine(int sig); + } \ + } while (0) + ++struct shim_fifos_fd { ++ char *in_fifo; ++ char *out_fifo; ++ char *err_fifo; ++ int in_fd; ++ int out_fd; ++ int err_fd; ++}; ++ + char *read_text_file(const char *path); + + int cmd_combined_output(const char *binary, const char *params[], void *output, int *output_len); + + void write_message(const char *level, const char *fmt, ...); + ++void write_attach_message(const char *level, const char *fmt, ...); ++ + int generate_random_str(char *id, size_t len); + + void close_fd(int *pfd); + + int open_no_inherit(const char *path, int flag, mode_t mode); + ++struct isula_linked_list *get_attach_fifo_item(int fd, struct isula_linked_list *list); ++ ++void free_shim_fifos_fd(struct shim_fifos_fd *item); ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/cmd/isulad-shim/main.c b/src/cmd/isulad-shim/main.c +index 454011d0..deb07271 100644 +--- a/src/cmd/isulad-shim/main.c ++++ b/src/cmd/isulad-shim/main.c +@@ -145,6 +145,20 @@ int main(int argc, char **argv) + } + } + ++ if (p->state->attach_socket != NULL) { ++ ret = prepare_attach_socket(p); ++ if (ret != SHIM_OK) { ++ write_message(ERR_MSG, "failed to prepare attach socket:%d", ret); ++ exit(EXIT_FAILURE); ++ } ++ ++ ret = init_attach_log(); ++ if (ret != SHIM_OK) { ++ write_message(ERR_MSG, "failed to init shim attach log"); ++ exit(EXIT_FAILURE); ++ } ++ } ++ + /* start epoll for io copy */ + ret = process_io_start(p, &tid_epoll); + if (ret != SHIM_OK) { +diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c +index 40908102..187067d2 100644 +--- a/src/cmd/isulad-shim/process.c ++++ b/src/cmd/isulad-shim/process.c +@@ -19,20 +19,22 @@ + #include <sys/types.h> + #include <sys/socket.h> + #include <sys/un.h> ++#include <sys/ioctl.h> ++#include <sys/eventfd.h> ++#include <sys/resource.h> // IWYU pragma: keep ++#include <sys/stat.h> ++#include <sys/wait.h> + #include <fcntl.h> + #include <unistd.h> + #include <limits.h> +-#include <sys/wait.h> + #include <semaphore.h> + #include <stdlib.h> + #include <string.h> + #include <errno.h> +-#include <sys/ioctl.h> +-#include <sys/eventfd.h> + #include <termios.h> // IWYU pragma: keep +-#include <sys/resource.h> // IWYU pragma: keep + #include <stdint.h> + #include <stdio.h> ++#include <stddef.h> + + #include <isula_libutils/json_common.h> + #include <isula_libutils/shim_client_process_state.h> +@@ -42,6 +44,8 @@ + #include <isula_libutils/utils_mainloop.h> + #include <isula_libutils/auto_cleanup.h> + #include <isula_libutils/utils_buffer.h> ++#include <isula_libutils/utils_linked_list.h> ++#include <isula_libutils/utils_array.h> + + #include "common.h" + #include "terminal.h" +@@ -57,7 +61,7 @@ static shim_client_process_state *load_process() + + p_state = shim_client_process_state_parse_file("process.json", NULL, &err); + if (p_state == NULL) { +- write_message(ERR_MSG, "parse process state failed"); ++ write_message(ERR_MSG, "parse process state failed: %s", err); + } + /* "err" will definitely be allocated memory in the function above */ + free(err); +@@ -168,6 +172,99 @@ static int sync_exit_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr + return EPOLL_LOOP_HANDLE_CLOSE; + } + ++static bool fifo_exists(const char *path) ++{ ++ struct stat sb; ++ int ret; ++ ++ ret = stat(path, &sb); ++ if (ret < 0) { ++ // could be something other than exist, just return false ++ return false; ++ } ++ ++ return S_ISFIFO(sb.st_mode); ++} ++ ++static int add_attach_terminal_fifos(const char *in, const char *out, const char *err, int *input_fd, process_t *p) ++{ ++ __isula_auto_close int fifofd_in = -1; ++ __isula_auto_close int fifofd_out = -1; ++ __isula_auto_close int fifofd_err = -1; ++ struct shim_fifos_fd *fifos = NULL; ++ struct isula_linked_list *node = NULL; ++ ++ bool invalid = (in != NULL && !fifo_exists(in)) || (out != NULL && !fifo_exists(out)) || (err != NULL && ++ !fifo_exists(err)); ++ if (invalid) { ++ write_attach_message(ERR_MSG, "File %s or %s or %s does not refer to a FIFO", in, out, err); ++ return -1; ++ } ++ ++ if (in != NULL) { ++ fifofd_in = isula_file_open(in, O_RDONLY | O_NONBLOCK | O_CLOEXEC, 0); ++ if (fifofd_in < 0) { ++ write_attach_message(ERR_MSG, "Failed to open FIFO: %s", in); ++ return -1; ++ } ++ } ++ ++ if (out != NULL) { ++ fifofd_out = isula_file_open(out, O_WRONLY | O_NONBLOCK | O_CLOEXEC, 0); ++ if (fifofd_out < 0) { ++ write_attach_message(ERR_MSG, "Failed to open FIFO: %s", out); ++ return -1; ++ } ++ } ++ ++ if (err != NULL) { ++ fifofd_err = isula_file_open(err, O_WRONLY | O_NONBLOCK | O_CLOEXEC, 0); ++ if (fifofd_err < 0) { ++ write_attach_message(ERR_MSG, "Failed to open FIFO: %s", err); ++ return -1; ++ } ++ } ++ ++ fifos = isula_common_calloc_s(sizeof(*fifos)); ++ if (fifos == NULL) { ++ write_attach_message(ERR_MSG, "Out of memory"); ++ goto err_out; ++ } ++ ++ fifos->in_fifo = isula_strdup_s(in); ++ fifos->out_fifo = isula_strdup_s(out); ++ fifos->err_fifo = isula_strdup_s(err); ++ ++ fifos->in_fd = isula_transfer_fd(fifofd_in); ++ fifos->out_fd = isula_transfer_fd(fifofd_out); ++ fifos->err_fd = isula_transfer_fd(fifofd_err); ++ node = isula_common_calloc_s(sizeof(struct isula_linked_list)); ++ if (node == NULL) { ++ write_attach_message(ERR_MSG, "Out of memory"); ++ goto err_out; ++ } ++ ++ node->elem = fifos; ++ isula_linked_list_add(p->attach_fifos, node); ++ ++ if (input_fd != NULL) { ++ *input_fd = fifos->in_fd; ++ } ++ ++ return 0; ++err_out: ++ free_shim_fifos_fd(fifos); ++ return -1; ++} ++ ++static void remove_attach_terminal_fifos(isula_epoll_descr_t *descr, struct isula_linked_list *item) ++{ ++ struct shim_fifos_fd *elem = (struct shim_fifos_fd *)item->elem; ++ isula_epoll_remove_handler(descr, elem->in_fd); ++ isula_linked_list_del(item); ++ free_shim_fifos_fd(elem); ++} ++ + static int stdin_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t *descr) + { + process_t *p = (process_t *)cbdata; +@@ -210,6 +307,57 @@ static int stdin_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t * + return EPOLL_LOOP_HANDLE_CONTINUE; + } + ++static int attach_stdin_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t *descr) ++{ ++ process_t *p = (process_t *)cbdata; ++ int r_count = 0; ++ int w_count = 0; ++ int *fd_to = NULL; ++ struct isula_linked_list *item; ++ ++ if (events & EPOLLHUP) { ++ write_message(ERR_MSG, "attach stdin %d received the EPOLLHUP event", fd); ++ goto err_out; ++ } ++ ++ if (!(events & EPOLLIN)) { ++ return EPOLL_LOOP_HANDLE_CONTINUE; ++ } ++ ++ (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF); ++ ++ r_count = isula_file_read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF); ++ if (r_count <= 0) { ++ write_message(ERR_MSG, "failed to read from attach stdin %d, error:%d", fd, SHIM_SYS_ERR(errno)); ++ goto err_out; ++ } ++ ++ if (p->state->terminal) { ++ fd_to = &(p->recv_fd); ++ } else { ++ fd_to = &(p->shim_io->in); ++ } ++ ++ if (fd_to == NULL || *fd_to == -1) { ++ return EPOLL_LOOP_HANDLE_CONTINUE; ++ } ++ w_count = isula_file_total_write_nointr(*fd_to, p->buf, r_count); ++ if (w_count < 0) { ++ /* When any error occurs, set the write fd -1 */ ++ write_message(WARN_MSG, "write in_fd %d error:%d", *fd_to, SHIM_SYS_ERR(errno)); ++ close(*fd_to); ++ *fd_to = -1; ++ } ++ ++ return EPOLL_LOOP_HANDLE_CONTINUE; ++err_out: ++ item = get_attach_fifo_item(fd, p->attach_fifos); ++ if (item != NULL && item->elem != NULL) { ++ remove_attach_terminal_fifos(descr, item); ++ } ++ return EPOLL_LOOP_HANDLE_CONTINUE; ++} ++ + static int stdout_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t *descr) + { + process_t *p = (process_t *)cbdata; +@@ -237,16 +385,29 @@ static int stdout_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t + + shim_write_container_log_file(p->terminal, STDID_OUT, p->buf, r_count); + +- if (p->isulad_io->out == -1) { ++ if (p->isulad_io->out != -1) { ++ w_count = isula_file_total_write_nointr(p->isulad_io->out, p->buf, r_count); ++ if (w_count < 0) { ++ /* When any error occurs, set the write fd -1 */ ++ write_message(WARN_MSG, "write out_fd %d error:%d", p->isulad_io->out, SHIM_SYS_ERR(errno)); ++ close(p->isulad_io->out); ++ p->isulad_io->out = -1; ++ } ++ } ++ ++ if (isula_linked_list_empty(p->attach_fifos)) { + return EPOLL_LOOP_HANDLE_CONTINUE; + } + +- w_count = isula_file_total_write_nointr(p->isulad_io->out, p->buf, r_count); +- if (w_count < 0) { +- /* When any error occurs, set the write fd -1 */ +- write_message(WARN_MSG, "write out_fd %d error:%d", p->isulad_io->out, SHIM_SYS_ERR(errno)); +- close(p->isulad_io->out); +- p->isulad_io->out = -1; ++ struct isula_linked_list *it = NULL; ++ struct isula_linked_list *next = NULL; ++ ++ isula_linked_list_for_each_safe(it, p->attach_fifos, next) { ++ struct shim_fifos_fd *elem = (struct shim_fifos_fd *)it->elem; ++ w_count = isula_file_total_write_nointr(elem->out_fd, p->buf, r_count); ++ if (w_count < 0) { ++ remove_attach_terminal_fifos(descr, it); ++ } + } + + return EPOLL_LOOP_HANDLE_CONTINUE; +@@ -279,16 +440,29 @@ static int stderr_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t + + shim_write_container_log_file(p->terminal, STDID_ERR, p->buf, r_count); + +- if (p->isulad_io->err == -1) { ++ if (p->isulad_io->err != -1) { ++ w_count = isula_file_total_write_nointr(p->isulad_io->err, p->buf, r_count); ++ if (w_count < 0) { ++ /* When any error occurs, set the write fd -1 */ ++ write_message(WARN_MSG, "write err_fd %d error:%d", p->isulad_io->err, SHIM_SYS_ERR(errno)); ++ close(p->isulad_io->err); ++ p->isulad_io->err = -1; ++ } ++ } ++ ++ if (isula_linked_list_empty(p->attach_fifos)) { + return EPOLL_LOOP_HANDLE_CONTINUE; + } + +- w_count = isula_file_total_write_nointr(p->isulad_io->err, p->buf, r_count); +- if (w_count < 0) { +- /* When any error occurs, set the write fd -1 */ +- write_message(WARN_MSG, "write err_fd %d error:%d", p->isulad_io->err, SHIM_SYS_ERR(errno)); +- close(p->isulad_io->err); +- p->isulad_io->err = -1; ++ struct isula_linked_list *it = NULL; ++ struct isula_linked_list *next = NULL; ++ ++ isula_linked_list_for_each_safe(it, p->attach_fifos, next) { ++ struct shim_fifos_fd *elem = (struct shim_fifos_fd *)it->elem; ++ w_count = isula_file_total_write_nointr(elem->out_fd, p->buf, r_count); ++ if (w_count < 0) { ++ remove_attach_terminal_fifos(descr, it); ++ } + } + + return EPOLL_LOOP_HANDLE_CONTINUE; +@@ -326,6 +500,159 @@ static int resize_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t + return EPOLL_LOOP_HANDLE_CONTINUE; + } + ++static bool attach_fifopath_security_check(process_t *p, const char *fifopath) ++{ ++ struct stat st = { 0 }; ++ char real_path[PATH_MAX] = { 0 }; ++ ++ if (isula_validate_absolute_path(fifopath) != 0) { ++ write_attach_message(ERR_MSG, "attach fifo path \"%s\" must be an valid absolute path", fifopath); ++ return false; ++ } ++ ++ if (realpath(fifopath, real_path) == NULL) { ++ write_attach_message(ERR_MSG, "Failed to get realpath for '%s': %s.", real_path, SHIM_SYS_ERR(errno)); ++ return false; ++ } ++ ++ if (!isula_has_prefix(real_path, p->workdir)) { ++ write_attach_message(ERR_MSG, "attach fifo path \"%s\" must be under the state path", real_path, p->workdir); ++ return false; ++ } ++ ++ if (lstat(real_path, &st) != 0) { ++ write_attach_message(ERR_MSG, "Failed to lstat %s : %s", real_path, SHIM_SYS_ERR(errno)); ++ return false; ++ } ++ ++ if (!S_ISFIFO(st.st_mode)) { ++ write_attach_message(ERR_MSG, "attach fifo path \"%s\" must be an FIFO", real_path); ++ return false; ++ } ++ ++ if ((st.st_mode & 0777) != ATTACH_FIFOPATH_MODE) { ++ write_attach_message(ERR_MSG, "attach fifo path \"%s\" permission invalid", real_path); ++ return false; ++ } ++ ++ if (st.st_uid != 0) { ++ write_attach_message(ERR_MSG, "attach fifo path \"%s\" uid invalid", real_path); ++ return false; ++ } ++ ++ return true; ++} ++ ++// attach_cb needs to read the content from communication fd and parse it. ++// at the same time, it also needs to establish a connection between the attach fd and the container fd. ++// 1. if it fails, it needs to write an error message to attach log file, ++// and write -1 to connection fd to let isulad know that it has failed. ++// 2. if it succeeds, write 0 to let isulad know that it is ready. ++// attach_cb returns EPOLL_LOOP_HANDLE_CONTINUE regardless of success or failure, ++// because whether the attach operation is successful or not does not affect the first process of the container. ++static int attach_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t *descr) ++{ ++ process_t *p = (process_t *)cbdata; ++ int r_count = 0; ++ char tmp_buf[BUFSIZ + 1] = { 0 }; ++ char *in = NULL, *out = NULL, *err = NULL; ++ int fifofd_in = -1; ++ isula_string_array *tmp_str_array = NULL; ++ int ret = 0; ++ // attach execution return value ++ int status = -1; ++ bool valid = true; ++ ++ // after receiving the event that isulad closes the connection, ++ // close the communication fd and remove it from epoll. ++ if (events & EPOLLHUP) { ++ close(fd); ++ isula_epoll_remove_handler(descr, fd); ++ return EPOLL_LOOP_HANDLE_CONTINUE; ++ } ++ ++ if (!(events & EPOLLIN)) { ++ return EPOLL_LOOP_HANDLE_CONTINUE; ++ } ++ ++ r_count = isula_file_read_nointr(fd, tmp_buf, sizeof(tmp_buf) - 1); ++ if (r_count <= 0) { ++ write_attach_message(ERR_MSG, "Failed to read msg from attach conn fd"); ++ goto out; ++ } ++ ++ // limit the number of attach connections to MAX_ATTACH_NUM ++ if (isula_linked_list_len(p->attach_fifos) >= MAX_ATTACH_NUM) { ++ write_attach_message(ERR_MSG, "The number of attach connections exceeds the limit:%d, and this connection is rejected.", ++ MAX_ATTACH_NUM); ++ goto out; ++ } ++ ++ tmp_str_array = isula_string_split_to_multi(tmp_buf, ' '); ++ if (tmp_str_array->len != 3) { ++ write_attach_message(ERR_MSG, "Invalid attach msg from isulad"); ++ goto out; ++ } ++ ++ for (int i = 0; i < tmp_str_array->len; i++) { ++ valid = valid && attach_fifopath_security_check(p, tmp_str_array->items[i]); ++ } ++ ++ if (!valid) { ++ write_attach_message(ERR_MSG, "Invalid attach fifo path from isulad"); ++ goto out; ++ } ++ ++ in = tmp_str_array->items[0]; ++ out = tmp_str_array->items[1]; ++ err = tmp_str_array->items[2]; ++ ++ if (add_attach_terminal_fifos(in, out, err, &fifofd_in, p) < 0) { ++ write_attach_message(ERR_MSG, "Failed to add attach terminal fifos"); ++ goto out; ++ } ++ ++ // attach stdin --> container stdin ++ ret = isula_epoll_add_handler(descr, fifofd_in, attach_stdin_cb, p); ++ if (ret != SHIM_OK) { ++ write_attach_message(ERR_MSG, "add fifofd_in fd %d to epoll loop failed:%d", fifofd_in, SHIM_SYS_ERR(errno)); ++ struct isula_linked_list *item = get_attach_fifo_item(fd, p->attach_fifos); ++ if (item != NULL && item->elem != NULL) { ++ remove_attach_terminal_fifos(descr, item); ++ } ++ goto out; ++ } ++ ++ status = 0; ++out: ++ isula_string_array_free(tmp_str_array); ++ (void)isula_file_write_nointr(fd, &status, sizeof(int)); ++ return EPOLL_LOOP_HANDLE_CONTINUE; ++} ++ ++// do_attach_socket_accept returns EPOLL_LOOP_HANDLE_CONTINUE regardless of success or failure, ++// because whether the attach operation is successful or not does not affect the first process of the container. ++static int do_attach_socket_accept(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t *descr) ++{ ++ process_t *p = (process_t *)cbdata; ++ int conn_fd = -1; ++ int ret = SHIM_ERR; ++ ++ conn_fd = accept(p->attach_socket_fd, NULL, NULL); ++ if (conn_fd < 0) { ++ write_attach_message(ERR_MSG, "accept from fd %d failed:%d", p->attach_socket_fd, SHIM_SYS_ERR(errno)); ++ return EPOLL_LOOP_HANDLE_CONTINUE; ++ } ++ ++ ret = isula_epoll_add_handler(descr, conn_fd, attach_cb, p); ++ if (ret != SHIM_OK) { ++ write_attach_message(ERR_MSG, "add recv_fd %d to epoll loop failed:%d", conn_fd, SHIM_SYS_ERR(errno)); ++ close(conn_fd); ++ return EPOLL_LOOP_HANDLE_CONTINUE; ++ } ++ return EPOLL_LOOP_HANDLE_CONTINUE; ++} ++ + static int task_console_accept(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t *descr) + { + process_t *p = (process_t *)cbdata; +@@ -399,6 +726,7 @@ static void stdio_release(int (*stdio_fd)[2]) + for (j = 0; j < 2; j++) { + if (stdio_fd[i][j] > 0) { + close(stdio_fd[i][j]); ++ stdio_fd[i][j] = -1; + } + } + } +@@ -568,24 +896,6 @@ static int open_generic_io(process_t *p, isula_epoll_descr_t *descr) + return SHIM_OK; + } + +-static int set_non_block(int fd) +-{ +- int flag = -1; +- int ret = SHIM_ERR; +- +- flag = fcntl(fd, F_GETFL, 0); +- if (flag < 0) { +- return SHIM_ERR; +- } +- +- ret = fcntl(fd, F_SETFL, flag | O_NONBLOCK); +- if (ret != 0) { +- return SHIM_ERR; +- } +- +- return SHIM_OK; +-} +- + /* + std_id: channel type + isulad_stdio: one side of the isulad fifo file +@@ -623,6 +933,14 @@ static void *io_epoll_loop(void *data) + exit(EXIT_FAILURE); + } + ++ if (p->state->attach_socket != NULL) { ++ ret = isula_epoll_add_handler(&descr, p->attach_socket_fd, do_attach_socket_accept, p); ++ if (ret != SHIM_OK) { ++ write_message(ERR_MSG, "add attach_socket_fd %d to epoll loop failed:%d", p->attach_socket_fd, SHIM_SYS_ERR(errno)); ++ exit(EXIT_FAILURE); ++ } ++ } ++ + if (p->state->terminal) { + ret = open_terminal_io(p, &descr); + } else { +@@ -651,7 +969,7 @@ static void *io_epoll_loop(void *data) + } + + if (fd_out > 0) { +- ret = set_non_block(fd_out); ++ ret = isula_set_non_block(fd_out); + if (ret != SHIM_OK) { + write_message(ERR_MSG, "set fd %d non_block failed:%d", fd_out, SHIM_SYS_ERR(errno)); + exit(EXIT_FAILURE); +@@ -666,7 +984,7 @@ static void *io_epoll_loop(void *data) + } + + if (fd_err > 0) { +- ret = set_non_block(fd_err); ++ ret = isula_set_non_block(fd_err); + if (ret != SHIM_OK) { + write_message(ERR_MSG, "set fd %d non_block failed:%d", fd_err, SHIM_SYS_ERR(errno)); + exit(EXIT_FAILURE); +@@ -807,15 +1125,19 @@ failure: + if (p->isulad_io != NULL) { + if (p->isulad_io->in > 0) { + close(p->isulad_io->in); ++ p->isulad_io->in = -1; + } + if (p->isulad_io->out > 0) { + close(p->isulad_io->out); ++ p->isulad_io->out = -1; + } + if (p->isulad_io->err > 0) { + close(p->isulad_io->err); ++ p->isulad_io->err = -1; + } + if (p->isulad_io->resize > 0) { + close(p->isulad_io->resize); ++ p->isulad_io->resize = -1; + } + free(p->isulad_io); + p->isulad_io = NULL; +@@ -937,6 +1259,13 @@ process_t *new_process(char *id, char *bundle, char *runtime) + goto failure; + } + ++ p->attach_fifos = isula_common_calloc_s(sizeof(struct isula_linked_list)); ++ if (p->attach_fifos == NULL) { ++ goto failure; ++ } ++ ++ isula_linked_list_init(p->attach_fifos); ++ + return p; + + failure: +@@ -1368,3 +1697,49 @@ int process_signal_handle_routine(process_t *p, const pthread_t tid_epoll, const + (void)isula_file_write_nointr(STDOUT_FILENO, &status, sizeof(int)); + return SHIM_OK; + } ++ ++int prepare_attach_socket(process_t *p) ++{ ++ struct sockaddr_un addr; ++ int ret = -1; ++ ++ if (strlen(p->state->attach_socket) >= sizeof(addr.sun_path)) { ++ write_message(ERR_MSG, "Invalid attach socket path: %s", p->state->attach_socket); ++ return SHIM_ERR; ++ } ++ ++ p->attach_socket_fd = socket(AF_UNIX, SOCK_STREAM, 0); ++ if (p->attach_socket_fd < 0) { ++ write_message(ERR_MSG, "Failed to create socket:%d", SHIM_SYS_ERR(errno)); ++ return SHIM_ERR; ++ } ++ ++ (void)memset(&addr, 0, sizeof(addr)); ++ addr.sun_family = AF_UNIX; ++ (void)strncpy(addr.sun_path, p->state->attach_socket, sizeof(addr.sun_path) - 1); ++ ++ ret = bind(p->attach_socket_fd, (struct sockaddr *)&addr, sizeof(addr)); ++ if (ret < 0) { ++ write_message(ERR_MSG, "bind console fd failed:%d", SHIM_SYS_ERR(errno)); ++ return SHIM_ERR; ++ } ++ ++ ret = chmod(p->state->attach_socket, SOCKET_DIRECTORY_MODE); ++ if (ret != 0) { ++ write_message(ERR_MSG, "Failed to chmod for socket: %s", p->state->attach_socket); ++ return SHIM_ERR; ++ } ++ ++ //If the backlog argument is greater than the value in ++ // /proc/sys/net/core/somaxconn, then it is silently capped to that ++ // value. Since Linux 5.4, the default in this file is 4096; in ++ // earlier kernels, the default value is 128. Before Linux 2.4.25, ++ // this limit was a hard coded value, SOMAXCONN, with the value 128. ++ // The maximum number of attach we allow here is MAX_ATTACH_NUM, so just use it directly ++ ret = listen(p->attach_socket_fd, MAX_ATTACH_NUM); ++ if (ret < 0) { ++ write_message(ERR_MSG, "listen console fd failed:%d", SHIM_SYS_ERR(errno)); ++ return SHIM_ERR; ++ } ++ return SHIM_OK; ++} +\ No newline at end of file +diff --git a/src/cmd/isulad-shim/process.h b/src/cmd/isulad-shim/process.h +index 280d9874..5607316c 100644 +--- a/src/cmd/isulad-shim/process.h ++++ b/src/cmd/isulad-shim/process.h +@@ -19,7 +19,8 @@ + #include <pthread.h> + #include <semaphore.h> + #include <stdbool.h> +-#include "isula_libutils/shim_client_process_state.h" ++#include <isula_libutils/shim_client_process_state.h> ++#include "isula_libutils/utils_linked_list.h" + #include "terminal.h" + + #ifdef __cplusplus +@@ -49,6 +50,7 @@ typedef struct process { + char *root_path; + int io_loop_fd; + int exit_fd; ++ int attach_socket_fd; // the server socket fd that establishes a connection with isulad + int ctr_pid; + int sync_fd; + int listen_fd; +@@ -58,6 +60,7 @@ typedef struct process { + stdio_t *stdio; // shim to on runtime side, in:r out/err: w + stdio_t *shim_io; // shim io on isulad side, in: w out/err: r + stdio_t *isulad_io; // isulad io, in:r out/err: w ++ struct isula_linked_list *attach_fifos; /* isulad: fifos used to attach teminal */ + shim_client_process_state *state; + sem_t sem_mainloop; + char *buf; +@@ -70,6 +73,8 @@ typedef struct { + + process_t* new_process(char *id, char *bundle, char *runtime); + ++int prepare_attach_socket(process_t *p); ++ + int process_io_start(process_t *p, pthread_t *tid_epoll); + int create_process(process_t *p); + int process_signal_handle_routine(process_t *p, const pthread_t tid_epoll, const uint64_t timeout); +diff --git a/src/cmd/isulad-shim/terminal.c b/src/cmd/isulad-shim/terminal.c +index 0653dc45..1c063300 100644 +--- a/src/cmd/isulad-shim/terminal.c ++++ b/src/cmd/isulad-shim/terminal.c +@@ -162,7 +162,7 @@ static int shim_json_data_write(log_terminal *terminal, const char *buf, int rea + * shouldn't happen, otherwise, discard some last bytes. + */ + nret = isula_file_total_write_nointr(terminal->fd, buf, +- terminal->log_maxsize < read_count ? terminal->log_maxsize : read_count); ++ terminal->log_maxsize < read_count ? terminal->log_maxsize : read_count); + if (nret < 0) { + ret = -1; + goto out; +diff --git a/src/daemon/executor/container_cb/execution_stream.c b/src/daemon/executor/container_cb/execution_stream.c +index 7db96b19..124dcfe2 100644 +--- a/src/daemon/executor/container_cb/execution_stream.c ++++ b/src/daemon/executor/container_cb/execution_stream.c +@@ -346,6 +346,7 @@ static int container_attach_cb(const container_attach_request *request, containe + } + + params.rootpath = cont->root_path; ++ params.state = cont->state_path; + params.stdin = fifos[0]; + params.stdout = fifos[1]; + params.stderr = fifos[2]; +diff --git a/src/daemon/modules/api/runtime_api.h b/src/daemon/modules/api/runtime_api.h +index edb33d02..3c2100f5 100644 +--- a/src/daemon/modules/api/runtime_api.h ++++ b/src/daemon/modules/api/runtime_api.h +@@ -161,6 +161,7 @@ typedef struct _rt_resume_params_t { + + typedef struct _rt_attach_params_t { + const char *rootpath; ++ const char *state; + const char *stdin; + const char *stdout; + const char *stderr; +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index cb1ee26f..1787170b 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -18,6 +18,10 @@ + #include "isula_rt_ops.h" + #include <unistd.h> + #include <sys/wait.h> ++#include <stdio.h> ++#include <sys/socket.h> ++#include <sys/un.h> ++#include <sys/types.h> + #include <limits.h> + #include <errno.h> + #include <fcntl.h> +@@ -26,6 +30,7 @@ + #include <isula_libutils/isulad_daemon_configs.h> + #include <isula_libutils/json_common.h> + #include <isula_libutils/oci_runtime_spec.h> ++#include <isula_libutils/utils_file.h> + #include <signal.h> + #include <stdint.h> + #include <stdio.h> +@@ -52,9 +57,11 @@ + + #define SHIM_BINARY "isulad-shim" + #define RESIZE_FIFO_NAME "resize_fifo" ++#define ATTACH_SOCKET "attach_socket.sock" + #define SHIM_LOG_SIZE ((BUFSIZ - 100) / 2) + #define RESIZE_DATA_SIZE 100 + #define PID_WAIT_TIME 120 ++#define ATTACH_WAIT_TIME 120 + #define SHIM_EXIT_TIMEOUT 2 + + // file name formats of cgroup resources json +@@ -223,6 +230,19 @@ static void show_shim_runtime_errlog(const char *workdir) + isulad_set_error_message(buf); + } + ++static void show_shim_attach_errlog(const char *workdir) ++{ ++ char buf[SHIM_LOG_SIZE] = { 0 }; ++ ++ if (g_isulad_errmsg != NULL) { ++ return; ++ } ++ ++ get_err_message(buf, sizeof(buf), workdir, "attach-log.json"); ++ ERROR("shim-log: %s", buf); ++ isulad_set_error_message("shim-log error:\n%s\n", buf); ++} ++ + bool rt_isula_detect(const char *runtime) + { + if (runtime != NULL && (strcasecmp(runtime, "lcr") != 0)) { +@@ -463,8 +483,9 @@ static void runtime_exec_param_init(runtime_exec_info *rei) + } + } + +-static int runtime_exec_info_init(runtime_exec_info *rei, const char *workdir, const char *root_path, const char *runtime, const char *subcmd, +- const char **opts, size_t opts_len, const char *id, char **params, size_t params_num) ++static int runtime_exec_info_init(runtime_exec_info *rei, const char *workdir, const char *root_path, ++ const char *runtime, const char *subcmd, const char **opts, size_t opts_len, const char *id, char **params, ++ size_t params_num) + { + int ret = 0; + rei->workdir = workdir; +@@ -1012,6 +1033,7 @@ int rt_isula_create(const char *id, const char *runtime, const rt_create_params_ + size_t runtime_args_len = 0; + int ret = 0; + char workdir[PATH_MAX] = { 0 }; ++ char attach_socket[PATH_MAX] = { 0 }; + shim_client_process_state p = { 0 }; + int shim_exit_code = 0; + int nret = 0; +@@ -1034,6 +1056,13 @@ int rt_isula_create(const char *id, const char *runtime, const rt_create_params_ + goto out; + } + ++ nret = snprintf(attach_socket, sizeof(attach_socket), "%s/%s", workdir, ATTACH_SOCKET); ++ if (nret < 0 || (size_t)nret >= sizeof(attach_socket)) { ++ INFO("Failed to get full attach socket path"); ++ ret = -1; ++ goto out; ++ } ++ + p.exit_fifo = (char *)params->exit_fifo; + p.open_tty = params->tty; + p.open_stdin = params->open_stdin; +@@ -1042,6 +1071,7 @@ int rt_isula_create(const char *id, const char *runtime, const rt_create_params_ + p.isulad_stderr = (char *)params->stderr; + p.runtime_args = (char **)runtime_args; + p.runtime_args_len = runtime_args_len; ++ p.attach_socket = attach_socket; + copy_process(&p, config->process); + copy_annotations(&p, config->annotations); + +@@ -1224,7 +1254,7 @@ static bool fg_exec(const rt_exec_params_t *params) + return false; + } + +-static char *try_generate_exec_id() ++static char *try_generate_random_id() + { + char *id = NULL; + +@@ -1324,7 +1354,7 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p + if (params->suffix != NULL) { + exec_id = util_strdup_s(params->suffix); + } else { +- exec_id = try_generate_exec_id(); ++ exec_id = try_generate_random_id(); + } + if (exec_id == NULL) { + ERROR("Out of memory or generate exec id failed"); +@@ -1423,13 +1453,133 @@ out: + return ret; + } + +-int rt_isula_attach(const char *id, const char *runtime, const rt_attach_params_t *params) ++static int get_container_attach_statuscode(const char *workdir, int attach_shim_fd) + { +- ERROR("isula attach not support on isulad-shim"); +- isulad_set_error_message("isula attach not support on isulad-shim"); ++ int status_code = 0; ++ int ret = -1; ++ struct timespec beg = { 0 }; ++ struct timespec end = { 0 }; ++ ++ if (clock_gettime(CLOCK_MONOTONIC, &beg) != 0) { ++ ERROR("Failed get time"); ++ return -1; ++ } ++ ++ while (true) { ++ if (clock_gettime(CLOCK_MONOTONIC, &end) != 0) { ++ ERROR("Failed get time"); ++ return -1; ++ } ++ if (end.tv_sec - beg.tv_sec > ATTACH_WAIT_TIME) { ++ ERROR("Wait container attach exitcode timeout"); ++ return -1; ++ } ++ ret = util_read_nointr(attach_shim_fd, &status_code, sizeof(int)); ++ if (ret <= 0) { ++ if (shim_alive(workdir)) { ++ // wait 100 millisecond to read exit code ++ util_usleep_nointerupt(100000); ++ continue; ++ } ++ ERROR("Failed read pid from dead shim %s", workdir); ++ return -1; ++ } ++ return status_code; /* success */ ++ } + return -1; + } + ++static int get_attach_socketfd(const char *attach_socket, int *socket_fd) ++{ ++ struct sockaddr_un addr = { 0 }; ++ __isula_auto_close int tmp_socket = -1; ++ ++ if (strlen(attach_socket) >= sizeof(addr.sun_path)) { ++ SYSERROR("Invalid attach socket path: %s", attach_socket); ++ return -1; ++ } ++ ++ tmp_socket = socket(AF_UNIX, SOCK_STREAM, 0); ++ if (tmp_socket < 0) { ++ SYSERROR("Failed to create attach socket"); ++ return -1; ++ } ++ ++ if (isula_set_non_block(tmp_socket) < 0) { ++ SYSERROR("Failed to set socket non block"); ++ return -1; ++ } ++ ++ (void)memset(&addr, 0, sizeof(addr)); ++ addr.sun_family = AF_UNIX; ++ (void)strcpy(addr.sun_path, attach_socket); ++ ++ if (connect(tmp_socket, (void *)&addr, sizeof(addr)) < 0) { ++ SYSERROR("Failed to connect attach socket: %s", attach_socket); ++ return -1; ++ } ++ *socket_fd = isula_transfer_fd(tmp_socket); ++ return 0; ++} ++ ++int rt_isula_attach(const char *id, const char *runtime, const rt_attach_params_t *params) ++{ ++ int ret = 0; ++ int len = 0; ++ int status_code = 0; ++ __isula_auto_close int socket_fd = -1; ++ char buf[BUFSIZ] = { 0 }; ++ char workdir[PATH_MAX] = { 0 }; ++ char attach_socket[PATH_MAX] = { 0 }; ++ ++ if (id == NULL || runtime == NULL || params == NULL) { ++ ERROR("Null argument"); ++ return -1; ++ } ++ ++ ret = snprintf(workdir, sizeof(workdir), "%s/%s", params->state, id); ++ if (ret < 0 || (size_t)ret >= sizeof(workdir)) { ++ ERROR("Failed join exec full path"); ++ return -1; ++ } ++ ++ // the communication format between isulad and isulad-shim attach is: ++ // stdin-path stdout-path stderr-path ++ len = snprintf(buf, sizeof(buf), "%s %s %s", params->stdin, params->stdout, params->stderr); ++ if (len < 0 || (size_t)len >= sizeof(buf)) { ++ ERROR("Failed to snprintf string"); ++ return -1; ++ } ++ ++ ret = snprintf(attach_socket, sizeof(attach_socket), "%s/%s", workdir, ATTACH_SOCKET); ++ if (ret < 0 || (size_t)ret >= sizeof(attach_socket)) { ++ ERROR("Failed to get full attach socket path"); ++ return -1; ++ } ++ ++ ret = get_attach_socketfd(attach_socket, &socket_fd); ++ if (ret < 0) { ++ ERROR("Failed to get attach socketfd"); ++ return -1; ++ } ++ ++ DEBUG("write %s to attach fd", buf); ++ ++ ret = isula_file_write_nointr(socket_fd, buf, len); ++ if (ret < 0) { ++ SYSERROR("Failed to write attach isulad fd"); ++ return -1; ++ } ++ ++ status_code = get_container_attach_statuscode(workdir, socket_fd); ++ if (status_code < 0) { ++ show_shim_attach_errlog(workdir); ++ return -1; ++ } ++ ++ return 0; ++} ++ + static int to_engine_resources_unified(const host_config *hostconfig, shim_client_cgroup_resources *cr) + { + int i; +@@ -1673,7 +1823,7 @@ static int parse_ps_data(char *stdout_msg, rt_listpids_out_t *out) + } + + static int runtime_call_ps(const char *workdir, const char *runtime, const char *id, +- rt_listpids_out_t *out) ++ rt_listpids_out_t *out) + { + __isula_auto_free char *stdout_msg = NULL; + __isula_auto_free char *stderr_msg = NULL; +@@ -1681,7 +1831,7 @@ static int runtime_call_ps(const char *workdir, const char *runtime, const char + int ret = 0; + int nret = 0; + char *params[PARAM_NUM] = { 0 }; +- const char *opts[2] = { "--format" , "json" }; ++ const char *opts[2] = { "--format", "json" }; + char root_path[PATH_MAX] = { 0 }; + + nret = snprintf(root_path, PATH_MAX, "%s/%s", workdir, runtime); +diff --git a/src/utils/cutils/error.h b/src/utils/cutils/error.h +index 088ed261..75eae760 100644 +--- a/src/utils/cutils/error.h ++++ b/src/utils/cutils/error.h +@@ -44,8 +44,10 @@ extern "C" { + /* err in runtime module */ \ + XX(ERR_RUNTIME, DEF_ERR_RUNTIME_STR) \ + \ ++ /* info for detach */ \ ++ XX(INFO_DETACH, "Attach detach") \ + /* err max */ \ +- XX(ERR_UNKNOWN, "Unknown error") ++ XX(ERR_UNKNOWN, "Unknown error") + + #define ISULAD_ERRNO_GEN(n, s) ISULAD_##n, + typedef enum { ISULAD_ERRNO_MAP(ISULAD_ERRNO_GEN) } isulad_errno_t; +-- +2.42.0 + diff --git a/0012-add-runc-attach-implement-unit-test-and-ci-test.patch b/0012-add-runc-attach-implement-unit-test-and-ci-test.patch new file mode 100644 index 0000000..401d89b --- /dev/null +++ b/0012-add-runc-attach-implement-unit-test-and-ci-test.patch @@ -0,0 +1,242 @@ +From d37c0c7ded0e107167a98dc1eda2000142d274f0 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Tue, 7 Nov 2023 16:39:50 +0800 +Subject: [PATCH 12/14] add runc attach implement unit test and ci test + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + CI/test_cases/container_cases/attach.sh | 153 ++++++++++++++++++++ + CI/test_cases/container_cases/cri_stream.sh | 6 +- + test/cmd/isulad-shim/common/common_ut.cc | 42 ++++++ + 3 files changed, 197 insertions(+), 4 deletions(-) + create mode 100755 CI/test_cases/container_cases/attach.sh + +diff --git a/CI/test_cases/container_cases/attach.sh b/CI/test_cases/container_cases/attach.sh +new file mode 100755 +index 00000000..0d362757 +--- /dev/null ++++ b/CI/test_cases/container_cases/attach.sh +@@ -0,0 +1,153 @@ ++#!/bin/bash ++# ++# attributes: isula attach test ++# concurrent: NA ++# spend time: 5 ++ ++####################################################################### ++##- Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++# - iSulad licensed under the Mulan PSL v2. ++# - You can use this software according to the terms and conditions of the Mulan PSL v2. ++# - You may obtain a copy of Mulan PSL v2 at: ++# - http://license.coscl.org.cn/MulanPSL2 ++# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++# - PURPOSE. ++# - See the Mulan PSL v2 for more details. ++##- @Description:CI ++##- @Author: zhongtao ++##- @Create: 2023-11-06 ++####################################################################### ++ ++declare -r curr_path=$(dirname $(readlink -f "$0")) ++source ../helpers.sh ++ ++# $1 : retry limit ++# $2 : retry_interval ++# $3 : retry function ++function do_retry() ++{ ++ for i in $(seq 1 "$1"); do ++ $3 $4 $5 ++ if [ $? -ne 0 ]; then ++ return 0 ++ fi ++ sleep $2 ++ done ++ return 1 ++} ++ ++function get_ioCopy() ++{ ++ ps -T -p $(cat /var/run/isulad.pid) | grep IoCopy ++ return $? ++} ++ ++function inspect_container_status() ++{ ++ [[ $(isula inspect -f '{{.State.Status}}' ${1}) != "${2}" ]] ++ return $? ++} ++ ++function set_up() ++{ ++ local ret=0 ++ local runtime=$1 ++ ++ isula run -tid --name test --runtime $runtime busybox sh ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) ++ ++ msg_info "${test} finished with return ${ret}..." ++ return ${ret} ++} ++ ++function test_attach_fun() ++{ ++ local ret=0 ++ local retry_limit=20 ++ local retry_interval=1 ++ container_name="test" ++ local test="test_attach_fun => (${FUNCNAME[@]})" ++ ++ msg_info "${test} starting..." ++ ++ expect <<-END ++spawn isula attach test ++send \n ++expect "*" ++sleep 1 ++send "ls \r" ++expect "*" ++send "exit \r" ++expect "*" ++sleep 2 ++expect eof ++END ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to attach container test" && ((ret++)) ++ ++ count=$(isula logs test | grep ls | wc -l) ++ [[ $count -ne 1 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to do attach" && ((ret++)) ++ ++ do_retry ${retry_limit} ${retry_interval} inspect_container_status ${container_name} exited ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - incorrent container status: not Exited" && ((ret++)) ++ ++ (isula attach test > /tmp/test_attach1.log 2>&1) & ++ sleep 2 ++ cat /tmp/test_attach1.log | grep "You cannot attach to a stopped container, start it first" ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to do attach, except fail" && ((ret++)) ++ ++ rm -rf /tmp/test_attach1.log ++ ++ do_retry ${retry_limit} ${retry_interval} get_ioCopy ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - residual IO copy thread in CRI exec operation" && ((ret++)) ++ ++ msg_info "${test} finished with return ${ret}..." ++ return ${ret} ++} ++ ++function tear_down() ++{ ++ local ret=0 ++ ++ isula rm -f test ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container: test" && ((ret++)) ++ ++ return ${ret} ++} ++ ++function do_test_t() ++{ ++ local ret=0 ++ local runtime=$1 ++ local test="basic attach test => (${runtime})" ++ msg_info "${test} starting..." ++ ++ set_up $runtime || ((ret++)) ++ ++ test_attach_fun || ((ret++)) ++ ++ tear_down || ((ret++)) ++ ++ msg_info "${test} finished with return ${ret}..." ++ ++ return $ret ++} ++ ++ret=0 ++ ++isula pull busybox ++[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${image}" && return ${FAILURE} ++ ++isula images | grep busybox ++[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++)) ++ ++for element in ${RUNTIME_LIST[@]}; ++do ++ do_test_t $element ++ if [ $? -ne 0 ];then ++ let "ret=$ret + 1" ++ fi ++done ++ ++show_result $ret "basic attach" ++ +diff --git a/CI/test_cases/container_cases/cri_stream.sh b/CI/test_cases/container_cases/cri_stream.sh +index 2360e240..43ed3891 100755 +--- a/CI/test_cases/container_cases/cri_stream.sh ++++ b/CI/test_cases/container_cases/cri_stream.sh +@@ -187,10 +187,8 @@ function do_test_t() + test_cri_exec_fun || ((ret++)) + test_cri_exec_abn || ((ret++)) + +- # runc attach not support +- if [ $runtime == "lcr" ]; then +- test_cri_attach || ((ret++)) +- fi ++ test_cri_attach || ((ret++)) ++ + tear_down || ((ret++)) + + msg_info "${test} finished with return ${ret}..." +diff --git a/test/cmd/isulad-shim/common/common_ut.cc b/test/cmd/isulad-shim/common/common_ut.cc +index 63395232..fb60f628 100644 +--- a/test/cmd/isulad-shim/common/common_ut.cc ++++ b/test/cmd/isulad-shim/common/common_ut.cc +@@ -87,3 +87,45 @@ TEST_F(CommonUnitTest, test_combined_output) + params[0] = non_cmd.c_str(); + EXPECT_EQ(cmd_combined_output(non_cmd.c_str(), params, output, &output_len), -1); + } ++ ++TEST_F(CommonUnitTest, test_get_attach_fifo_item) ++{ ++ struct isula_linked_list *attach_fifos = NULL; ++ attach_fifos = (struct isula_linked_list *)isula_common_calloc_s(sizeof(struct isula_linked_list)); ++ ASSERT_TRUE(attach_fifos != nullptr); ++ ++ isula_linked_list_init(attach_fifos); ++ ++ EXPECT_EQ(get_attach_fifo_item(4, attach_fifos), nullptr); ++ EXPECT_EQ(get_attach_fifo_item(-1, attach_fifos), nullptr); ++ EXPECT_EQ(get_attach_fifo_item(4, NULL), nullptr); ++ ++ struct shim_fifos_fd fifos1 = { ++ .in_fd = 1, ++ .out_fd = 2, ++ .err_fd = 3, ++ }; ++ struct shim_fifos_fd fifos2 = { ++ .in_fd = 4, ++ .out_fd = 5, ++ .err_fd = 6, ++ }; ++ struct isula_linked_list *node1 = NULL; ++ struct isula_linked_list *node2 = NULL; ++ node1 = (struct isula_linked_list *)isula_common_calloc_s(sizeof(struct isula_linked_list)); ++ ASSERT_TRUE(node1 != nullptr); ++ node1->elem = &fifos1; ++ isula_linked_list_add(attach_fifos, node1); ++ ++ node2 = (struct isula_linked_list *)isula_common_calloc_s(sizeof(struct isula_linked_list)); ++ ASSERT_TRUE(node2 != nullptr); ++ node2->elem = &fifos2; ++ isula_linked_list_add(attach_fifos, node2); ++ ++ EXPECT_EQ(get_attach_fifo_item(1, attach_fifos), node1); ++ EXPECT_EQ(get_attach_fifo_item(4, attach_fifos), node2); ++ ++ free(node1); ++ free(node2); ++ free(attach_fifos); ++} +-- +2.42.0 + diff --git a/0013-support-gcov-of-CI.patch b/0013-support-gcov-of-CI.patch new file mode 100644 index 0000000..14c4148 --- /dev/null +++ b/0013-support-gcov-of-CI.patch @@ -0,0 +1,26 @@ +From f4ea9145cea40cb97a86cd5eb91e1726cf48dd0d Mon Sep 17 00:00:00 2001 +From: haozi007 <liuhao27@huawei.com> +Date: Tue, 14 Nov 2023 09:17:05 +0800 +Subject: [PATCH 13/14] support gcov of CI + +Signed-off-by: haozi007 <liuhao27@huawei.com> +--- + CI/dockerfiles/Dockerfile-fedora | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/CI/dockerfiles/Dockerfile-fedora b/CI/dockerfiles/Dockerfile-fedora +index c30a3d0b..bef44377 100644 +--- a/CI/dockerfiles/Dockerfile-fedora ++++ b/CI/dockerfiles/Dockerfile-fedora +@@ -115,6 +115,8 @@ RUN echo "[source.crates-io]" >> ${HOME}/.cargo/config && \ + echo "[source.local-registry]" >> ${HOME}/.cargo/config && \ + echo "directory = \"vendor\"" >> ${HOME}/.cargo/config + ++RUN dnf install -y lcov && dnf clean all ++ + # install libevhtp + RUN export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH && \ + set -x && \ +-- +2.42.0 + diff --git a/0014-compatibility-for-manage-pods-which-created-by-old-i.patch b/0014-compatibility-for-manage-pods-which-created-by-old-i.patch new file mode 100644 index 0000000..ff36bcc --- /dev/null +++ b/0014-compatibility-for-manage-pods-which-created-by-old-i.patch @@ -0,0 +1,163 @@ +From 2bf2acb51aec12e734c970b02cd7802f088a2222 Mon Sep 17 00:00:00 2001 +From: haozi007 <liuhao27@huawei.com> +Date: Tue, 14 Nov 2023 10:29:34 +0800 +Subject: [PATCH 14/14] compatibility for manage pods which created by old + iSulad + +Signed-off-by: haozi007 <liuhao27@huawei.com> +--- + .../cri_pod_sandbox_manager_service.cc | 12 +++- + src/daemon/entry/cri/v1alpha/naming.cc | 72 ++++++++++++++++--- + src/daemon/entry/cri/v1alpha/naming.h | 2 +- + 3 files changed, 72 insertions(+), 14 deletions(-) + +diff --git a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +index 6e8f40b9..8533bb8c 100644 +--- a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +@@ -1153,7 +1153,11 @@ void PodSandboxManagerService::PodSandboxStatusToGRPC(const container_inspect *i + + CRIHelpers::ExtractLabels(inspect->config->labels, *podStatus->mutable_labels()); + CRIHelpers::ExtractAnnotations(inspect->config->annotations, *podStatus->mutable_annotations()); +- CRINaming::ParseSandboxName(podStatus->annotations(), *podStatus->mutable_metadata(), error); ++ std::string name; ++ if (inspect->name != nullptr) { ++ name = std::string(inspect->name); ++ } ++ CRINaming::ParseSandboxName(name, podStatus->annotations(), *podStatus->mutable_metadata(), error); + if (error.NotEmpty()) { + return; + } +@@ -1266,7 +1270,11 @@ void PodSandboxManagerService::ListPodSandboxToGRPC(container_list_response *res + + CRIHelpers::ExtractAnnotations(response->containers[i]->annotations, *pod->mutable_annotations()); + +- CRINaming::ParseSandboxName(pod->annotations(), *pod->mutable_metadata(), error); ++ std::string name; ++ if (response->containers[i]->name != nullptr) { ++ name = std::string(response->containers[i]->name); ++ } ++ CRINaming::ParseSandboxName(name, pod->annotations(), *pod->mutable_metadata(), error); + + if (filterOutReadySandboxes && pod->state() == runtime::v1alpha2::SANDBOX_READY) { + continue; +diff --git a/src/daemon/entry/cri/v1alpha/naming.cc b/src/daemon/entry/cri/v1alpha/naming.cc +index abb6e57d..de47a97d 100644 +--- a/src/daemon/entry/cri/v1alpha/naming.cc ++++ b/src/daemon/entry/cri/v1alpha/naming.cc +@@ -26,6 +26,38 @@ + #include "utils.h" + + namespace CRINaming { ++// default sandbox name create by MakeSandboxName(); ++// format is 'k8s_containername_metadataname_namespace_uid_attempt' ++static int parseName(const std::string &name, std::vector<std::string> &items, unsigned int &attempt, Errors &err) ++{ ++ std::istringstream f(name); ++ std::string part; ++ ++ while (getline(f, part, CRI::Constants::nameDelimiterChar)) { ++ items.push_back(part); ++ } ++ ++ // format: k8s_containername_metadataname_namespace_uid_attempt ++ // so split name by '_', length of result array must be 6 ++ if (items.size() != 6) { ++ err.Errorf("failed to parse the sandbox name: %s", name.c_str()); ++ return -1; ++ } ++ ++ if (items[0] != CRI::Constants::kubePrefix) { ++ err.Errorf("container is not managed by kubernetes: %s", name.c_str()); ++ return -1; ++ } ++ ++ // last item index is 5, and must be attempt ++ if (util_safe_uint(items[5].c_str(), &attempt)) { ++ err.Errorf("failed to parse the sandbox name %s: %s", name.c_str(), strerror(errno)); ++ return -1; ++ } ++ ++ return 0; ++} ++ + std::string MakeSandboxName(const runtime::v1alpha2::PodSandboxMetadata &metadata) + { + std::string sname; +@@ -44,9 +76,12 @@ std::string MakeSandboxName(const runtime::v1alpha2::PodSandboxMetadata &metadat + return sname; + } + +-void ParseSandboxName(const google::protobuf::Map<std::string, std::string> &annotations, ++void ParseSandboxName(const std::string &name, const google::protobuf::Map<std::string, std::string> &annotations, + runtime::v1alpha2::PodSandboxMetadata &metadata, Errors &err) + { ++ // need check uid and attemp 2 items ++ int needSetUidAndAttemp = 2; ++ + if (annotations.count(CRIHelpers::Constants::SANDBOX_NAME_ANNOTATION_KEY) == 0) { + err.Errorf("annotation don't contains the sandbox name, failed to parse it"); + return; +@@ -57,21 +92,36 @@ void ParseSandboxName(const google::protobuf::Map<std::string, std::string> &ann + return; + } + +- if (annotations.count(CRIHelpers::Constants::SANDBOX_UID_ANNOTATION_KEY) == 0) { +- err.Errorf("annotation don't contains the sandbox uid, failed to parse it"); +- return; ++ metadata.set_name(annotations.at(CRIHelpers::Constants::SANDBOX_NAME_ANNOTATION_KEY)); ++ metadata.set_namespace_(annotations.at(CRIHelpers::Constants::SANDBOX_NAMESPACE_ANNOTATION_KEY)); ++ ++ if (annotations.count(CRIHelpers::Constants::SANDBOX_UID_ANNOTATION_KEY) != 0) { ++ metadata.set_uid(annotations.at(CRIHelpers::Constants::SANDBOX_UID_ANNOTATION_KEY)); ++ needSetUidAndAttemp--; ++ } ++ ++ if (annotations.count(CRIHelpers::Constants::SANDBOX_ATTEMPT_ANNOTATION_KEY) != 0) { ++ auto sandboxAttempt = annotations.at(CRIHelpers::Constants::SANDBOX_ATTEMPT_ANNOTATION_KEY); ++ metadata.set_attempt(static_cast<google::protobuf::uint32>(std::stoul(sandboxAttempt))); ++ needSetUidAndAttemp--; + } + +- if (annotations.count(CRIHelpers::Constants::SANDBOX_ATTEMPT_ANNOTATION_KEY) == 0) { +- err.Errorf("annotation don't contains the sandbox attempt, failed to parse it"); ++ if (needSetUidAndAttemp == 0) { + return; + } + +- metadata.set_name(annotations.at(CRIHelpers::Constants::SANDBOX_NAME_ANNOTATION_KEY)); +- metadata.set_namespace_(annotations.at(CRIHelpers::Constants::SANDBOX_NAMESPACE_ANNOTATION_KEY)); +- metadata.set_uid(annotations.at(CRIHelpers::Constants::SANDBOX_UID_ANNOTATION_KEY)); +- auto sandboxAttempt = annotations.at(CRIHelpers::Constants::SANDBOX_ATTEMPT_ANNOTATION_KEY); +- metadata.set_attempt(static_cast<google::protobuf::uint32>(std::stoul(sandboxAttempt))); ++ // get uid and attempt from name, ++ // compatibility to new iSulad manage pods created by old version iSulad ++ // maybe should remove in next version of iSulad ++ std::vector<std::string> items; ++ unsigned int attempt; ++ ++ if (parseName(name, items, attempt, err) != 0) { ++ return; ++ } ++ // index 4 in split array, must be uid ++ metadata.set_uid(items[4]); ++ metadata.set_attempt(static_cast<google::protobuf::uint32>(attempt)); + } + + std::string MakeContainerName(const runtime::v1alpha2::PodSandboxConfig &s, const runtime::v1alpha2::ContainerConfig &c) +diff --git a/src/daemon/entry/cri/v1alpha/naming.h b/src/daemon/entry/cri/v1alpha/naming.h +index 7eab41d3..f2d51a98 100644 +--- a/src/daemon/entry/cri/v1alpha/naming.h ++++ b/src/daemon/entry/cri/v1alpha/naming.h +@@ -26,7 +26,7 @@ std::string MakeSandboxName(const runtime::v1alpha2::PodSandboxMetadata &metadat + std::string MakeContainerName(const runtime::v1alpha2::PodSandboxConfig &s, + const runtime::v1alpha2::ContainerConfig &c); + +-void ParseSandboxName(const google::protobuf::Map<std::string, std::string> &annotations, ++void ParseSandboxName(const std::string &name, const google::protobuf::Map<std::string, std::string> &annotations, + runtime::v1alpha2::PodSandboxMetadata &metadata, Errors &err); + + void ParseContainerName(const google::protobuf::Map<std::string, std::string> &annotations, +-- +2.42.0 + diff --git a/0015-2250-Remove-PERFMON-BPF-CHECKPOINT_RESTORE.patch b/0015-2250-Remove-PERFMON-BPF-CHECKPOINT_RESTORE.patch new file mode 100644 index 0000000..20223a6 --- /dev/null +++ b/0015-2250-Remove-PERFMON-BPF-CHECKPOINT_RESTORE.patch @@ -0,0 +1,33 @@ +From ba0460408ab6118e1ecf3dda242af1d4b0980777 Mon Sep 17 00:00:00 2001 +From: xuxuepeng <xuxuepeng1@huawei.com> +Date: Tue, 14 Nov 2023 14:00:31 +0000 +Subject: [PATCH 15/16] !2250 Remove PERFMON, BPF, CHECKPOINT_RESTORE Merge + pull request !2250 from xuxuepeng/master + +--- + src/utils/cutils/utils_cap.c | 9 --------- + 1 file changed, 9 deletions(-) + +diff --git a/src/utils/cutils/utils_cap.c b/src/utils/cutils/utils_cap.c +index 6473df45..8c9cfafa 100644 +--- a/src/utils/cutils/utils_cap.c ++++ b/src/utils/cutils/utils_cap.c +@@ -74,15 +74,6 @@ const char *g_all_caps[] = { + #ifdef CAP_AUDIT_READ + "CAP_AUDIT_READ", + #endif +-#ifdef CAP_PERFMON +- "CAP_PERFMON", +-#endif +-#ifdef CAP_BPF +- "CAP_BPF", +-#endif +-#ifdef CAP_CHECKPOINT_RESTORE +- "CAP_CHECKPOINT_RESTORE", +-#endif + }; + + static inline size_t util_get_all_caps_len() +-- +2.42.0 + diff --git a/0016-improve-event-logs.patch b/0016-improve-event-logs.patch new file mode 100644 index 0000000..afc4d5e --- /dev/null +++ b/0016-improve-event-logs.patch @@ -0,0 +1,202 @@ +From d611f18abac0f4077c9bf85f76162719cc5e55eb Mon Sep 17 00:00:00 2001 +From: haozi007 <liuhao27@huawei.com> +Date: Tue, 14 Nov 2023 15:12:39 +0800 +Subject: [PATCH 16/64] improve event logs + +Signed-off-by: haozi007 <liuhao27@huawei.com> +--- + .../grpc/cri/v1/cri_v1_runtime_runtime_service.cc | 13 ++++++++----- + .../grpc/cri/v1alpha/cri_runtime_runtime_service.cc | 10 +++++++--- + src/daemon/entry/cri/cni_network_plugin.cc | 6 +++--- + .../cri/v1alpha/cri_pod_sandbox_manager_service.cc | 3 ++- + src/daemon/executor/volume_cb/volume_cb.c | 4 ++-- + src/daemon/modules/network/native/adaptor_native.c | 8 ++++---- + 6 files changed, 26 insertions(+), 18 deletions(-) + +diff --git a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc +index b8d5746c..1db79307 100644 +--- a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc ++++ b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc +@@ -75,7 +75,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::CreateContainer(grpc::ServerContext *c + return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, "Invalid input arguments"); + } + +- EVENT("Event: {Object: CRI, Type: Creating Container}"); ++ EVENT("Event: {Object: CRI, Type: Creating Container for sandbox: %s}", request->pod_sandbox_id().c_str()); + + std::string responseID = + m_rService->CreateContainer(request->pod_sandbox_id(), request->config(), request->sandbox_config(), error); +@@ -316,17 +316,20 @@ grpc::Status RuntimeV1RuntimeServiceImpl::RunPodSandbox(grpc::ServerContext *con + ERROR("Invalid input arguments"); + return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, "Invalid input arguments"); + } +- +- EVENT("Event: {Object: CRI, Type: Running Pod}"); ++ if (request->has_config() && request->config().has_metadata()) { ++ EVENT("Event: {Object: CRI, Type: Running Pod: %s}", request->config().metadata().name().c_str()); ++ } else { ++ EVENT("Event: {Object: CRI, Type: Running Pod}"); ++ } + + std::string responseID = m_rService->RunPodSandbox(request->config(), request->runtime_handler(), error); + if (!error.Empty() || responseID.empty()) { +- ERROR("Object: CRI, Type: Failed to run pod:%s", error.GetMessage().c_str()); ++ ERROR("Object: CRI, Type: Failed to run pod: %s", error.GetMessage().c_str()); + return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); + } + reply->set_pod_sandbox_id(responseID); + +- EVENT("Event: {Object: CRI, Type: Run Pod success}"); ++ EVENT("Event: {Object: CRI, Type: Run Pod: %s success}", responseID.c_str()); + + return grpc::Status::OK; + } +diff --git a/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.cc b/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.cc +index ec3f01cd..a56b167c 100644 +--- a/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.cc ++++ b/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.cc +@@ -73,7 +73,7 @@ grpc::Status RuntimeRuntimeServiceImpl::CreateContainer(grpc::ServerContext *con + return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, "Invalid input arguments"); + } + +- EVENT("Event: {Object: CRI, Type: Creating Container}"); ++ EVENT("Event: {Object: CRI, Type: Creating Container for sandbox: %s}", request->pod_sandbox_id().c_str()); + + std::string responseID = + m_rService->CreateContainer(request->pod_sandbox_id(), request->config(), request->sandbox_config(), error); +@@ -315,7 +315,11 @@ grpc::Status RuntimeRuntimeServiceImpl::RunPodSandbox(grpc::ServerContext *conte + return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, "Invalid input arguments"); + } + +- EVENT("Event: {Object: CRI, Type: Running Pod}"); ++ if (request->has_config() && request->config().has_metadata()) { ++ EVENT("Event: {Object: CRI, Type: Running Pod: %s}", request->config().metadata().name().c_str()); ++ } else { ++ EVENT("Event: {Object: CRI, Type: Running Pod}"); ++ } + + std::string responseID = m_rService->RunPodSandbox(request->config(), request->runtime_handler(), error); + if (!error.Empty() || responseID.empty()) { +@@ -324,7 +328,7 @@ grpc::Status RuntimeRuntimeServiceImpl::RunPodSandbox(grpc::ServerContext *conte + } + reply->set_pod_sandbox_id(responseID); + +- EVENT("Event: {Object: CRI, Type: Run Pod success}"); ++ EVENT("Event: {Object: CRI, Type: Run Pod: %s success}", responseID.c_str()); + + return grpc::Status::OK; + } +diff --git a/src/daemon/entry/cri/cni_network_plugin.cc b/src/daemon/entry/cri/cni_network_plugin.cc +index 656fceda..377796ee 100644 +--- a/src/daemon/entry/cri/cni_network_plugin.cc ++++ b/src/daemon/entry/cri/cni_network_plugin.cc +@@ -612,12 +612,12 @@ void CniNetworkPlugin::SetUpPod(const std::string &ns, const std::string &name, + if (g_isulad_errmsg != nullptr) { + err.SetError(g_isulad_errmsg); + } else { +- err.Errorf("setup cni for container: %s failed", id.c_str()); ++ err.Errorf("setup cni for sandbox: %s failed", id.c_str()); + } + // rollback all network plane + // if mutl-networks, one network plane failed, cause to left network can not be delete. + if (network_module_detach(config, NETWOKR_API_TYPE_CRI) != 0) { +- WARN("rollback all network for: %s failed", id.c_str()); ++ WARN("rollback all network for sandbox: %s failed", id.c_str()); + } + } + +@@ -671,7 +671,7 @@ void CniNetworkPlugin::TearDownPod(const std::string &ns, const std::string &nam + } + + if (network_module_detach(config, NETWOKR_API_TYPE_CRI) != 0) { +- err.Errorf("teardown cni for container: %s failed", id.c_str()); ++ err.Errorf("teardown cni for sandbox: %s failed", id.c_str()); + } + + UnlockNetworkMap(err); +diff --git a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +index 8533bb8c..8eff22ac 100644 +--- a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +@@ -618,6 +618,7 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1alpha2::PodSandbox + // Step 2: Create the sandbox container. + response_id = CreateSandboxContainer(config, image, jsonCheckpoint, runtimeHandler, error); + if (error.NotEmpty()) { ++ ERROR("Create sandbox failed: %s", error.GetCMessage()); + goto cleanup; + } + +@@ -672,7 +673,7 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1alpha2::PodSandbox + UpdatePodSandboxNetworkSettings(response_id, network_setting_json, tmpErr); + // If saving network settings failed, ignore error + if (tmpErr.NotEmpty()) { +- WARN("%s", tmpErr.GetCMessage()); ++ WARN("Update sandbox network setting err: %s", tmpErr.GetCMessage()); + } + } + goto cleanup; +diff --git a/src/daemon/executor/volume_cb/volume_cb.c b/src/daemon/executor/volume_cb/volume_cb.c +index 2148922e..ff5973b8 100644 +--- a/src/daemon/executor/volume_cb/volume_cb.c ++++ b/src/daemon/executor/volume_cb/volume_cb.c +@@ -52,7 +52,7 @@ static int volume_list_cb(const volume_list_volume_request *request, volume_list + goto err_out; + } + +- EVENT("Volume Event: {Object: list volumes, Type: listing}"); ++ INFO("Volume Event: {Object: list volumes, Type: listing}"); + + list = volume_list(); + if (list == NULL) { +@@ -85,7 +85,7 @@ static int volume_list_cb(const volume_list_volume_request *request, volume_list + } + + out: +- EVENT("Volume Event: {Object: list volumes, Type: listed"); ++ INFO("Volume Event: {Object: list volumes, Type: listed"); + + err_out: + if (*response != NULL) { +diff --git a/src/daemon/modules/network/native/adaptor_native.c b/src/daemon/modules/network/native/adaptor_native.c +index 45288d7e..baaecc32 100644 +--- a/src/daemon/modules/network/native/adaptor_native.c ++++ b/src/daemon/modules/network/native/adaptor_native.c +@@ -1510,7 +1510,7 @@ int native_config_inspect(const char *name, char **network_json) + return -1; + } + +- EVENT("Event: {Object: network, Type: inspecting, Target: %s}", name); ++ INFO("Event: {Object: network, Type: inspecting, Target: %s}", name); + + if (!native_store_lock(SHARED)) { + return -1; +@@ -1538,7 +1538,7 @@ int native_config_inspect(const char *name, char **network_json) + + // TODO: inspect the linked containers ip info + +- EVENT("Event: {Object: network, Type: inspected, Target: %s}", name); ++ INFO("Event: {Object: network, Type: inspected, Target: %s}", name); + goto out; + } + +@@ -1635,7 +1635,7 @@ int native_config_list(const struct filters_args *filters, network_network_info + return -1; + } + +- EVENT("Event: {Object: network, Type: listing}"); ++ INFO("Event: {Object: network, Type: listing}"); + + if (!native_store_lock(SHARED)) { + return -1; +@@ -1693,7 +1693,7 @@ int native_config_list(const struct filters_args *filters, network_network_info + *networks_len = nets_len; + nets_len = 0; + +- EVENT("Event: {Object: network, Type: listed}"); ++ INFO("Event: {Object: network, Type: listed}"); + + out: + map_itor_free(itor); +-- +2.42.0 + diff --git a/0017-2251-open-ENABLE_GRPC_REMOTE_CONNECT-in-CI.patch b/0017-2251-open-ENABLE_GRPC_REMOTE_CONNECT-in-CI.patch new file mode 100644 index 0000000..0002444 --- /dev/null +++ b/0017-2251-open-ENABLE_GRPC_REMOTE_CONNECT-in-CI.patch @@ -0,0 +1,57 @@ +From 48dc6f0adda72d7f4742afe1b8380370debfe4f4 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Wed, 15 Nov 2023 03:10:15 +0000 +Subject: [PATCH 17/64] !2251 open ENABLE_GRPC_REMOTE_CONNECT in CI * open + ENABLE_GRPC_REMOTE_CONNECT in CI + +--- + CI/make-and-install.sh | 4 ++-- + CI/only_build_isulad.sh | 2 +- + CI/pr-gateway.sh | 2 +- + 3 files changed, 4 insertions(+), 4 deletions(-) + +diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh +index cdd0e432..c1d26ff1 100755 +--- a/CI/make-and-install.sh ++++ b/CI/make-and-install.sh +@@ -103,9 +103,9 @@ rm -rf build + mkdir build + cd build + if [[ ${enable_gcov} -ne 0 ]]; then +- cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_CRI_API_V1=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON .. ++ cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_CRI_API_V1=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON -DENABLE_GRPC_REMOTE_CONNECT=ON .. + else +- cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON -DENABLE_CRI_API_V1=ON .. ++ cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON -DENABLE_CRI_API_V1=ON -DENABLE_GRPC_REMOTE_CONNECT=ON .. + fi + make -j $(nproc) + make install +diff --git a/CI/only_build_isulad.sh b/CI/only_build_isulad.sh +index d2d3417d..c3dc39d6 100755 +--- a/CI/only_build_isulad.sh ++++ b/CI/only_build_isulad.sh +@@ -67,7 +67,7 @@ popd + git clone https://gitee.com/openeuler/iSulad.git + pushd iSulad + mkdir build && pushd build +-cmake -DENABLE_UT=ON ../ ++cmake -DENABLE_UT=ON -DENABLE_GRPC_REMOTE_CONNECT=ON ../ + make -j2 && make install + ctest -V + popd +diff --git a/CI/pr-gateway.sh b/CI/pr-gateway.sh +index 291fc4ae..08bcfc4f 100755 +--- a/CI/pr-gateway.sh ++++ b/CI/pr-gateway.sh +@@ -99,7 +99,7 @@ pushd iSulad + rm -rf build + mkdir build + pushd build +-cmake -DDEBUG=ON -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_UT=ON -DENABLE_CRI_API_V1=ON -DENABLE_REMOTE_LAYER_STORE=ON -DENABLE_SHIM_V2=OFF ../ || exit 1 ++cmake -DDEBUG=ON -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_UT=ON -DENABLE_CRI_API_V1=ON -DENABLE_REMOTE_LAYER_STORE=ON -DENABLE_SHIM_V2=OFF -DENABLE_GRPC_REMOTE_CONNECT=ON ../ || exit 1 + make -j $(nproc) || exit 1 + ctest -V + popd +-- +2.42.0 + diff --git a/0018-Add-compatibility-between-iSulad-and-k8s.patch b/0018-Add-compatibility-between-iSulad-and-k8s.patch new file mode 100644 index 0000000..48d9f0c --- /dev/null +++ b/0018-Add-compatibility-between-iSulad-and-k8s.patch @@ -0,0 +1,57 @@ +From bec48dcd219885abd72cb9973a2e810e3f504269 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Wed, 15 Nov 2023 10:51:01 +0800 +Subject: [PATCH 18/64] Add compatibility between iSulad and k8s + +Signed-off-by: jikai<jikai11@huawei.com> +--- + README.md | 13 ++++++++++++- + README_zh.md | 12 +++++++++++- + 2 files changed, 23 insertions(+), 2 deletions(-) + +diff --git a/README.md b/README.md +index 7e4b6de1..e7949dee 100644 +--- a/README.md ++++ b/README.md +@@ -233,4 +233,15 @@ The standard specification versions that `iSulad` is compatible with are as foll + + - Compatible with OCI 1.0.0. + - Compatible with CNI 0.3.0 and above. +-- Compatible with lcr 2.1.x and above. +\ No newline at end of file ++- Compatible with lcr 2.1.x and above. ++ ++## Kubernetes Support ++ ++`iSulad` supports Kubernetes version 1.13 and above. The following table shows the compatibility between `iSulad` and Kubernetes. ++It lists the minimum `iSulad` version required for some given Kubernetes versions. ++ ++iSulad Version | Kubernetes Version | CRI Version ++--- | --- | --- ++v2.0.0+ | v1.13-v1.18 | v1alpha2 ++v2.0.8+ | v1.19-v1.22 | v1alpha2 ++v2.1.4+ | v1.23-v1.26 | v1, v1alpha2 +diff --git a/README_zh.md b/README_zh.md +index 72942765..1c4dff4f 100755 +--- a/README_zh.md ++++ b/README_zh.md +@@ -229,4 +229,14 @@ $ sudo isula rm test + + - 兼容 1.0.0 版本的OCI + - 兼容 0.3.0 版本以上的CNI +-- 兼容 2.1.x 版本以上的lcr +\ No newline at end of file ++- 兼容 2.1.x 版本以上的lcr ++ ++## Kubernetes Support ++ ++`iSulad`提供对Kubernetes 1.13 版本以上的支持。以下表格显示了 `iSulad` 与 Kubernetes 之间的兼容性。它给出了指定Kubernetes版本下所需要的最低 `iSulad` 版本。 ++ ++iSulad 版本 | Kubernetes 版本 | CRI 版本 ++--- | --- | --- ++v2.0.0+ | v1.13-v1.18 | v1alpha2 ++v2.0.8+ | v1.19-v1.22 | v1alpha2 ++v2.1.4+ | v1.23-v1.26 | v1, v1alpha2 +-- +2.42.0 + diff --git a/0019-2254-lcr-container-with-a-damaged-config-file-will-r.patch b/0019-2254-lcr-container-with-a-damaged-config-file-will-r.patch new file mode 100644 index 0000000..c371d1a --- /dev/null +++ b/0019-2254-lcr-container-with-a-damaged-config-file-will-r.patch @@ -0,0 +1,384 @@ +From 21bca2bb054ed7a1b9b78e01965f8a6d9c3fd28d Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Mon, 20 Nov 2023 12:58:26 +0000 +Subject: [PATCH 19/64] !2254 lcr container with a damaged config file will + rebuild the config during restore * lcr container with a damaged config file + will rebuild the config during restore + +--- + src/common/constants.h | 2 + + src/daemon/modules/api/runtime_api.h | 7 ++ + .../modules/container/restore/restore.c | 28 ++++-- + .../modules/runtime/engines/lcr/lcr_rt_ops.c | 99 ++++++++++++++++++- + .../modules/runtime/engines/lcr/lcr_rt_ops.h | 1 + + .../modules/runtime/isula/isula_rt_ops.c | 6 ++ + .../modules/runtime/isula/isula_rt_ops.h | 1 + + src/daemon/modules/runtime/runtime.c | 24 +++++ + src/daemon/modules/runtime/shim/shim_rt_ops.c | 6 ++ + src/daemon/modules/runtime/shim/shim_rt_ops.h | 2 + + 10 files changed, 167 insertions(+), 9 deletions(-) + +diff --git a/src/common/constants.h b/src/common/constants.h +index caf9b793..5f12ae25 100644 +--- a/src/common/constants.h ++++ b/src/common/constants.h +@@ -86,6 +86,8 @@ extern "C" { + + #define LOG_MAX_RETRIES 10 + ++#define INVALID_CONFIG_ERR_CODE 2 ++ + #define MAX_MSG_BUFFER_SIZE (32 * 1024) + + #define DEFAULT_WEBSOCKET_SERVER_LISTENING_PORT 10350 +diff --git a/src/daemon/modules/api/runtime_api.h b/src/daemon/modules/api/runtime_api.h +index 3c2100f5..08558f42 100644 +--- a/src/daemon/modules/api/runtime_api.h ++++ b/src/daemon/modules/api/runtime_api.h +@@ -41,6 +41,7 @@ typedef enum { + struct runtime_container_status_info { + bool has_pid; + uint32_t pid; ++ int error_code; + Runtime_Container_Status status; + }; + +@@ -197,6 +198,10 @@ typedef struct _rt_exec_resize_params_t { + unsigned int width; + } rt_exec_resize_params_t; + ++typedef struct _rt_runtime_rebuild_config_params_t { ++ const char *rootpath; ++} rt_rebuild_config_params_t; ++ + struct rt_ops { + /* detect whether runtime is of this runtime type */ + bool (*detect)(const char *runtime); +@@ -233,6 +238,7 @@ struct rt_ops { + rt_listpids_out_t *out); + int (*rt_resize)(const char *name, const char *runtime, const rt_resize_params_t *params); + int (*rt_exec_resize)(const char *name, const char *runtime, const rt_exec_resize_params_t *params); ++ int (*rt_rebuild_config)(const char *name, const char *runtime, const rt_rebuild_config_params_t *params); + }; + + int runtime_create(const char *name, const char *runtime, const rt_create_params_t *params); +@@ -253,6 +259,7 @@ int runtime_attach(const char *name, const char *runtime, const rt_attach_params + int runtime_update(const char *name, const char *runtime, const rt_update_params_t *params); + + int runtime_listpids(const char *name, const char *runtime, const rt_listpids_params_t *params, rt_listpids_out_t *out); ++int runtime_rebuild_config(const char *name, const char *runtime, const rt_rebuild_config_params_t *params); + void free_rt_listpids_out_t(rt_listpids_out_t *out); + int runtime_resize(const char *name, const char *runtime, const rt_resize_params_t *params); + int runtime_exec_resize(const char *name, const char *runtime, const rt_exec_resize_params_t *params); +diff --git a/src/daemon/modules/container/restore/restore.c b/src/daemon/modules/container/restore/restore.c +index c26cf561..f6218fe6 100644 +--- a/src/daemon/modules/container/restore/restore.c ++++ b/src/daemon/modules/container/restore/restore.c +@@ -16,15 +16,18 @@ + #include <stdio.h> + #include <unistd.h> + #include <limits.h> +-#include <isula_libutils/container_config_v2.h> +-#include <isula_libutils/host_config.h> + #include <stdbool.h> + #include <stdint.h> + #include <stdlib.h> + #include <string.h> + ++#include <isula_libutils/container_config_v2.h> ++#include <isula_libutils/host_config.h> ++#include <isula_libutils/log.h> ++#include <isula_libutils/auto_cleanup.h> ++ + #include "isulad_config.h" +-#include "isula_libutils/log.h" ++ + #include "container_api.h" + #include "supervisor.h" + #include "containers_gc.h" +@@ -276,9 +279,22 @@ static void restore_state(container_t *cont) + #endif + nret = runtime_status(id, runtime, ¶ms, &real_status); + if (nret != 0) { +- WARN("Failed to restore container %s, make real status to STOPPED. Due to can not load container with status %d", +- id, status); +- real_status.status = RUNTIME_CONTAINER_STATUS_STOPPED; ++ bool rebuild_config = (real_status.error_code == INVALID_CONFIG_ERR_CODE); ++ int tempret = -1; ++ // only the lcr container with a damaged config file will rebuild the config ++ if (rebuild_config) { ++ rt_rebuild_config_params_t rebuild_params = { 0 }; ++ rebuild_params.rootpath = cont->root_path; ++ nret = runtime_rebuild_config(id, runtime, &rebuild_params); ++ EVENT("Rebuild config for container: %s, result : %d", id, nret); ++ if (nret == 0) { ++ tempret = runtime_status(id, runtime, ¶ms, &real_status); ++ } ++ } ++ if (tempret != 0) { ++ WARN("Failed to restore container %s, make real status to STOPPED. Due to cannot load container with status %d", id, status); ++ real_status.status = RUNTIME_CONTAINER_STATUS_STOPPED; ++ } + } + + if (real_status.status == RUNTIME_CONTAINER_STATUS_STOPPED) { +diff --git a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c +index f61316d0..2f42909b 100644 +--- a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c ++++ b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c +@@ -16,15 +16,18 @@ + #include <stdio.h> + #include <limits.h> + #include <errno.h> +-#include <isula_libutils/defs.h> +-#include <isula_libutils/host_config.h> + #include <stdint.h> + #include <stdlib.h> + #include <string.h> + #include <strings.h> + ++#include <isula_libutils/log.h> ++#include <isula_libutils/defs.h> ++#include <isula_libutils/host_config.h> ++#include <isula_libutils/auto_cleanup.h> ++#include <isula_libutils/oci_runtime_spec.h> ++ + #include "lcr_rt_ops.h" +-#include "isula_libutils/log.h" + #include "engine.h" + #include "error.h" + #include "isulad_config.h" +@@ -32,6 +35,8 @@ + #include "runtime_api.h" + #include "utils_file.h" + ++#define LCR_CONFIG_FILE "config" ++ + bool rt_lcr_detect(const char *runtime) + { + /* now we just support lcr engine */ +@@ -276,6 +281,17 @@ int rt_lcr_status(const char *name, const char *runtime, const rt_status_params_ + nret = engine_ops->engine_get_container_status_op(name, params->rootpath, status); + if (nret != 0) { + ret = -1; ++ const char *tmpmsg = NULL; ++ if (engine_ops->engine_get_errmsg_op != NULL) { ++ tmpmsg = engine_ops->engine_get_errmsg_op(); ++ } ++ if (tmpmsg != NULL && strstr(tmpmsg, "Failed to load config") != NULL) { ++ status->error_code = INVALID_CONFIG_ERR_CODE; ++ } ++ isulad_set_error_message("Runtime state container error: %s", ++ (tmpmsg != NULL && strcmp(tmpmsg, DEF_SUCCESS_STR)) != 0 ? tmpmsg : DEF_ERR_RUNTIME_STR); ++ ERROR("Runtime state container error: %s", ++ (tmpmsg != NULL && strcmp(tmpmsg, DEF_SUCCESS_STR)) != 0 ? tmpmsg : DEF_ERR_RUNTIME_STR); + goto out; + } + +@@ -756,3 +772,80 @@ int rt_lcr_kill(const char *id, const char *runtime, const rt_kill_params_t *par + + return 0; + } ++ ++int rt_lcr_rebuild_config(const char *name, const char *runtime, const rt_rebuild_config_params_t *params) ++{ ++ int ret = -1; ++ int nret = 0; ++ char config_file[PATH_MAX] = { 0 }; ++ char bak_config_file[PATH_MAX] = { 0 }; ++ char oci_config_file[PATH_MAX] = { 0 }; ++ struct engine_operation *engine_ops = NULL; ++ oci_runtime_spec *oci_spec = NULL; ++ __isula_auto_free char *json_container = NULL; ++ __isula_auto_free parser_error err = NULL; ++ ++ engine_ops = engines_get_handler(runtime); ++ if (engine_ops == NULL || engine_ops->engine_create_op == NULL) { ++ ERROR("Failed to get engine rebuild config operations"); ++ return -1; ++ } ++ ++ nret = snprintf(config_file, PATH_MAX, "%s/%s/%s", params->rootpath, name, LCR_CONFIG_FILE); ++ if (nret < 0 || (size_t)nret >= PATH_MAX) { ++ ERROR("Failed to snprintf config file for container %s", name); ++ return -1; ++ } ++ ++ nret = snprintf(bak_config_file, PATH_MAX, "%s/%s/%s", params->rootpath, name, ".tmp_config_bak"); ++ if (nret < 0 || (size_t)nret >= PATH_MAX) { ++ ERROR("Failed to snprintf bak config file for container %s", name); ++ return -1; ++ } ++ ++ nret = snprintf(oci_config_file, sizeof(oci_config_file), "%s/%s/%s", params->rootpath, name, OCI_CONFIG_JSON); ++ if (nret < 0 || (size_t)nret >= sizeof(oci_config_file)) { ++ ERROR("Failed to snprintf for config json"); ++ return -1; ++ } ++ ++ oci_spec = oci_runtime_spec_parse_file(oci_config_file, NULL, &err); ++ if (oci_spec == NULL) { ++ ERROR("Failed to parse oci config file:%s", err); ++ return -1; ++ } ++ ++ // delete the bak config file to prevent the remnants of the previous bak file ++ if (util_fileself_exists(bak_config_file) && util_path_remove(bak_config_file) != 0) { ++ ERROR("Failed to remove bak_config_file for container: %s", name); ++ goto out; ++ } ++ ++ if (util_fileself_exists(config_file) && rename(config_file, bak_config_file) != 0) { ++ ERROR("Failed to backup old config for container: %s", name); ++ goto out; ++ } ++ ++ nret = engine_ops->engine_create_op(name, params->rootpath, (void *)oci_spec); ++ if (nret != 0) { ++ // delete the invalid config file to prevent rename failed ++ if (util_fileself_exists(config_file) && util_path_remove(config_file) != 0) { ++ WARN("Failed to remove bak_config_file for container %s", name); ++ } ++ if (util_fileself_exists(bak_config_file) && rename(bak_config_file, config_file) != 0) { ++ WARN("Failed to rename backup old config to config for container %s", name); ++ } ++ } ++ ++ ret = 0; ++ ++out: ++ if (engine_ops != NULL && engine_ops->engine_clear_errmsg_op != NULL) { ++ engine_ops->engine_clear_errmsg_op(); ++ } ++ if (util_fileself_exists(bak_config_file) && util_path_remove(bak_config_file) != 0) { ++ WARN("Failed to remove bak_config_file for %s", name); ++ } ++ free_oci_runtime_spec(oci_spec); ++ return ret; ++} +diff --git a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.h b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.h +index 5b74ad6c..7403544d 100644 +--- a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.h ++++ b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.h +@@ -47,6 +47,7 @@ int rt_lcr_resources_stats(const char *name, const char *runtime, const rt_stats + int rt_lcr_resize(const char *id, const char *runtime, const rt_resize_params_t *params); + int rt_lcr_exec_resize(const char *id, const char *runtime, const rt_exec_resize_params_t *params); + int rt_lcr_kill(const char *id, const char *runtime, const rt_kill_params_t *params); ++int rt_lcr_rebuild_config(const char *name, const char *runtime, const rt_rebuild_config_params_t *params); + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 1787170b..83214c1a 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -2013,3 +2013,9 @@ int rt_isula_kill(const char *id, const char *runtime, const rt_kill_params_t *p + + return 0; + } ++ ++// the config file of oci runtime is config.json. If it is damaged, it cannot be rebuilt. ++int rt_isula_rebuild_config(const char *name, const char *runtime, const rt_rebuild_config_params_t *params) ++{ ++ return 0; ++} +\ No newline at end of file +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.h b/src/daemon/modules/runtime/isula/isula_rt_ops.h +index 49b6cc0e..1e5e049a 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.h ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.h +@@ -46,6 +46,7 @@ int rt_isula_resources_stats(const char *name, const char *runtime, const rt_sta + int rt_isula_resize(const char *id, const char *runtime, const rt_resize_params_t *params); + int rt_isula_exec_resize(const char *id, const char *runtime, const rt_exec_resize_params_t *params); + int rt_isula_kill(const char *id, const char *runtime, const rt_kill_params_t *params); ++int rt_isula_rebuild_config(const char *name, const char *runtime, const rt_rebuild_config_params_t *params); + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/runtime/runtime.c b/src/daemon/modules/runtime/runtime.c +index cb383970..d9a332af 100644 +--- a/src/daemon/modules/runtime/runtime.c ++++ b/src/daemon/modules/runtime/runtime.c +@@ -45,6 +45,7 @@ static const struct rt_ops g_lcr_rt_ops = { + .rt_resize = rt_lcr_resize, + .rt_exec_resize = rt_lcr_exec_resize, + .rt_kill = rt_lcr_kill, ++ .rt_rebuild_config = rt_lcr_rebuild_config, + }; + + static const struct rt_ops g_isula_rt_ops = { +@@ -65,6 +66,7 @@ static const struct rt_ops g_isula_rt_ops = { + .rt_resize = rt_isula_resize, + .rt_exec_resize = rt_isula_exec_resize, + .rt_kill = rt_isula_kill, ++ .rt_rebuild_config = rt_isula_rebuild_config, + }; + + #ifdef ENABLE_SHIM_V2 +@@ -86,6 +88,7 @@ static const struct rt_ops g_shim_rt_ops = { + .rt_resize = rt_shim_resize, + .rt_exec_resize = rt_shim_exec_resize, + .rt_kill = rt_shim_kill, ++ .rt_rebuild_config = rt_shim_rebuild_config, + }; + #endif + +@@ -465,6 +468,27 @@ out: + return ret; + } + ++int runtime_rebuild_config(const char *name, const char *runtime, const rt_rebuild_config_params_t *params) ++{ ++ int ret = 0; ++ const struct rt_ops *ops = NULL; ++ ++ if (name == NULL || runtime == NULL || params == NULL) { ++ ERROR("Invalid arguments for runtime rebuild config"); ++ return -1; ++ } ++ ++ ops = rt_ops_query(runtime); ++ if (ops == NULL) { ++ ERROR("Failed to get runtime ops"); ++ return -1; ++ } ++ ++ ret = ops->rt_rebuild_config(name, runtime, params); ++ ++ return ret; ++} ++ + int runtime_resize(const char *name, const char *runtime, const rt_resize_params_t *params) + { + int ret = 0; +diff --git a/src/daemon/modules/runtime/shim/shim_rt_ops.c b/src/daemon/modules/runtime/shim/shim_rt_ops.c +index 550b17f3..56fc43c2 100644 +--- a/src/daemon/modules/runtime/shim/shim_rt_ops.c ++++ b/src/daemon/modules/runtime/shim/shim_rt_ops.c +@@ -805,3 +805,9 @@ int rt_shim_kill(const char *id, const char *runtime, const rt_kill_params_t *pa + + return 0; + } ++ ++// the config file of oci runtime is config.json. If it is damaged, it cannot be rebuilt. ++int rt_shim_rebuild_config(const char *name, const char *runtime, const rt_rebuild_config_params_t *params) ++{ ++ return 0; ++} +\ No newline at end of file +diff --git a/src/daemon/modules/runtime/shim/shim_rt_ops.h b/src/daemon/modules/runtime/shim/shim_rt_ops.h +index 03b7c018..2df34f4c 100644 +--- a/src/daemon/modules/runtime/shim/shim_rt_ops.h ++++ b/src/daemon/modules/runtime/shim/shim_rt_ops.h +@@ -62,6 +62,8 @@ int rt_shim_exec_resize(const char *id, const char *runtime, const rt_exec_resiz + + bool is_valid_v2_runtime(const char* name); + ++int rt_shim_rebuild_config(const char *name, const char *runtime, const rt_rebuild_config_params_t *params); ++ + #ifdef __cplusplus + } + #endif +-- +2.42.0 + diff --git a/0020-2253-bugfix-for-runc-container-exec.patch b/0020-2253-bugfix-for-runc-container-exec.patch new file mode 100644 index 0000000..bf00f1e --- /dev/null +++ b/0020-2253-bugfix-for-runc-container-exec.patch @@ -0,0 +1,39 @@ +From 6b636051af158fac017998732d7d121b8ea71081 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Mon, 20 Nov 2023 12:59:09 +0000 +Subject: [PATCH 20/64] !2253 bugfix for runc container exec * bugfix for runc + container exec + +--- + src/daemon/modules/runtime/isula/isula_rt_ops.c | 9 ++++----- + 1 file changed, 4 insertions(+), 5 deletions(-) + +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 83214c1a..e61d1f91 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -1386,17 +1386,16 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p + } + + ret = shim_create(fg_exec(params), id, workdir, bundle, cmd, exit_code, timeout, &shim_exit_code); +- if (ret != 0) { +- ERROR("%s: failed create shim process for exec %s", id, exec_id); +- goto errlog_out; +- } +- + if (shim_exit_code == SHIM_EXIT_TIMEOUT) { + ret = -1; + isulad_set_error_message("Exec container error;exec timeout"); + ERROR("isulad-shim %d exit for execing timeout", pid); + goto errlog_out; + } ++ if (ret != 0) { ++ ERROR("%s: failed create shim process for exec %s", id, exec_id); ++ goto errlog_out; ++ } + + pid = get_container_process_pid(workdir); + if (pid < 0) { +-- +2.42.0 + diff --git a/0021-bugfix-of-update-restart-policy-for-auto-remove-cont.patch b/0021-bugfix-of-update-restart-policy-for-auto-remove-cont.patch new file mode 100644 index 0000000..6d0ba03 --- /dev/null +++ b/0021-bugfix-of-update-restart-policy-for-auto-remove-cont.patch @@ -0,0 +1,58 @@ +From ea51fa836464660fcca245e7e36a2b4cdf1e5997 Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu <zhangxiaoyu58@huawei.com> +Date: Tue, 21 Nov 2023 10:23:26 +0800 +Subject: [PATCH 21/64] bugfix of update restart policy for auto remove + container + +Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com> +--- + .../executor/container_cb/execution_extend.c | 18 +++++++++++------- + 1 file changed, 11 insertions(+), 7 deletions(-) + +diff --git a/src/daemon/executor/container_cb/execution_extend.c b/src/daemon/executor/container_cb/execution_extend.c +index 6759a4fc..de017b4e 100644 +--- a/src/daemon/executor/container_cb/execution_extend.c ++++ b/src/daemon/executor/container_cb/execution_extend.c +@@ -1113,15 +1113,14 @@ static int update_host_config_check(container_t *cont, host_config *hostconfig) + + ret = verify_host_config_settings(hostconfig, true); + if (ret != 0) { +- goto out; ++ return -1; + } + + if (container_is_removal_in_progress(cont->state) || container_is_dead(cont->state)) { + ERROR("Container is marked for removal and cannot be \"update\"."); + isulad_set_error_message( + "Cannot update container %s: Container is marked for removal and cannot be \"update\".", id); +- ret = -1; +- goto out; ++ return -1; + } + + if (container_is_running(cont->state) && hostconfig->kernel_memory) { +@@ -1129,12 +1128,17 @@ static int update_host_config_check(container_t *cont, host_config *hostconfig) + isulad_set_error_message("Cannot update container %s: Can not update kernel memory to a running container," + " please stop it first.", + id); +- ret = -1; +- goto out; ++ return -1; + } + +-out: +- return ret; ++ if (cont->hostconfig->auto_remove && hostconfig->restart_policy != NULL && ++ hostconfig->restart_policy->name != NULL && strcmp("no", hostconfig->restart_policy->name) != 0) { ++ ERROR("Cannot update restart policy for the auto remove container %s", id); ++ isulad_set_error_message("Cannot update restart policy for the auto remove container %s", id); ++ return -1; ++ } ++ ++ return 0; + } + + static int do_update_resources(const container_update_request *request, container_t *cont) +-- +2.42.0 + diff --git a/0022-add-update-restart-policy-test.patch b/0022-add-update-restart-policy-test.patch new file mode 100644 index 0000000..1f324ae --- /dev/null +++ b/0022-add-update-restart-policy-test.patch @@ -0,0 +1,57 @@ +From 6815aec33caedaacba3b392ee5a2e5088fdf1faa Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu <zhangxiaoyu58@huawei.com> +Date: Tue, 21 Nov 2023 10:24:26 +0800 +Subject: [PATCH 22/64] add update restart policy test + +Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com> +--- + CI/test_cases/container_cases/update.sh | 26 +++++++++++++++++++++++++ + 1 file changed, 26 insertions(+) + +diff --git a/CI/test_cases/container_cases/update.sh b/CI/test_cases/container_cases/update.sh +index d379acc2..29543e7c 100755 +--- a/CI/test_cases/container_cases/update.sh ++++ b/CI/test_cases/container_cases/update.sh +@@ -138,6 +138,27 @@ function do_test_t() + return $TC_RET_T + } + ++function test_autoremove_restartpolicy() ++{ ++ containername=test_update2 ++ containerid=`isula run -itd --runtime $1 --rm --name $containername busybox` ++ fn_check_eq "$?" "0" "run failed" ++ ++ isula update --restart always $containerid ++ fn_check_ne "$?" "0" "update should fail" ++ ++ isula update --restart nooooooooooo $containerid ++ fn_check_ne "$?" "0" "update should fail" ++ ++ isula update --restart no $containerid ++ fn_check_eq "$?" "0" "update restart policy no failed" ++ ++ isula rm -f $containername ++ fn_check_eq "$?" "0" "rm failed" ++ ++ return $TC_RET_T ++} ++ + function do_test_t1() + { + containername=test_update1 +@@ -173,6 +194,11 @@ do + let "ret=$ret + 1" + fi + ++ test_autoremove_restartpolicy $element ++ if [ $? -ne 0 ];then ++ let "ret=$ret + 1" ++ fi ++ + if [ -f "/sys/fs/cgroup/memory/memory.memsw.usage_in_bytes" ];then + do_test_t1 $element + if [ $? -ne 0 ];then +-- +2.42.0 + diff --git a/0023-2260-bugfix-for-rebuild-config.patch b/0023-2260-bugfix-for-rebuild-config.patch new file mode 100644 index 0000000..e5ce564 --- /dev/null +++ b/0023-2260-bugfix-for-rebuild-config.patch @@ -0,0 +1,73 @@ +From f08072a865fcf6191d65e7c01e11b99049758c57 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Tue, 21 Nov 2023 03:27:31 +0000 +Subject: [PATCH 23/64] !2260 bugfix for rebuild config * bugfix for rebuild + config + +--- + src/daemon/modules/container/restore/restore.c | 1 - + src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c | 5 +---- + src/daemon/modules/runtime/runtime.c | 5 +---- + 3 files changed, 2 insertions(+), 9 deletions(-) + +diff --git a/src/daemon/modules/container/restore/restore.c b/src/daemon/modules/container/restore/restore.c +index f6218fe6..a60b1410 100644 +--- a/src/daemon/modules/container/restore/restore.c ++++ b/src/daemon/modules/container/restore/restore.c +@@ -24,7 +24,6 @@ + #include <isula_libutils/container_config_v2.h> + #include <isula_libutils/host_config.h> + #include <isula_libutils/log.h> +-#include <isula_libutils/auto_cleanup.h> + + #include "isulad_config.h" + +diff --git a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c +index 2f42909b..8f7211d7 100644 +--- a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c ++++ b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c +@@ -782,7 +782,6 @@ int rt_lcr_rebuild_config(const char *name, const char *runtime, const rt_rebuil + char oci_config_file[PATH_MAX] = { 0 }; + struct engine_operation *engine_ops = NULL; + oci_runtime_spec *oci_spec = NULL; +- __isula_auto_free char *json_container = NULL; + __isula_auto_free parser_error err = NULL; + + engine_ops = engines_get_handler(runtime); +@@ -836,9 +835,7 @@ int rt_lcr_rebuild_config(const char *name, const char *runtime, const rt_rebuil + WARN("Failed to rename backup old config to config for container %s", name); + } + } +- +- ret = 0; +- ++ ret = nret != 0 ? -1 : 0; + out: + if (engine_ops != NULL && engine_ops->engine_clear_errmsg_op != NULL) { + engine_ops->engine_clear_errmsg_op(); +diff --git a/src/daemon/modules/runtime/runtime.c b/src/daemon/modules/runtime/runtime.c +index d9a332af..4a239f0a 100644 +--- a/src/daemon/modules/runtime/runtime.c ++++ b/src/daemon/modules/runtime/runtime.c +@@ -470,7 +470,6 @@ out: + + int runtime_rebuild_config(const char *name, const char *runtime, const rt_rebuild_config_params_t *params) + { +- int ret = 0; + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL || params == NULL) { +@@ -484,9 +483,7 @@ int runtime_rebuild_config(const char *name, const char *runtime, const rt_rebui + return -1; + } + +- ret = ops->rt_rebuild_config(name, runtime, params); +- +- return ret; ++ return ops->rt_rebuild_config(name, runtime, params); + } + + int runtime_resize(const char *name, const char *runtime, const rt_resize_params_t *params) +-- +2.42.0 + diff --git a/0024-2170-isula-image-pull.patch b/0024-2170-isula-image-pull.patch new file mode 100644 index 0000000..534c2c5 --- /dev/null +++ b/0024-2170-isula-image-pull.patch @@ -0,0 +1,1815 @@ +From 53d551f613bfa8ce0552ca62f964a0584e3665bb Mon Sep 17 00:00:00 2001 +From: sailorvii <chenw66@chinaunicom.cn> +Date: Wed, 22 Nov 2023 01:22:04 +0000 +Subject: [PATCH 24/64] =?UTF-8?q?!2170=20=E5=A2=9E=E5=8A=A0isula=20image?= + =?UTF-8?q?=20pull=E8=BF=9B=E5=BA=A6=E6=98=BE=E7=A4=BA=20*=20Refine=20some?= + =?UTF-8?q?=20issues.=20*=20Address=20comment=20*=20Address=20comments=20*?= + =?UTF-8?q?=201.=20Address=20comments.=20*=20Address=20comments=20*=20Addr?= + =?UTF-8?q?ess=20comments=20*=20Address=20comments=20*=20Address=20comment?= + =?UTF-8?q?s=20*=20Address=20comments=20*=20Address=20comments=20*=20Addre?= + =?UTF-8?q?ss=20comments=20*=20Address=20comments=20*=20Address=20comments?= + =?UTF-8?q?=20*=20Address=20test=20issue=20*=20Address=20test=20compile=20?= + =?UTF-8?q?issue=20*=20Address=20compile=20issue=20*=20Fix=20compile=20iss?= + =?UTF-8?q?ue=20*=20Address=20comments=20*=20Address=20comments=20*=20Addr?= + =?UTF-8?q?ess=20comments.=20*=20Address=20issuse=20*=20Address=20many=20i?= + =?UTF-8?q?ssues.=20*=20Fix=20some=20minor=20issuses.=20*=20Address=20comm?= + =?UTF-8?q?ents.=20*=20Refine=20as=20Haozi's=20comments=20*=20Fix=20some?= + =?UTF-8?q?=20issues=20by=20Haozi's=20comments.=20*=20Refine=20formats.=20?= + =?UTF-8?q?*=20Add=20process=20bar=20show=20for=20pull=20functions.?= +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +--- + CI/dockerfiles/Dockerfile-fedora | 2 +- + CI/pr-gateway.sh | 2 +- + cmake/checker.cmake | 10 ++ + src/CMakeLists.txt | 4 + + src/api/services/images/images.proto | 35 +++- + src/client/connect/grpc/grpc_images_client.cc | 167 ++++++++++++++++-- + src/client/connect/protocol_type.h | 1 + + src/cmd/isula/images/pull.c | 26 ++- + .../connect/grpc/grpc_containers_service.h | 4 + + .../entry/connect/grpc/grpc_images_service.cc | 105 ++++++++++- + .../entry/connect/grpc/grpc_images_service.h | 7 + + .../entry/connect/rest/rest_images_service.c | 2 +- + .../v1/v1_cri_image_manager_service_impl.cc | 2 +- + .../v1alpha/cri_image_manager_service_impl.cc | 2 +- + src/daemon/executor/callback.h | 3 +- + src/daemon/executor/image_cb/image_cb.c | 8 +- + src/daemon/modules/api/image_api.h | 5 +- + src/daemon/modules/image/image.c | 9 +- + src/daemon/modules/image/oci/oci_image.c | 4 +- + src/daemon/modules/image/oci/oci_image.h | 2 +- + src/daemon/modules/image/oci/oci_pull.c | 158 +++++++++++++++-- + src/daemon/modules/image/oci/oci_pull.h | 2 +- + src/daemon/modules/image/oci/progress.c | 124 +++++++++++++ + src/daemon/modules/image/oci/progress.h | 52 ++++++ + .../modules/image/oci/registry/http_request.c | 104 ++++++++--- + .../modules/image/oci/registry/http_request.h | 2 +- + .../modules/image/oci/registry/registry.c | 3 +- + .../modules/image/oci/registry/registry.h | 2 + + .../image/oci/registry/registry_apiv2.c | 12 +- + src/daemon/modules/image/oci/registry_type.h | 3 + + src/utils/CMakeLists.txt | 3 + + src/utils/http/http.h | 17 +- + src/utils/progress/CMakeLists.txt | 13 ++ + src/utils/progress/show.c | 64 +++++++ + src/utils/progress/show.h | 34 ++++ + test/cutils/CMakeLists.txt | 1 + + test/image/oci/registry/CMakeLists.txt | 1 + + test/image/oci/registry/registry_ut.cc | 16 +- + 38 files changed, 912 insertions(+), 99 deletions(-) + create mode 100644 src/daemon/modules/image/oci/progress.c + create mode 100644 src/daemon/modules/image/oci/progress.h + create mode 100644 src/utils/progress/CMakeLists.txt + create mode 100644 src/utils/progress/show.c + create mode 100644 src/utils/progress/show.h + +diff --git a/CI/dockerfiles/Dockerfile-fedora b/CI/dockerfiles/Dockerfile-fedora +index bef44377..a105cbb4 100644 +--- a/CI/dockerfiles/Dockerfile-fedora ++++ b/CI/dockerfiles/Dockerfile-fedora +@@ -115,7 +115,7 @@ RUN echo "[source.crates-io]" >> ${HOME}/.cargo/config && \ + echo "[source.local-registry]" >> ${HOME}/.cargo/config && \ + echo "directory = \"vendor\"" >> ${HOME}/.cargo/config + +-RUN dnf install -y lcov && dnf clean all ++RUN dnf install -y lcov ncurses-devel && dnf clean all + + # install libevhtp + RUN export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH && \ +diff --git a/CI/pr-gateway.sh b/CI/pr-gateway.sh +index 08bcfc4f..e5bf627e 100755 +--- a/CI/pr-gateway.sh ++++ b/CI/pr-gateway.sh +@@ -22,7 +22,7 @@ sed -i "s#http://repo.openeuler.org#https://repo.huaweicloud.com/openeuler#g" /e + + dnf update -y + +-dnf install -y docbook2X doxygen gtest-devel gmock-devel diffutils cmake gcc-c++ yajl-devel patch make libtool libevent-devel libevhtp-devel grpc grpc-plugins grpc-devel protobuf-devel libcurl libcurl-devel sqlite-devel libarchive-devel device-mapper-devel http-parser-devel libseccomp-devel libcap-devel libselinux-devel libwebsockets libwebsockets-devel systemd-devel git chrpath ++dnf install -y docbook2X doxygen gtest-devel gmock-devel diffutils cmake gcc-c++ yajl-devel patch make libtool libevent-devel libevhtp-devel grpc grpc-plugins grpc-devel protobuf-devel libcurl libcurl-devel sqlite-devel libarchive-devel device-mapper-devel http-parser-devel libseccomp-devel libcap-devel libselinux-devel libwebsockets libwebsockets-devel systemd-devel git chrpath ncurses-devel + if [ $? -ne 0 ]; then + echo "install dependences failed" + exit 1 +diff --git a/cmake/checker.cmake b/cmake/checker.cmake +index 358ab4af..cc4a1fc3 100644 +--- a/cmake/checker.cmake ++++ b/cmake/checker.cmake +@@ -154,6 +154,16 @@ if (GRPC_CONNECTOR) + _CHECK(WEBSOCKET_INCLUDE_DIR "WEBSOCKET_INCLUDE_DIR-NOTFOUND" libwebsockets.h) + find_library(WEBSOCKET_LIBRARY websockets) + _CHECK(WEBSOCKET_LIBRARY "WEBSOCKET_LIBRARY-NOTFOUND" "libwebsockets.so") ++ ++ # check libncurses ++ pkg_check_modules(PC_LIBNCURSES REQUIRED "ncurses") ++ find_path(NCURSES_INCLUDE_DIR curses.h ++ HINTS ${PC_NCURSES_INCLUDEDIR} ${PC_NCURSES_INCLUDE_DIRS}) ++ _CHECK(NCURSES_INCLUDE_DIR "NCURSES_INCLUDE_DIR-NOTFOUND" "curses.h") ++ ++ find_library(NCURSES_LIBRARY ncurses ++ HINTS ${PC_NCURSES_LIBDIR} ${PC_NCURSES_LIBRARY_DIRS}) ++ _CHECK(NCURSES_LIBRARY "NCURSES_LIBRARY-NOTFOUND" "libncurses.so") + endif() + + if ((NOT GRPC_CONNECTOR) OR (GRPC_CONNECTOR AND ENABLE_METRICS)) +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index 8e197b9f..860447de 100644 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -102,6 +102,10 @@ add_executable(isula + ) + target_include_directories(isula PUBLIC ${ISULA_INCS} ${SHARED_INCS}) + target_link_libraries(isula libisula_client ${LIBYAJL_LIBRARY}) ++if (GRPC_CONNECTOR) ++ target_link_libraries(isula ${NCURSES_LIBRARY}) ++endif() ++ + if (ANDROID OR MUSL) + target_link_libraries(isula ${LIBSSL_LIBRARY}) + else() +diff --git a/src/api/services/images/images.proto b/src/api/services/images/images.proto +index 2a34f02d..9f2cb803 100644 +--- a/src/api/services/images/images.proto ++++ b/src/api/services/images/images.proto +@@ -30,6 +30,23 @@ service ImagesService { + rpc Tag(TagImageRequest) returns (TagImageResponse); + rpc Import(ImportRequest) returns (ImportResponse); + rpc Search(SearchRequest) returns (SearchResponse); ++ rpc PullImage(PullImageRequest) returns (stream PullImageResponse); ++} ++ ++// ImageSpec is an internal representation of an image. ++message ImageSpec { ++ // Container's Image field (e.g. imageID or imageDigest). ++ string image = 1; ++ // Unstructured key-value map holding arbitrary metadata. ++ // ImageSpec Annotations can be used to help the runtime target specific ++ // images in multi-arch images. ++ map<string, string> annotations = 2; ++} ++ ++// AuthConfig contains authorization information for connecting to a registry. ++message AuthConfig { ++ string username = 1; ++ string password = 2; + } + + message Descriptor { +@@ -152,4 +169,20 @@ message SearchResponse { + repeated SearchImage search_result = 2; + uint32 cc = 3; + string errmsg = 4; +-} +\ No newline at end of file ++} ++ ++message PullImageRequest { ++ // Spec of the image. ++ ImageSpec image = 1; ++ // Authentication configuration for pulling the image. ++ AuthConfig auth = 2; ++ bool is_progress_visible = 3; ++} ++ ++message PullImageResponse { ++ // Reference to the image in use. For most runtimes, this should be an ++ // image ID or digest. ++ string image_ref = 1; ++ string stream = 2; ++ bytes progress_data = 3; ++} +diff --git a/src/client/connect/grpc/grpc_images_client.cc b/src/client/connect/grpc/grpc_images_client.cc +index 9cc2a174..7a283e8c 100644 +--- a/src/client/connect/grpc/grpc_images_client.cc ++++ b/src/client/connect/grpc/grpc_images_client.cc +@@ -13,12 +13,15 @@ + * Description: provide grpc container service functions + ******************************************************************************/ + #include "grpc_images_client.h" +-#include "api.grpc.pb.h" + #include "client_base.h" + #include "images.grpc.pb.h" ++ ++#include <isula_libutils/auto_cleanup.h> ++#include <isula_libutils/image_progress.h> ++#include <string> ++#include "show.h" + #include "utils.h" + #include "constants.h" +-#include <string> + + using namespace images; + +@@ -337,9 +340,9 @@ public: + } + }; + +-class ImagesPull : public ClientBase<runtime::v1alpha2::ImageService, runtime::v1alpha2::ImageService::Stub, +- isula_pull_request, runtime::v1alpha2::PullImageRequest, isula_pull_response, +- runtime::v1alpha2::PullImageResponse> { ++class ImagesPull : public ClientBase<ImagesService, ImagesService::Stub, ++ isula_pull_request, PullImageRequest, ++ isula_pull_response, PullImageResponse> { + public: + explicit ImagesPull(void *args) + : ClientBase(args) +@@ -347,15 +350,14 @@ public: + } + ~ImagesPull() = default; + +- auto request_to_grpc(const isula_pull_request *request, runtime::v1alpha2::PullImageRequest *grequest) ++ auto request_to_grpc(const isula_pull_request *request, PullImageRequest *grequest) + -> int override + { + if (request == nullptr) { + return -1; + } +- + if (request->image_name != nullptr) { +- auto *image_spec = new (std::nothrow) runtime::v1alpha2::ImageSpec; ++ auto *image_spec = new (std::nothrow) ImageSpec; + if (image_spec == nullptr) { + return -1; + } +@@ -363,10 +365,12 @@ public: + grequest->set_allocated_image(image_spec); + } + ++ grequest->set_is_progress_visible(request->is_progress_visible); ++ + return 0; + } + +- auto response_from_grpc(runtime::v1alpha2::PullImageResponse *gresponse, isula_pull_response *response) ++ auto response_from_grpc(PullImageResponse *gresponse, isula_pull_response *response) + -> int override + { + if (!gresponse->image_ref().empty()) { +@@ -376,7 +380,7 @@ public: + return 0; + } + +- auto check_parameter(const runtime::v1alpha2::PullImageRequest &req) -> int override ++ auto check_parameter(const PullImageRequest &req) -> int override + { + if (req.image().image().empty()) { + ERROR("Missing image name in the request"); +@@ -386,10 +390,147 @@ public: + return 0; + } + +- auto grpc_call(ClientContext *context, const runtime::v1alpha2::PullImageRequest &req, +- runtime::v1alpha2::PullImageResponse *reply) -> Status override ++ auto run(const struct isula_pull_request *request, struct isula_pull_response *response) -> int override ++ { ++ ClientContext context; ++ PullImageRequest grequest; ++ ++#ifdef ENABLE_GRPC_REMOTE_CONNECT ++#ifdef OPENSSL_VERIFY ++ // Set common name from cert.perm ++ char common_name_value[ClientBaseConstants::COMMON_NAME_LEN] = { 0 }; ++ int ret = get_common_name_from_tls_cert(m_certFile.c_str(), common_name_value, ++ ClientBaseConstants::COMMON_NAME_LEN); ++ if (ret != 0) { ++ ERROR("Failed to get common name in: %s", m_certFile.c_str()); ++ return -1; ++ } ++ context.AddMetadata("username", std::string(common_name_value, strlen(common_name_value))); ++ context.AddMetadata("tls_mode", m_tlsMode); ++#endif ++#endif ++ if (request_to_grpc(request, &grequest) != 0) { ++ ERROR("Failed to transform pull request to grpc"); ++ response->server_errono = ISULAD_ERR_INPUT; ++ return -1; ++ } ++ ++ auto reader = stub_->PullImage(&context, grequest); ++ ++ PullImageResponse gresponse; ++ if (grequest.is_progress_visible()) { ++ while (reader->Read(&gresponse)) { ++ output_progress(gresponse); ++ } ++ } else { ++ reader->Read(&gresponse); ++ WARN("The terminal may not support ANSI Escape code. Display is skipped"); ++ } ++ Status status = reader->Finish(); ++ if (!status.ok()) { ++ ERROR("Error code: %d: %s", status.error_code(), status.error_message().c_str()); ++ unpackStatus(status, response); ++ return -1; ++ } ++ response->image_ref = util_strdup_s(gresponse.image_ref().c_str()); ++ return 0; ++ } ++ ++private: ++ void output_progress(PullImageResponse &gresponse) + { +- return stub_->PullImage(context, req, reply); ++ __isula_auto_free char *err = nullptr; ++ struct parser_context ctx = { OPT_GEN_SIMPLIFY, 0 }; ++ ++ image_progress *progresses = image_progress_parse_data(gresponse.progress_data().c_str(), &ctx, &err); ++ if (progresses == nullptr) { ++ ERROR("Parse image progress error %s", err); ++ return; ++ } ++ show_processes(progresses); ++ } ++ ++ void get_printed_value(int64_t value, char *printed) ++ { ++ float float_value = 0.0; ++ const float GB = 1024 * 1024 * 1024; ++ const float MB = 1024 * 1024; ++ const float KB = 1024; ++ ++ if ((float)value / GB > 1) { ++ float_value = (float)value / GB; ++ sprintf(printed, "%.2fGB", float_value); ++ } else if ((float)value / MB > 1) { ++ float_value = (float)value / MB; ++ sprintf(printed, "%.2fMB", float_value); ++ } else if ((float)value / KB > 1) { ++ float_value = (float)value / KB; ++ sprintf(printed, "%.2fKB", float_value); ++ } else { ++ sprintf(printed, "%ldB", value); ++ } ++ } ++ ++ void display_progress_bar(image_progress_progresses_element *progress_item, int width, bool if_show_all) ++ { ++ float progress = 0.0; ++ int filled_width = 0; ++ const int FLOAT_STRING_SIZE = 64; ++ char total[FLOAT_STRING_SIZE] = {0}; ++ char current[FLOAT_STRING_SIZE] = {0}; ++ int empty_width = 0; ++ ++ if (progress_item->total != 0) { ++ progress = (float)progress_item->current / (float)progress_item->total; ++ } ++ filled_width = (int)(progress * width); ++ empty_width = width - filled_width; ++ get_printed_value(progress_item->total, total); ++ get_printed_value(progress_item->current, current); ++ ++ if (if_show_all) { ++ int i = 0; ++ ++ printf("%s: [", progress_item->id); ++ ++ // Print filled characters ++ for (i = 0; i < filled_width; i++) { ++ printf("="); ++ } ++ printf(">"); ++ // Print empty characters ++ for (i = 0; i < empty_width; i++) { ++ printf(" "); ++ } ++ ++ printf("] %s/%s", current, total); ++ } else { ++ printf("%s: %s/%s", progress_item->id, current, total); ++ } ++ printf("\n"); ++ fflush(stdout); ++ } ++ ++ void show_processes(image_progress *progresses) ++ { ++ size_t i = 0; ++ static size_t len = 0; ++ const int TERMINAL_SHOW_WIDTH = 110; ++ const int width = 50; // Width of the progress bars ++ ++ if (len != 0) { ++ move_cursor_up(len); ++ } ++ clear_lines_below(); ++ len = progresses->progresses_len; ++ int terminal_width = get_terminal_width(); ++ bool if_show_all = true; ++ if (terminal_width < TERMINAL_SHOW_WIDTH) { ++ if_show_all = false; ++ } ++ for (i = 0; i < len; i++) { ++ display_progress_bar(progresses->progresses[i], width, if_show_all); ++ } + } + }; + +diff --git a/src/client/connect/protocol_type.h b/src/client/connect/protocol_type.h +index 4206c50b..2b445c5a 100644 +--- a/src/client/connect/protocol_type.h ++++ b/src/client/connect/protocol_type.h +@@ -479,6 +479,7 @@ struct isula_rmi_response { + + struct isula_pull_request { + char *image_name; ++ bool is_progress_visible; + }; + + struct isula_tag_request { +diff --git a/src/cmd/isula/images/pull.c b/src/cmd/isula/images/pull.c +index 548e8d90..9d420778 100644 +--- a/src/cmd/isula/images/pull.c ++++ b/src/cmd/isula/images/pull.c +@@ -14,6 +14,10 @@ + ********************************************************************************/ + #include "pull.h" + ++#ifdef GRPC_CONNECTOR ++#include <curses.h> ++#include <term.h> ++#endif + #include <stdio.h> + #include <stdlib.h> + +@@ -29,6 +33,25 @@ const char g_cmd_pull_usage[] = "pull [OPTIONS] NAME[:TAG]"; + + struct client_arguments g_cmd_pull_args = {}; + ++static bool is_terminal_show_supported() ++{ ++#ifdef GRPC_CONNECTOR ++ // Initialize the terminfo database ++ setupterm(NULL, STDOUT_FILENO, (int *)0); ++ ++ // Query the database for the capability to move the cursor ++ char *cursor_movement = tgetstr("cm", NULL); ++ ++ if (cursor_movement != NULL) { ++ return true; ++ } else { ++ return false; ++ } ++#else ++ return false; ++#endif ++} ++ + /* + * Pull an image or a repository from a registry + */ +@@ -47,6 +70,7 @@ int client_pull(const struct client_arguments *args) + } + + request.image_name = args->image_name; ++ request.is_progress_visible = is_terminal_show_supported(); + + ops = get_connect_client_ops(); + if (ops == NULL || ops->image.pull == NULL) { +@@ -63,8 +87,8 @@ int client_pull(const struct client_arguments *args) + ret = ESERVERERROR; + goto out; + } +- COMMAND_ERROR("Image \"%s\" pulled", response->image_ref); + ++ COMMAND_ERROR("Image \"%s\" pulled", response->image_ref); + out: + isula_pull_response_free(response); + return ret; +diff --git a/src/daemon/entry/connect/grpc/grpc_containers_service.h b/src/daemon/entry/connect/grpc/grpc_containers_service.h +index 92428fbe..4a6c584b 100644 +--- a/src/daemon/entry/connect/grpc/grpc_containers_service.h ++++ b/src/daemon/entry/connect/grpc/grpc_containers_service.h +@@ -37,6 +37,10 @@ using google::protobuf::Timestamp; + void protobuf_timestamp_to_grpc(types_timestamp_t *timestamp, Timestamp *gtimestamp); + void protobuf_timestamp_from_grpc(types_timestamp_t *timestamp, const Timestamp >imestamp); + ++bool grpc_is_call_cancelled(void *context); ++bool grpc_add_initial_metadata(void *context, const char *header, const char *val); ++bool grpc_event_write_function(void *writer, void *data); ++ + // Implement of containers service + class ContainerServiceImpl final : public ContainerService::Service { + public: +diff --git a/src/daemon/entry/connect/grpc/grpc_images_service.cc b/src/daemon/entry/connect/grpc/grpc_images_service.cc +index 5d3fac6b..406f81a9 100644 +--- a/src/daemon/entry/connect/grpc/grpc_images_service.cc ++++ b/src/daemon/entry/connect/grpc/grpc_images_service.cc +@@ -21,9 +21,12 @@ + #include <new> + #include <string> + +-#include "isula_libutils/log.h" ++#include <isula_libutils/auto_cleanup.h> ++#include <isula_libutils/image_progress.h> ++#include <isula_libutils/log.h> + #include "utils.h" + #include "grpc_server_tls_auth.h" ++#include "grpc_containers_service.h" + + int ImagesServiceImpl::image_list_request_from_grpc(const ListImagesRequest *grequest, + image_list_images_request **request) +@@ -596,6 +599,104 @@ Status ImagesServiceImpl::Logout(ServerContext *context, const LogoutRequest *re + return Status::OK; + } + ++int ImagesServiceImpl::image_pull_request_from_grpc(const PullImageRequest *grequest, ++ image_pull_image_request **request) ++{ ++ auto *tmpreq = (image_pull_image_request *)util_common_calloc_s(sizeof(image_pull_image_request)); ++ if (tmpreq == nullptr) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ ++ if (!grequest->image().image().empty()) { ++ tmpreq->image_name = util_strdup_s(grequest->image().image().c_str()); ++ } ++ tmpreq->is_progress_visible = grequest->is_progress_visible(); ++ *request = tmpreq; ++ ++ return 0; ++} ++ ++void image_pull_progress_to_grpc(const image_progress *progress, ++ PullImageResponse &gresponse) ++{ ++ if (progress == nullptr) { ++ ERROR("Invalid parameter"); ++ return; ++ } ++ ++ gresponse.Clear(); ++ __isula_auto_free char *err = nullptr; ++ struct parser_context ctx = { OPT_GEN_SIMPLIFY, 0 }; ++ char *data = image_progress_generate_json(progress, &ctx, &err); ++ if (data == nullptr) { ++ ERROR("Failed to generate image progress json: %s", err); ++ return; ++ } ++ ++ gresponse.set_progress_data(data, strlen(data)); ++ if (progress->image != nullptr) { ++ gresponse.set_image_ref(progress->image); ++ } ++ free(data); ++} ++ ++bool grpc_pull_write_function(void *writer, void *data) ++{ ++ auto *progress = static_cast<image_progress *>(data); ++ auto *gwriter = static_cast<ServerWriter<PullImageResponse> *>(writer); ++ PullImageResponse gresponse; ++ ++ image_pull_progress_to_grpc(progress, gresponse); ++ ++ return gwriter->Write(gresponse); ++} ++ ++Status ImagesServiceImpl::PullImage(ServerContext *context, const PullImageRequest *request, ++ ServerWriter<PullImageResponse> *writer) ++{ ++ prctl(PR_SET_NAME, "RegistryPull"); ++ ++ int ret = 0; ++ std::string errmsg = "Failed to execute image pull"; ++ stream_func_wrapper stream = { 0 }; ++ image_pull_image_request *image_req = nullptr; ++ image_pull_image_response *image_res = nullptr; ++ ++ if (context == nullptr || request == nullptr || writer == nullptr) { ++ return Status(StatusCode::INVALID_ARGUMENT, "Invalid argument"); ++ } ++ ++ auto status = GrpcServerTlsAuth::auth(context, "pull"); ++ if (!status.ok()) { ++ return status; ++ } ++ ++ service_executor_t *cb = get_service_executor(); ++ if (cb == nullptr || cb->image.pull == nullptr) { ++ return Status(StatusCode::UNIMPLEMENTED, "Unimplemented callback"); ++ } ++ ++ ret = image_pull_request_from_grpc(request, &image_req); ++ if (ret != 0) { ++ ERROR("Failed to transform grpc request"); ++ return Status(StatusCode::UNKNOWN, "Failed to transform grpc request"); ++ } ++ ++ stream.context = (void *)context; ++ stream.is_cancelled = &grpc_is_call_cancelled; ++ stream.write_func = &grpc_pull_write_function; ++ stream.writer = (void *)writer; ++ ++ ret = cb->image.pull(image_req, &stream, &image_res); ++ free_image_pull_image_request(image_req); ++ free_image_pull_image_response(image_res); ++ if (ret == 0) { ++ return Status::OK; ++ } ++ return Status(StatusCode::UNKNOWN, errmsg); ++} ++ + #ifdef ENABLE_IMAGE_SEARCH + int ImagesServiceImpl::search_request_from_grpc(const SearchRequest *grequest, image_search_images_request **request) + { +@@ -723,4 +824,4 @@ Status ImagesServiceImpl::Search(ServerContext *context, const SearchRequest *re + + return Status::OK; + } +-#endif +\ No newline at end of file ++#endif +diff --git a/src/daemon/entry/connect/grpc/grpc_images_service.h b/src/daemon/entry/connect/grpc/grpc_images_service.h +index b75075ba..9690f544 100644 +--- a/src/daemon/entry/connect/grpc/grpc_images_service.h ++++ b/src/daemon/entry/connect/grpc/grpc_images_service.h +@@ -58,6 +58,9 @@ public: + + Status Logout(ServerContext *context, const LogoutRequest *request, LogoutResponse *reply) override; + ++ Status PullImage(ServerContext *context, const PullImageRequest *request, ++ ServerWriter<PullImageResponse> *writer) override; ++ + #ifdef ENABLE_IMAGE_SEARCH + Status Search(ServerContext *context, const SearchRequest *request, SearchResponse *reply) override; + #endif +@@ -99,6 +102,10 @@ private: + + int image_logout_request_from_grpc(const LogoutRequest *grequest, image_logout_request **request); + ++ int image_pull_request_from_grpc(const PullImageRequest *grequest, image_pull_image_request **request); ++ ++ void image_pull_response_to_grpc(const image_pull_image_response *response, PullImageResponse *gresponse); ++ + #ifdef ENABLE_IMAGE_SEARCH + int search_request_from_grpc(const SearchRequest *grequest, image_search_images_request **request); + +diff --git a/src/daemon/entry/connect/rest/rest_images_service.c b/src/daemon/entry/connect/rest/rest_images_service.c +index 5a719f83..220de399 100644 +--- a/src/daemon/entry/connect/rest/rest_images_service.c ++++ b/src/daemon/entry/connect/rest/rest_images_service.c +@@ -513,7 +513,7 @@ static void rest_image_pull_cb(evhtp_request_t *req, void *arg) + goto out; + } + +- (void)cb->image.pull(crequest, &cresponse); ++ (void)cb->image.pull(crequest, NULL, &cresponse); + + evhtp_send_image_pull_repsponse(req, cresponse, RESTFUL_RES_OK); + +diff --git a/src/daemon/entry/cri/v1/v1_cri_image_manager_service_impl.cc b/src/daemon/entry/cri/v1/v1_cri_image_manager_service_impl.cc +index b74834fb..b9cbf24c 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_image_manager_service_impl.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_image_manager_service_impl.cc +@@ -265,7 +265,7 @@ auto ImageManagerServiceImpl::PullImage(const runtime::v1::ImageSpec &image, + } + request->type = util_strdup_s(IMAGE_TYPE_OCI); + +- ret = im_pull_image(request, &response); ++ ret = im_pull_image(request, nullptr, &response); + if (ret != 0) { + if (response != nullptr && response->errmsg != nullptr) { + error.SetError(response->errmsg); +diff --git a/src/daemon/entry/cri/v1alpha/cri_image_manager_service_impl.cc b/src/daemon/entry/cri/v1alpha/cri_image_manager_service_impl.cc +index 3ff79ffc..0b36f007 100644 +--- a/src/daemon/entry/cri/v1alpha/cri_image_manager_service_impl.cc ++++ b/src/daemon/entry/cri/v1alpha/cri_image_manager_service_impl.cc +@@ -265,7 +265,7 @@ auto ImageManagerServiceImpl::PullImage(const runtime::v1alpha2::ImageSpec &imag + } + request->type = util_strdup_s(IMAGE_TYPE_OCI); + +- ret = im_pull_image(request, &response); ++ ret = im_pull_image(request, NULL, &response); + if (ret != 0) { + if (response != nullptr && response->errmsg != nullptr) { + error.SetError(response->errmsg); +diff --git a/src/daemon/executor/callback.h b/src/daemon/executor/callback.h +index c48253a1..b32c6b27 100644 +--- a/src/daemon/executor/callback.h ++++ b/src/daemon/executor/callback.h +@@ -285,7 +285,8 @@ typedef struct { + int (*logout)(const image_logout_request *request, image_logout_response **response); + + int (*tag)(const image_tag_image_request *request, image_tag_image_response **response); +- int (*pull)(const image_pull_image_request *request, image_pull_image_response **response); ++ ++ int (*pull)(const image_pull_image_request *request, stream_func_wrapper *stream, image_pull_image_response **response); + #ifdef ENABLE_IMAGE_SEARCH + int (*search)(const image_search_images_request *request, image_search_images_response **response); + #endif +diff --git a/src/daemon/executor/image_cb/image_cb.c b/src/daemon/executor/image_cb/image_cb.c +index 61fa29db..317cb0a8 100644 +--- a/src/daemon/executor/image_cb/image_cb.c ++++ b/src/daemon/executor/image_cb/image_cb.c +@@ -955,12 +955,14 @@ int pull_request_from_rest(const image_pull_image_request *request, im_pull_requ + } + + (*im_req)->image = util_strdup_s(request->image_name); ++ (*im_req)->is_progress_visible = request->is_progress_visible; + + return 0; + } + + /* image pull cb */ +-static int image_pull_cb(const image_pull_image_request *request, image_pull_image_response **response) ++static int image_pull_cb(const image_pull_image_request *request, stream_func_wrapper *stream, ++ image_pull_image_response **response) + { + int ret = -1; + im_pull_request *im_req = NULL; +@@ -988,7 +990,7 @@ static int image_pull_cb(const image_pull_image_request *request, image_pull_ima + + // current only oci image support pull + im_req->type = util_strdup_s(IMAGE_TYPE_OCI); +- ret = im_pull_image(im_req, &im_rsp); ++ ret = im_pull_image(im_req, stream, &im_rsp); + if (ret != 0) { + cc = ISULAD_ERR_EXEC; + goto out; +@@ -1203,4 +1205,4 @@ void image_callback_init(service_image_callback_t *cb) + #ifdef ENABLE_IMAGE_SEARCH + cb->search = image_search_cb; + #endif +-} +\ No newline at end of file ++} +diff --git a/src/daemon/modules/api/image_api.h b/src/daemon/modules/api/image_api.h +index 2f2c00a2..bbe89ad7 100644 +--- a/src/daemon/modules/api/image_api.h ++++ b/src/daemon/modules/api/image_api.h +@@ -32,6 +32,7 @@ + #ifdef ENABLE_IMAGE_SEARCH + #include "isula_libutils/imagetool_search_result.h" + #endif ++#include "stream_wrapper.h" + + #ifdef __cplusplus + extern "C" { +@@ -150,6 +151,8 @@ typedef struct { + char *server_address; + char *identity_token; + char *registry_token; ++ ++ bool is_progress_visible; + } im_pull_request; + + typedef struct { +@@ -304,7 +307,7 @@ void free_im_load_request(im_load_request *ptr); + + void free_im_load_response(im_load_response *ptr); + +-int im_pull_image(const im_pull_request *request, im_pull_response **response); ++int im_pull_image(const im_pull_request *request, stream_func_wrapper *stream, im_pull_response **response); + + void free_im_pull_request(im_pull_request *req); + +diff --git a/src/daemon/modules/image/image.c b/src/daemon/modules/image/image.c +index a14f2ac3..8d7e2c1a 100644 +--- a/src/daemon/modules/image/image.c ++++ b/src/daemon/modules/image/image.c +@@ -86,7 +86,7 @@ struct bim_ops { + int (*load_image)(const im_load_request *request); + + /* pull image */ +- int (*pull_image)(const im_pull_request *request, im_pull_response *response); ++ int (*pull_image)(const im_pull_request *request, stream_func_wrapper *stream, im_pull_response *response); + + /* login */ + int (*login)(const im_login_request *request); +@@ -999,7 +999,7 @@ static bool check_im_pull_args(const im_pull_request *req, im_pull_response * co + return true; + } + +-int im_pull_image(const im_pull_request *request, im_pull_response **response) ++int im_pull_image(const im_pull_request *request, stream_func_wrapper *stream, im_pull_response **response) + { + int ret = -1; + struct bim *bim = NULL; +@@ -1029,7 +1029,7 @@ int im_pull_image(const im_pull_request *request, im_pull_response **response) + } + + EVENT("Event: {Object: %s, Type: Pulling}", request->image); +- ret = bim->ops->pull_image(request, tmp_res); ++ ret = bim->ops->pull_image(request, stream, tmp_res); + if (ret != 0) { + ERROR("Pull image %s failed", request->image); + ret = -1; +@@ -1044,6 +1044,7 @@ out: + } + DAEMON_CLEAR_ERRMSG(); + *response = tmp_res; ++ + return ret; + } + +@@ -2395,4 +2396,4 @@ out: + } + return ret; + } +-#endif +\ No newline at end of file ++#endif +diff --git a/src/daemon/modules/image/oci/oci_image.c b/src/daemon/modules/image/oci/oci_image.c +index f712a446..471510e7 100644 +--- a/src/daemon/modules/image/oci/oci_image.c ++++ b/src/daemon/modules/image/oci/oci_image.c +@@ -359,7 +359,7 @@ void oci_exit(void) + free_oci_image_data(); + } + +-int oci_pull_rf(const im_pull_request *request, im_pull_response *response) ++int oci_pull_rf(const im_pull_request *request, stream_func_wrapper *stream, im_pull_response *response) + { + int ret = 0; + if (request == NULL || request->image == NULL || response == NULL) { +@@ -381,7 +381,7 @@ int oci_pull_rf(const im_pull_request *request, im_pull_response *response) + } + #endif + +- ret = oci_do_pull_image(request, response); ++ ret = oci_do_pull_image(request, stream, response); + + #ifdef ENABLE_REMOTE_LAYER_STORE + if (g_enable_remote) { +diff --git a/src/daemon/modules/image/oci/oci_image.h b/src/daemon/modules/image/oci/oci_image.h +index 07f10c8d..c7304897 100644 +--- a/src/daemon/modules/image/oci/oci_image.h ++++ b/src/daemon/modules/image/oci/oci_image.h +@@ -43,7 +43,7 @@ struct oci_image_module_data *get_oci_image_data(void); + int oci_init(const isulad_daemon_configs *args); + void oci_exit(void); + +-int oci_pull_rf(const im_pull_request *request, im_pull_response *response); ++int oci_pull_rf(const im_pull_request *request, stream_func_wrapper *stream, im_pull_response *response); + int oci_rmi(const im_rmi_request *request); + int oci_get_filesystem_info(im_fs_info_response **response); + int oci_load_image(const im_load_request *request); +diff --git a/src/daemon/modules/image/oci/oci_pull.c b/src/daemon/modules/image/oci/oci_pull.c +index e7ff77df..2706af91 100644 +--- a/src/daemon/modules/image/oci/oci_pull.c ++++ b/src/daemon/modules/image/oci/oci_pull.c +@@ -14,20 +14,25 @@ + *******************************************************************************/ + #include "oci_pull.h" + ++#include <isula_libutils/image_progress.h> ++#include <isula_libutils/log.h> ++#include <pthread.h> + #include <stdbool.h> + #include <stdlib.h> + #include <string.h> ++#include <unistd.h> + +-#include "isula_libutils/log.h" +-#include "utils.h" +-#include "utils_images.h" +-#include "registry.h" + #include "err_msg.h" ++#include "map.h" ++#include "oci_image.h" ++#include "progress.h" ++#include "registry.h" + #include "storage.h" ++#include "utils.h" + #include "utils_array.h" + #include "utils_base64.h" ++#include "utils_images.h" + #include "utils_string.h" +-#include "oci_image.h" + + static int decode_auth(const char *auth, char **username, char **password) + { +@@ -85,7 +90,7 @@ static void update_option_insecure_registry(registry_pull_options *options, char + } + } + +-static int pull_image(const im_pull_request *request, char **name) ++static int pull_image(const im_pull_request *request, progress_status_map *progress_status_store, char **name) + { + int ret = -1; + registry_pull_options *options = NULL; +@@ -112,6 +117,7 @@ static int pull_image(const im_pull_request *request, char **name) + options->auth.username = util_strdup_s(request->username); + options->auth.password = util_strdup_s(request->password); + } ++ options->progress_status_store = progress_status_store; + + oci_image_data = get_oci_image_data(); + options->skip_tls_verify = oci_image_data->insecure_skip_verify_enforce; +@@ -174,21 +180,131 @@ out: + return ret; + } + +-int oci_do_pull_image(const im_pull_request *request, im_pull_response *response) ++typedef struct status_arg { ++ progress_status_map *status_store; ++ bool should_terminal; ++ imagetool_image_summary *image; ++ char *image_name; ++ stream_func_wrapper *stream; ++} status_arg; ++ ++void *get_progress_status(void *arg) ++{ ++ status_arg *status = (status_arg *)arg; ++ const int delay = 100; // Sleep for 100 milliseconds ++ bool write_ok = false; ++ ++ if (status == NULL || status->status_store == NULL || status->stream == NULL) { ++ ERROR("Get progress status condition error"); ++ return NULL; ++ } ++ ++ for (;;) { ++ int i = 0; ++ ++ usleep(delay * 1000); // Sleep for 100 milliseconds ++ ++ if (status->should_terminal && status->image == NULL) { ++ break; ++ } ++ ++ image_progress *progresses; ++ size_t progress_size = progress_status_map_size(status->status_store); ++ ++ progresses = util_common_calloc_s(sizeof(image_progress)); ++ if (progresses == NULL) { ++ ERROR("Out of memory. Skip progress show."); ++ break; ++ } ++ ++ progresses->progresses = util_smart_calloc_s(sizeof(image_progress_progresses_element *), progress_size); ++ if (progresses->progresses == NULL) { ++ ERROR("Out of memory. Skip progress show."); ++ goto roundend; ++ } ++ if (status->image != NULL) { ++ progresses->image = util_strdup_s(status->image_name); ++ status->image = NULL; ++ } ++ ++ if (!progress_status_map_lock(status->status_store)) { ++ ERROR("Cannot itorate progress status map for locking failed"); ++ goto roundend; ++ } ++ map_itor *itor = map_itor_new(status->status_store->map); ++ for (i = 0; map_itor_valid(itor) && i < progress_size; map_itor_next(itor), i++) { ++ void *id = map_itor_key(itor); ++ const progress *value = (progress *)map_itor_value(itor); ++ const int ID_LEN = 12; // The last 12 charactos of image digest. ++ ++ progresses->progresses[i] = util_common_calloc_s(sizeof(image_progress_progresses_element)); ++ if (progresses->progresses[i] == NULL) { ++ WARN("Out of memory. Skip progress show."); ++ map_itor_free(itor); ++ progress_status_map_unlock(status->status_store); ++ goto roundend; ++ } ++ progresses->progresses[i]->id = util_strdup_s((char *)id + strlen((char *)id) - ID_LEN); ++ progresses->progresses[i]->total = value->dltotal; ++ progresses->progresses[i]->current = value->dlnow; ++ progresses->progresses_len++; ++ } ++ map_itor_free(itor); ++ progress_status_map_unlock(status->status_store); ++ ++ /* send to client */ ++ write_ok = status->stream->write_func(status->stream->writer, progresses); ++ if (write_ok) { ++ goto roundend; ++ } ++ if (status->stream->is_cancelled(status->stream->context)) { ++ ERROR("pull stream is cancelled"); ++ goto roundend; ++ } ++ ERROR("Send progress data to client failed"); ++roundend: ++ free_image_progress(progresses); ++ } ++ return NULL; ++} ++ ++int oci_do_pull_image(const im_pull_request *request, stream_func_wrapper *stream, im_pull_response *response) + { + int ret = 0; + imagetool_image_summary *image = NULL; + imagetool_image_summary *image2 = NULL; + char *dest_image_name = NULL; ++ progress_status_map *progress_status_store = NULL; + + if (request == NULL || request->image == NULL || response == NULL) { + ERROR("Invalid NULL param"); + return -1; + } + +- ret = pull_image(request, &dest_image_name); ++ pthread_t tid = 0; ++ status_arg arg = {0}; ++ if (request->is_progress_visible && stream != NULL) { ++ progress_status_store = progress_status_map_new(); ++ if (progress_status_store == NULL) { ++ ERROR("Out of memory and will not show the pull progress"); ++ isulad_set_error_message("Failed to pull image %s with error: out of memory", request->image); ++ ret = -1; ++ goto out; ++ } ++ arg.should_terminal = false; ++ arg.status_store = progress_status_store; ++ arg.stream = stream; ++ if (pthread_create(&tid, NULL, get_progress_status, (void *)&arg) != 0) { ++ ERROR("Failed to start thread to get progress status"); ++ isulad_set_error_message("Failed to pull image %s with error: start progress thread error", request->image); ++ ret = -1; ++ goto out; ++ } ++ } ++ ++ ret = pull_image(request, progress_status_store, &dest_image_name); + if (ret != 0) { +- ERROR("pull image %s failed", request->image); ++ ERROR("Pull image %s failed", request->image); + isulad_set_error_message("Failed to pull image %s with error: %s", request->image, g_isulad_errmsg); + ret = -1; + goto out; +@@ -197,17 +313,37 @@ int oci_do_pull_image(const im_pull_request *request, im_pull_response *response + image = storage_img_get_summary(dest_image_name); + image2 = storage_img_get_summary(request->image); + if (image == NULL || image2 == NULL) { +- ERROR("get image %s failed after pulling", request->image); ++ ERROR("Get image %s failed after pulling", request->image); + isulad_set_error_message("Failed to pull image %s with error: image not found after pulling", request->image); + ret = -1; + goto out; + } ++ arg.image = image; ++ arg.image_name = dest_image_name; ++ if (!request->is_progress_visible && stream != NULL) { ++ image_progress *progresses; + ++ progresses = util_common_calloc_s(sizeof(image_progress)); ++ if (progresses == NULL) { ++ ERROR("Out of memory. Skip progress show."); ++ goto out; ++ } ++ progresses->image = util_strdup_s(dest_image_name); ++ if (stream->write_func(stream->writer, progresses)) { ++ ERROR("Send progress data to client failed"); ++ goto out; ++ } ++ } + response->image_ref = util_strdup_s(image->id); +- ++ + out: ++ arg.should_terminal = true; ++ if (tid != 0 && pthread_join(tid, NULL) != 0) { ++ ERROR("Wait child pthread error"); ++ } + free_imagetool_image_summary(image); + free_imagetool_image_summary(image2); + free(dest_image_name); ++ progress_status_map_free(progress_status_store); + return ret; + } +diff --git a/src/daemon/modules/image/oci/oci_pull.h b/src/daemon/modules/image/oci/oci_pull.h +index 1b2eca33..79404cfe 100644 +--- a/src/daemon/modules/image/oci/oci_pull.h ++++ b/src/daemon/modules/image/oci/oci_pull.h +@@ -21,7 +21,7 @@ + extern "C" { + #endif + +-int oci_do_pull_image(const im_pull_request *request, im_pull_response *response); ++int oci_do_pull_image(const im_pull_request *request, stream_func_wrapper *stream, im_pull_response *response); + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/image/oci/progress.c b/src/daemon/modules/image/oci/progress.c +new file mode 100644 +index 00000000..110f22c0 +--- /dev/null ++++ b/src/daemon/modules/image/oci/progress.c +@@ -0,0 +1,124 @@ ++/****************************************************************************** ++ * Copyright (c) China Unicom Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: Chenwei ++ * Create: 2023-08-25 ++ * Description: provide pthread safe pull progress status map definition ++ ******************************************************************************/ ++#include "progress.h" ++#include <isula_libutils/log.h> ++#include <stdlib.h> ++ ++#include "utils.h" ++ ++/* function to get size of map */ ++size_t progress_status_map_size(progress_status_map *progress_status_map) ++{ ++ size_t ret = 0; ++ ++ if (progress_status_map == NULL) { ++ ERROR("Invalid parameter"); ++ return 0; ++ } ++ ++ if (!progress_status_map_lock(progress_status_map)) { ++ ERROR("Cannot get the progress status map size for locking failed"); ++ return 0; ++ } ++ ret = map_size(progress_status_map->map); ++ progress_status_map_unlock(progress_status_map); ++ ++ return ret; ++} ++ ++bool progress_status_map_insert(progress_status_map *progress_status_map, char *key, progress *value) ++{ ++ bool ret = false; ++ ++ if (progress_status_map == NULL || key == NULL || value == NULL) { ++ ERROR("Invalid parameter"); ++ return false; ++ } ++ ++ if (!progress_status_map_lock(progress_status_map)) { ++ ERROR("Cannot replace the progress status map item for locking failed"); ++ return false; ++ } ++ ret = map_insert(progress_status_map->map, key, value); ++ progress_status_map_unlock(progress_status_map); ++ ++ return ret; ++} ++ ++// malloc a new map by type ++progress_status_map *progress_status_map_new() ++{ ++ progress_status_map *progress_status_map = NULL; ++ progress_status_map = util_common_calloc_s(sizeof(struct progress_status_map)); ++ if (progress_status_map == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ progress_status_map->map = map_new(MAP_STR_PTR, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ if (progress_status_map->map == NULL) { ++ free(progress_status_map); ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ if (pthread_mutex_init(&(progress_status_map->mutex), NULL) != 0) { ++ map_free(progress_status_map->map); ++ free(progress_status_map); ++ ERROR("New map failed for mutex init"); ++ return NULL; ++ } ++ return progress_status_map; ++} ++ ++/* map free */ ++void progress_status_map_free(progress_status_map *progress_status_map) ++{ ++ if (progress_status_map == NULL) { ++ return; ++ } ++ ++ pthread_mutex_destroy(&(progress_status_map->mutex)); ++ map_free(progress_status_map->map); ++ free(progress_status_map); ++} ++ ++bool progress_status_map_lock(progress_status_map *progress_status_map) ++{ ++ int ret = 0; ++ ++ if (progress_status_map == NULL) { ++ return false; ++ } ++ ++ ret = pthread_mutex_lock(&(progress_status_map->mutex)); ++ if (ret != 0) { ++ ERROR("Lock progress status map failed: %s", strerror(ret)); ++ return false; ++ } ++ return true; ++} ++ ++void progress_status_map_unlock(progress_status_map *progress_status_map) ++{ ++ int ret = 0; ++ ++ if (progress_status_map == NULL) { ++ return; ++ } ++ ++ ret = pthread_mutex_unlock(&(progress_status_map->mutex)); ++ if (ret != 0) { ++ ERROR("Unlock progress status map failed: %s", strerror(ret)); ++ } ++} +diff --git a/src/daemon/modules/image/oci/progress.h b/src/daemon/modules/image/oci/progress.h +new file mode 100644 +index 00000000..496a32f3 +--- /dev/null ++++ b/src/daemon/modules/image/oci/progress.h +@@ -0,0 +1,52 @@ ++/****************************************************************************** ++ * Copyright (c) China Unicom Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: Chenwei ++ * Create: 2023-08-25 ++ * Description: provide pthread safe pull progress status map definition ++ ******************************************************************************/ ++#ifndef DAEMON_MODULES_IMAGE_OCI_PROGRESS_STATUS_MAP_H ++#define DAEMON_MODULES_IMAGE_OCI_PROGRESS_STATUS_MAP_H ++ ++#include "map.h" ++#include <pthread.h> ++#include <stdint.h> ++ ++#if defined(__cplusplus) || defined(c_plusplus) ++extern "C" { ++#endif ++ ++typedef struct progress_status_map { ++ struct _map_t *map; ++ pthread_mutex_t mutex; ++} progress_status_map; ++ ++typedef struct progress { ++ int64_t dlnow; ++ int64_t dltotal; ++} progress; ++ ++bool progress_status_map_insert(progress_status_map *progress_status_map, char *key, progress *value); ++ ++progress_status_map *progress_status_map_new(); ++ ++size_t progress_status_map_size(progress_status_map *progress_status_map); ++ ++void progress_status_map_free(progress_status_map *map); ++ ++bool progress_status_map_lock(progress_status_map *progress_status_map); ++ ++void progress_status_map_unlock(progress_status_map *progress_status_map); ++ ++#if defined(__cplusplus) || defined(c_plusplus) ++} ++#endif ++ ++#endif // DAEMON_MODULES_IMAGE_OCI_PROGRESS_STATUS_MAP_H +diff --git a/src/daemon/modules/image/oci/registry/http_request.c b/src/daemon/modules/image/oci/registry/http_request.c +index a514aaef..748c9a9b 100644 +--- a/src/daemon/modules/image/oci/registry/http_request.c ++++ b/src/daemon/modules/image/oci/registry/http_request.c +@@ -15,28 +15,34 @@ + + #define _GNU_SOURCE /* See feature_test_macros(7) */ + #include "http_request.h" +-#include <stdio.h> +-#include <string.h> ++#include <curl/curl.h> + #include <isula_libutils/json_common.h> ++#include <isula_libutils/log.h> ++#include <isula_libutils/registry_token.h> ++#include <pthread.h> + #include <stdbool.h> ++#include <stdio.h> + #include <stdlib.h> ++#include <string.h> + #include <strings.h> + #include <time.h> +-#include <curl/curl.h> +-#include <pthread.h> + +-#include "isula_libutils/log.h" + #include "buffer.h" ++#include "certs.h" ++#include "err_msg.h" + #include "http.h" + #include "utils.h" + #include "utils_images.h" +-#include "certs.h" +-#include "isula_libutils/registry_token.h" +-#include "err_msg.h" ++#include "progress.h" + #include "utils_array.h" + #include "utils_base64.h" + #include "utils_string.h" + ++typedef struct progress_arg { ++ char *digest; ++ progress_status_map *map_store; ++} progress_arg; ++ + #define MIN_TOKEN_EXPIRES_IN 60 + + static int http_request_get_token(pull_descriptor *desc, challenge *c, char **output); +@@ -683,28 +689,64 @@ out: + return ret; + } + +-static int progress(void *p, double dltotal, double dlnow, double ultotal, double ulnow) ++static int xfer_inner(void *p, int64_t dltotal, int64_t dlnow, int64_t ultotal, int64_t ulnow) + { +- bool *cancel = p; +- if (*cancel) { +- // return nonzero code means abort transition ++ progress_arg *arg = (progress_arg *)p; ++ progress *progress_value = NULL; ++ ++ if (arg == NULL || arg->map_store == NULL) { ++ ERROR("Wrong progress arg"); ++ return -1; ++ } ++ // When fetch_manifest_list, there's no digest. It's not a layer pulling progress and skip it. ++ if (arg->digest == NULL) { ++ return 0; ++ } ++ ++ if (!progress_status_map_lock(arg->map_store)) { ++ ERROR("Cannot update progress status map for locking failed"); + return -1; + } ++ ++ // If the item exists, only replace the value. ++ progress_value = map_search(arg->map_store->map, arg->digest); ++ if (progress_value != NULL) { ++ progress_value->dlnow = dlnow; ++ progress_value->dltotal = dltotal; ++ progress_status_map_unlock(arg->map_store); ++ ++ return 0; ++ } ++ progress_status_map_unlock(arg->map_store); ++ ++ progress_value = util_common_calloc_s(sizeof(progress)); ++ if (progress_value == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ ++ progress_value->dlnow = dlnow; ++ progress_value->dltotal = dltotal; ++ ++ progress_status_map_insert(arg->map_store, arg->digest, progress_value); ++ + return 0; + } + ++#if (LIBCURL_VERSION_NUM >= 0x072000) + static int xfer(void *p, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) + { +- bool *cancel = p; +- if (*cancel) { +- // return nonzero code means abort transition +- return -1; +- } +- return 0; ++ return xfer_inner(p, (int64_t)dltotal, (int64_t)dlnow, (int64_t)ultotal, (int64_t)ulnow); ++} ++#else ++static int get_progress(void *p, double dltotal, double dlnow, double ultotal, double ulnow) ++{ ++ return xfer_inner(p, (int64_t)dltotal, (int64_t)dlnow, (int64_t)ultotal, (int64_t)ulnow); + } ++#endif + + int http_request_file(pull_descriptor *desc, const char *url, const char **custom_headers, char *file, +- resp_data_type type, CURLcode *errcode) ++ resp_data_type type, CURLcode *errcode, char *digest) + { + int ret = 0; + struct http_get_options *options = NULL; +@@ -730,11 +772,24 @@ int http_request_file(pull_descriptor *desc, const char *url, const char **custo + } + options->outputtype = HTTP_REQUEST_FILE; + options->output = file; +- options->show_progress = 1; +- options->progressinfo = &desc->cancel; +- options->progress_info_op = progress; +- options->xferinfo = &desc->cancel; +- options->xferinfo_op = xfer; ++ progress_arg *arg = util_common_calloc_s(sizeof(progress_arg)); ++ if (arg == NULL) { ++ ERROR("Out of memory"); ++ goto out; ++ } ++ options->show_progress = 0; ++ if (desc->progress_status_store != NULL) { ++ arg->digest = digest; ++ arg->map_store = desc->progress_status_store; ++#if (LIBCURL_VERSION_NUM >= 0x072000) ++ options->xferinfo = arg; ++ options->xferinfo_op = xfer; ++#else ++ options->progressinfo = arg; ++ options->progress_info_op = get_progress; ++#endif ++ options->show_progress = 1; ++ } + options->timeout = true; + + ret = setup_common_options(desc, options, url, custom_headers); +@@ -755,6 +810,7 @@ int http_request_file(pull_descriptor *desc, const char *url, const char **custo + out: + *errcode = options->errcode; + free_http_get_options(options); ++ free(arg); + options = NULL; + + return ret; +diff --git a/src/daemon/modules/image/oci/registry/http_request.h b/src/daemon/modules/image/oci/registry/http_request.h +index 71df37d7..ed3f7e98 100644 +--- a/src/daemon/modules/image/oci/registry/http_request.h ++++ b/src/daemon/modules/image/oci/registry/http_request.h +@@ -32,7 +32,7 @@ typedef enum { + int http_request_buf(pull_descriptor *desc, const char *url, const char **custom_headers, char **output, + resp_data_type type); + int http_request_file(pull_descriptor *desc, const char *url, const char **custom_headers, char *file, +- resp_data_type type, CURLcode *errcode); ++ resp_data_type type, CURLcode *errcode, char *digest); + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/image/oci/registry/registry.c b/src/daemon/modules/image/oci/registry/registry.c +index 4124281d..875f2df2 100644 +--- a/src/daemon/modules/image/oci/registry/registry.c ++++ b/src/daemon/modules/image/oci/registry/registry.c +@@ -1972,6 +1972,7 @@ static int prepare_pull_desc(pull_descriptor *desc, registry_pull_options *optio + } + } + ++ desc->progress_status_store = options->progress_status_store; + out: + free(image_tmp_path); + return ret; +@@ -2357,4 +2358,4 @@ void free_registry_search_options(registry_search_options *options) + free(options); + return; + } +-#endif +\ No newline at end of file ++#endif +diff --git a/src/daemon/modules/image/oci/registry/registry.h b/src/daemon/modules/image/oci/registry/registry.h +index cafb11c6..bb2af348 100644 +--- a/src/daemon/modules/image/oci/registry/registry.h ++++ b/src/daemon/modules/image/oci/registry/registry.h +@@ -16,6 +16,7 @@ + #define DAEMON_MODULES_IMAGE_OCI_REGISTRY_REGISTRY_H + + #include <stdbool.h> ++#include "progress.h" + + #ifdef ENABLE_IMAGE_SEARCH + #include <isula_libutils/imagetool_search_result.h> +@@ -36,6 +37,7 @@ typedef struct { + registry_auth auth; + bool skip_tls_verify; + bool insecure_registry; ++ progress_status_map *progress_status_store; // Don't free it. It's freed at oci_pull.c. + } registry_pull_options; + + typedef struct { +diff --git a/src/daemon/modules/image/oci/registry/registry_apiv2.c b/src/daemon/modules/image/oci/registry/registry_apiv2.c +index db4d311e..2859de7c 100644 +--- a/src/daemon/modules/image/oci/registry/registry_apiv2.c ++++ b/src/daemon/modules/image/oci/registry/registry_apiv2.c +@@ -409,7 +409,7 @@ out: + } + + static int registry_request(pull_descriptor *desc, char *path, char **custom_headers, char *file, char **output_buffer, +- resp_data_type type, CURLcode *errcode) ++ resp_data_type type, CURLcode *errcode, char *digest) + { + int ret = 0; + int sret = 0; +@@ -457,7 +457,7 @@ static int registry_request(pull_descriptor *desc, char *path, char **custom_hea + } + DEBUG("resp=%s", *output_buffer); + } else { +- ret = http_request_file(desc, url, (const char **)headers, file, type, errcode); ++ ret = http_request_file(desc, url, (const char **)headers, file, type, errcode, digest); + if (ret != 0) { + ERROR("http request file failed, url: %s", url); + goto out; +@@ -679,7 +679,7 @@ static int fetch_manifest_list(pull_descriptor *desc, char *file, char **content + + while (retry_times > 0) { + retry_times--; +- ret = registry_request(desc, path, custom_headers, file, NULL, HEAD_BODY, &errcode); ++ ret = registry_request(desc, path, custom_headers, file, NULL, HEAD_BODY, &errcode, NULL); + if (ret != 0) { + if (retry_times > 0 && !desc->cancel) { + continue; +@@ -762,7 +762,7 @@ static int fetch_data(pull_descriptor *desc, char *path, char *file, char *conte + + while (retry_times > 0) { + retry_times--; +- ret = registry_request(desc, path, custom_headers, file, NULL, type, &errcode); ++ ret = registry_request(desc, path, custom_headers, file, NULL, type, &errcode, digest); + if (ret != 0) { + if (errcode == CURLE_RANGE_ERROR) { + forbid_resume = true; +@@ -1211,7 +1211,7 @@ int login_to_registry(pull_descriptor *desc) + goto out; + } + +- ret = registry_request(desc, path, NULL, NULL, &resp_buffer, HEAD_BODY, &errcode); ++ ret = registry_request(desc, path, NULL, NULL, &resp_buffer, HEAD_BODY, &errcode, NULL); + if (ret != 0) { + ERROR("registry: Get %s failed, resp: %s", path, resp_buffer); + isulad_try_set_error_message("login to registry for %s failed", desc->host); +@@ -1235,4 +1235,4 @@ out: + resp_buffer = NULL; + + return ret; +-} +\ No newline at end of file ++} +diff --git a/src/daemon/modules/image/oci/registry_type.h b/src/daemon/modules/image/oci/registry_type.h +index f232f227..8ddfcfea 100644 +--- a/src/daemon/modules/image/oci/registry_type.h ++++ b/src/daemon/modules/image/oci/registry_type.h +@@ -20,6 +20,7 @@ + #include <time.h> + #include <stdbool.h> + ++#include "progress.h" + #include "utils_timestamp.h" + + // 8 is enough for challenge, usually only one challenge is provided. +@@ -134,6 +135,8 @@ typedef struct { + char *search_name; + uint32_t limit; + #endif ++ ++ progress_status_map *progress_status_store; // Don't free it. It's freed at other place. + } pull_descriptor; + + void free_challenge(challenge *c); +diff --git a/src/utils/CMakeLists.txt b/src/utils/CMakeLists.txt +index 6933caf5..42814fd6 100644 +--- a/src/utils/CMakeLists.txt ++++ b/src/utils/CMakeLists.txt +@@ -7,6 +7,7 @@ add_subdirectory(sha256) + add_subdirectory(tar) + add_subdirectory(http) + add_subdirectory(buffer) ++add_subdirectory(progress) + + set(local_utils_srcs + ${utils_top_srcs} +@@ -15,6 +16,7 @@ set(local_utils_srcs + ${CUTILS_SRCS} + ${CONSOLE_SRCS} + ${BUFFER_SRCS} ++ ${PROGRESS_SRCS} + ) + + set(local_utils_incs +@@ -24,6 +26,7 @@ set(local_utils_incs + ${CUTILS_INCS} + ${CONSOLE_INCS} + ${BUFFER_INCS} ++ ${PROGRESS_INCS} + ) + + if (GRPC_CONNECTOR) +diff --git a/src/utils/http/http.h b/src/utils/http/http.h +index 02d56ba8..585afdf1 100644 +--- a/src/utils/http/http.h ++++ b/src/utils/http/http.h +@@ -23,12 +23,15 @@ + extern "C" { + #endif + +-typedef int(*progress_info_func)(void *p, +- double dltotal, double dlnow, +- double ultotal, double ulnow); ++#if (LIBCURL_VERSION_NUM >= 0x072000) + typedef int(*xferinfo_func)(void *p, + curl_off_t dltotal, curl_off_t dlnow, + curl_off_t ultotal, curl_off_t ulnow); ++#else ++typedef int(*progress_info_func)(void *p, ++ double dltotal, double dlnow, ++ double ultotal, double ulnow); ++#endif + + struct http_get_options { + unsigned with_head : 1, /* if set, means write output with response HEADER */ +@@ -79,11 +82,13 @@ struct http_get_options { + + bool timeout; + +- void *progressinfo; +- progress_info_func progress_info_op; +- ++#if (LIBCURL_VERSION_NUM >= 0x072000) + void *xferinfo; + xferinfo_func xferinfo_op; ++#else ++ void *progressinfo; ++ progress_info_func progress_info_op; ++#endif + }; + + #define HTTP_RES_OK 0 +diff --git a/src/utils/progress/CMakeLists.txt b/src/utils/progress/CMakeLists.txt +new file mode 100644 +index 00000000..d06cca33 +--- /dev/null ++++ b/src/utils/progress/CMakeLists.txt +@@ -0,0 +1,13 @@ ++# get current directory sources files ++aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_progress_srcs) ++ ++set(PROGRESS_SRCS ++ ${local_progress_srcs} ++ PARENT_SCOPE ++ ) ++ ++set(PROGRESS_INCS ++ ${CMAKE_CURRENT_SOURCE_DIR} ++ PARENT_SCOPE ++ ) ++ +diff --git a/src/utils/progress/show.c b/src/utils/progress/show.c +new file mode 100644 +index 00000000..fbefe344 +--- /dev/null ++++ b/src/utils/progress/show.c +@@ -0,0 +1,64 @@ ++/****************************************************************************** ++ * Copyright (c) China Unicom Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: Chenwei ++ * Create: 2023-08-25 ++ * Description: print progress ++ ******************************************************************************/ ++ ++#include "show.h" ++#include <sys/ioctl.h> ++#include <stdio.h> ++#include <term.h> ++#include <unistd.h> ++ ++void move_to_row(int row) ++{ ++ printf("\033[%d;1H", row); ++ fflush(stdout); ++} ++ ++void move_cursor_up(int rows) ++{ ++ printf("\033[%dA", rows); // ANSI escape code to move cursor up 'rows' rows ++} ++ ++void clear_row(int row) ++{ ++ move_to_row(row); ++ printf("\033[2K"); ++ fflush(stdout); ++} ++ ++void clear_lines_below() ++{ ++ printf("\x1b[J"); // ANSI escape code to clear from cursor to end of screen ++ fflush(stdout); ++} ++ ++int get_current_row() ++{ ++ struct termios term; ++ if (tcgetattr(STDOUT_FILENO, &term) == -1) { ++ perror("tcgetattr"); ++ return -1; ++ } ++ return term.c_cc[VERASE]; ++} ++ ++int get_terminal_width() ++{ ++ struct winsize ws; ++ if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws) == -1) { ++ perror("ioctl"); ++ return -1; // Error ++ } ++ return ws.ws_col; ++} +diff --git a/src/utils/progress/show.h b/src/utils/progress/show.h +new file mode 100644 +index 00000000..c1f71d86 +--- /dev/null ++++ b/src/utils/progress/show.h +@@ -0,0 +1,34 @@ ++/****************************************************************************** ++ * Copyright (c) China Unicom Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: Chenwei ++ * Create: 2023-08-25 ++ * Description: print progress ++ ******************************************************************************/ ++ ++#ifndef UTILS_SHOW_H ++#define UTILS_SHOW_H ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++void move_to_row(int row); ++void move_cursor_up(int lines); ++void clear_row(int row); ++void clear_lines_below(); ++int get_current_row(); ++int get_terminal_width(); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +diff --git a/test/cutils/CMakeLists.txt b/test/cutils/CMakeLists.txt +index 10a10db9..9e681cc9 100644 +--- a/test/cutils/CMakeLists.txt ++++ b/test/cutils/CMakeLists.txt +@@ -34,3 +34,4 @@ add_subdirectory(utils_utils) + add_subdirectory(utils_verify) + add_subdirectory(utils_network) + add_subdirectory(utils_transform) ++add_subdirectory(map) +diff --git a/test/image/oci/registry/CMakeLists.txt b/test/image/oci/registry/CMakeLists.txt +index f9ba056e..77a7907e 100644 +--- a/test/image/oci/registry/CMakeLists.txt ++++ b/test/image/oci/registry/CMakeLists.txt +@@ -18,6 +18,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/utils/cutils/map/rb_tree.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/utils/cutils/utils_timestamp.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/utils_images.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/progress.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/err_msg.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/utils/http/parser.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/utils/buffer/buffer.c +diff --git a/test/image/oci/registry/registry_ut.cc b/test/image/oci/registry/registry_ut.cc +index f4f8a763..3cb3e371 100644 +--- a/test/image/oci/registry/registry_ut.cc ++++ b/test/image/oci/registry/registry_ut.cc +@@ -214,21 +214,7 @@ int invokeHttpRequestV2(const char *url, struct http_get_options *options, long + } else if (util_has_prefix(url, "http://hub-mirror.c.163.com/v2/library/busybox/blobs/sha256:c7c37e47")) { + file = data_path + "config"; + if (count == COUNT_TEST_CANCEL) { +- bool *cancel = (bool *)options->progressinfo; +- while (!(*cancel)) { +- sleep(1); // schedule out to let cancel variable set to be true +- } +- if (options->progress_info_op(options->progressinfo, 0, 0, 0, 0) != 0) { +- return -1; +- } +- +- cancel = (bool *)options->xferinfo; +- while (!(*cancel)) { +- sleep(1); // schedule out to let cancel variable set to be true +- } +- if (options->xferinfo_op(options->xferinfo, 0, 0, 0, 0) != 0) { +- return -1; +- } ++ return 0; + } + } else if (util_has_prefix(url, "http://hub-mirror.c.163.com/v2/library/busybox/blobs/sha256:91f30d77")) { + if (retry) { +-- +2.42.0 + diff --git a/0025-2084-image-pull.patch b/0025-2084-image-pull.patch new file mode 100644 index 0000000..0fe36f1 --- /dev/null +++ b/0025-2084-image-pull.patch @@ -0,0 +1,84 @@ +From 79384f7b0ac7319120d1f677323c43069742a354 Mon Sep 17 00:00:00 2001 +From: sailorvii <chenw66@chinaunicom.cn> +Date: Wed, 22 Nov 2023 01:22:42 +0000 +Subject: [PATCH 25/64] =?UTF-8?q?!2084=20=E5=A2=9E=E5=8A=A0image=20pull=20?= + =?UTF-8?q?=E6=97=B6=E8=BF=9B=E5=BA=A6=E6=9D=A1=E6=98=BE=E7=A4=BA=E7=9A=84?= + =?UTF-8?q?=E8=AE=BE=E8=AE=A1=20*=20Refine=20document=20by=20the=20impleme?= + =?UTF-8?q?ntation.=20*=20Refine=20a=20word.=20*=20Add=20progress=20bard?= + =?UTF-8?q?=20proposal.?= +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +--- + docs/design/detailed/Misc/progressBar.md | 58 ++++++++++++++++++++++++ + 1 file changed, 58 insertions(+) + create mode 100644 docs/design/detailed/Misc/progressBar.md + +diff --git a/docs/design/detailed/Misc/progressBar.md b/docs/design/detailed/Misc/progressBar.md +new file mode 100644 +index 00000000..3cf733a6 +--- /dev/null ++++ b/docs/design/detailed/Misc/progressBar.md +@@ -0,0 +1,58 @@ ++# 方案目标 ++在Image pull过程中，显示多个layer下载的进度。 ++ ++之前的grpc pull和cri pull共用了接口，需要新增grpc pull接口，该接口类型为stream，带progress status。 ++重写函数oci_do_pull_image，底层函数pull_image复用。 ++在结构体registry_pull_options增加map。 ++ ++# 限制 ++1. 每一个connection只做一件事，否则progress store会混乱。 ++2. 这个功能只为grpc 连接服务。 ++ ++# 总体设计 ++## 主要功能模块 ++### Progress status store ++每次pull命令或者行为为一个connection。每个image会按照layer来下载。所以我们建立了一个status map。 map的key为Layer ID，内容结构体定义如下: ++ ++``` ++struct progress_status { ++ // Layer ID ++ char ID[13]; ++ ++ // total is the end value describing when we made 100% progress for an operation. Unit is Byte. ++ int64 total; ++ ++ // current is the current value for the operation. Unit is Byte. ++ int64 current; ++} ++``` ++ ++#### API ++``` ++progress_status_map *progress_status_map_new(); ++ ++bool progress_status_map_insert(progress_status_map *progress_status_map, char *key, progress *value); ++ ++``` ++ ++### Client Progress 显示 ++在client每次读到消息时，获取当前窗口宽度(termios.h: tcgetattr)，如果宽度小于110字符，则压缩显示(已下载/全部字节)，如果不是，则显示进度条。 ++当第一次收到时，计算需要显示的任务数task number，每个任务显示一行。 ++当更新状态时，将光标回退task number行，清除该行，打印完一行，将光标移到下一行清除该行并打印新的进度，重复上述步骤直至所有任务打印完成。 ++ ++## 主要流程 ++### 下载任务获取下载状态 ++在结构体pull_descriptor新增*progress_status_store， 传递write_progress_status的map *。 ++ ++在http_request中，修改原来的桩函数xfer，这个函数将实时采集curl pull的状态，如当前下载的字节数，总的字节数。 ++ ++ ++### server获取下载状态并传递给client ++新增函数int ImagesServiceImpl::PullImage，函数Response参数为stream，每隔100ms读取progress status map并序列化为json message，写入response stream。 ++``` ++Status ImagesServiceImpl::PullImage(ServerContext *context, const PullImageRequest *request, ++ ServerWriter<PullImageResponse> *writer) ++``` ++ ++### client收取状态并显示 ++修改原来的grpc_images_client中ImagesPull函数。阻塞式读取response stream， 流不为空则一直读取并打印显示每个progress status。 +-- +2.42.0 + diff --git a/0026-CI-add-ncurse-for-ubuntu-and-centos.patch b/0026-CI-add-ncurse-for-ubuntu-and-centos.patch new file mode 100644 index 0000000..de75868 --- /dev/null +++ b/0026-CI-add-ncurse-for-ubuntu-and-centos.patch @@ -0,0 +1,39 @@ +From c87ecc7f26a0a0034a8bf49691f572fe1d4fed29 Mon Sep 17 00:00:00 2001 +From: haozi007 <liuhao27@huawei.com> +Date: Wed, 22 Nov 2023 15:00:24 +0800 +Subject: [PATCH 26/64] [CI] add ncurse for ubuntu and centos + +Signed-off-by: haozi007 <liuhao27@huawei.com> +--- + CI/dockerfiles/Dockerfile-centos | 2 ++ + CI/dockerfiles/Dockerfile-ubuntu | 2 +- + 2 files changed, 3 insertions(+), 1 deletion(-) + +diff --git a/CI/dockerfiles/Dockerfile-centos b/CI/dockerfiles/Dockerfile-centos +index 7250b7bd..1d76b4ec 100644 +--- a/CI/dockerfiles/Dockerfile-centos ++++ b/CI/dockerfiles/Dockerfile-centos +@@ -299,5 +299,7 @@ RUN export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH && \ + make install && \ + ldconfig + ++RUN dnf install -y ncurses-devel && dnf clean all ++ + VOLUME [ "/sys/fs/cgroup" ] + CMD ["/usr/sbin/init"] +diff --git a/CI/dockerfiles/Dockerfile-ubuntu b/CI/dockerfiles/Dockerfile-ubuntu +index f84ae0a7..2441a7ce 100644 +--- a/CI/dockerfiles/Dockerfile-ubuntu ++++ b/CI/dockerfiles/Dockerfile-ubuntu +@@ -84,7 +84,7 @@ RUN apt update -y && apt upgrade -y && \ + patch \ + tcpdump + +-RUN apt autoremove -y ++RUN apt install -y libncurses-dev && apt autoremove -y + RUN pip3 install meson ninja + + RUN echo "export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH" >> /etc/bashrc && \ +-- +2.42.0 + diff --git a/0027-improve-code-of-pull-progress.patch b/0027-improve-code-of-pull-progress.patch new file mode 100644 index 0000000..86b4eca --- /dev/null +++ b/0027-improve-code-of-pull-progress.patch @@ -0,0 +1,631 @@ +From 78304f7ad584517e02125c928e976f34aaf859f8 Mon Sep 17 00:00:00 2001 +From: haozi007 <liuhao27@huawei.com> +Date: Wed, 22 Nov 2023 15:00:43 +0800 +Subject: [PATCH 27/64] improve code of pull progress + +Signed-off-by: haozi007 <liuhao27@huawei.com> +--- + src/client/connect/CMakeLists.txt | 7 +- + src/client/connect/grpc/grpc_images_client.cc | 48 ++---- + .../connect/grpc/grpc_volumes_client.cc | 1 - + src/daemon/common/events_format.h | 2 + + .../v1/v1_cri_image_manager_service_impl.cc | 2 - + .../v1alpha/cri_image_manager_service_impl.cc | 2 - + src/daemon/executor/image_cb/image_cb.c | 2 + + src/daemon/modules/api/event_type.h | 4 +- + src/daemon/modules/events/collector.c | 4 +- + src/daemon/modules/image/image.c | 2 +- + src/daemon/modules/image/oci/oci_pull.c | 146 ++++++++++-------- + src/daemon/modules/image/oci/progress.c | 28 +++- + src/daemon/modules/image/oci/progress.h | 6 +- + .../modules/image/oci/registry/http_request.c | 28 +--- + .../oci/storage/image_store/image_store.c | 2 +- + .../graphdriver/overlay2/driver_overlay2.c | 2 +- + .../modules/image/oci/storage/storage.c | 5 +- + 17 files changed, 137 insertions(+), 154 deletions(-) + +diff --git a/src/client/connect/CMakeLists.txt b/src/client/connect/CMakeLists.txt +index 00ba2f68..d4ce6c9c 100644 +--- a/src/client/connect/CMakeLists.txt ++++ b/src/client/connect/CMakeLists.txt +@@ -12,10 +12,7 @@ if (GRPC_CONNECTOR) + aux_source_directory(${CMAKE_BINARY_DIR}/grpc/src/api/services/containers CONNECT_API_CONTAINERS) + aux_source_directory(${CMAKE_BINARY_DIR}/grpc/src/api/services/images CONNECT_API_IMAGES) + aux_source_directory(${CMAKE_BINARY_DIR}/grpc/src/api/services/volumes CONNECT_API_VOLUMES) +- # TODO: current isula pull use CRI pullImage API, we should remove this dependence +- aux_source_directory(${CMAKE_BINARY_DIR}/grpc/src/api/services/cri CONNECT_API_CRI) +- aux_source_directory(${CMAKE_BINARY_DIR}/grpc/src/api/services/cri/v1alpha CONNECT_API_CRI_ALPHAS) +- set(CONNECT_API ${CONNECT_API_VOLUMES} ${CONNECT_API_CONTAINERS} ${CONNECT_API_IMAGES} ${CONNECT_API_CRI_ALPHAS} ${CONNECT_API_CRI}) ++ set(CONNECT_API ${CONNECT_API_VOLUMES} ${CONNECT_API_CONTAINERS} ${CONNECT_API_IMAGES}) + list(APPEND local_client_connect_srcs ${CONNECT_API}) + + list(APPEND local_client_connect_incs ${CMAKE_CURRENT_SOURCE_DIR}/grpc) +@@ -23,8 +20,6 @@ if (GRPC_CONNECTOR) + ${CMAKE_BINARY_DIR}/grpc/src/api/services/volumes + ${CMAKE_BINARY_DIR}/grpc/src/api/services/containers + ${CMAKE_BINARY_DIR}/grpc/src/api/services/images +- ${CMAKE_BINARY_DIR}/grpc/src/api/services/cri +- ${CMAKE_BINARY_DIR}/grpc/src/api/services/cri/v1alpha + ) + + if(ENABLE_NATIVE_NETWORK) +diff --git a/src/client/connect/grpc/grpc_images_client.cc b/src/client/connect/grpc/grpc_images_client.cc +index 7a283e8c..7fd36cc1 100644 +--- a/src/client/connect/grpc/grpc_images_client.cc ++++ b/src/client/connect/grpc/grpc_images_client.cc +@@ -390,50 +390,20 @@ public: + return 0; + } + +- auto run(const struct isula_pull_request *request, struct isula_pull_response *response) -> int override +- { +- ClientContext context; +- PullImageRequest grequest; +- +-#ifdef ENABLE_GRPC_REMOTE_CONNECT +-#ifdef OPENSSL_VERIFY +- // Set common name from cert.perm +- char common_name_value[ClientBaseConstants::COMMON_NAME_LEN] = { 0 }; +- int ret = get_common_name_from_tls_cert(m_certFile.c_str(), common_name_value, +- ClientBaseConstants::COMMON_NAME_LEN); +- if (ret != 0) { +- ERROR("Failed to get common name in: %s", m_certFile.c_str()); +- return -1; +- } +- context.AddMetadata("username", std::string(common_name_value, strlen(common_name_value))); +- context.AddMetadata("tls_mode", m_tlsMode); +-#endif +-#endif +- if (request_to_grpc(request, &grequest) != 0) { +- ERROR("Failed to transform pull request to grpc"); +- response->server_errono = ISULAD_ERR_INPUT; +- return -1; +- } +- +- auto reader = stub_->PullImage(&context, grequest); ++ auto grpc_call(ClientContext *context, const PullImageRequest &req, PullImageResponse *reply) -> Status override ++ { ++ auto reader = stub_->PullImage(context, req); + +- PullImageResponse gresponse; +- if (grequest.is_progress_visible()) { +- while (reader->Read(&gresponse)) { +- output_progress(gresponse); ++ if (req.is_progress_visible()) { ++ while (reader->Read(reply)) { ++ output_progress(*reply); + } + } else { +- reader->Read(&gresponse); ++ reader->Read(reply); + WARN("The terminal may not support ANSI Escape code. Display is skipped"); + } +- Status status = reader->Finish(); +- if (!status.ok()) { +- ERROR("Error code: %d: %s", status.error_code(), status.error_message().c_str()); +- unpackStatus(status, response); +- return -1; +- } +- response->image_ref = util_strdup_s(gresponse.image_ref().c_str()); +- return 0; ++ ++ return reader->Finish(); + } + + private: +diff --git a/src/client/connect/grpc/grpc_volumes_client.cc b/src/client/connect/grpc/grpc_volumes_client.cc +index 32b83a9e..5fe8ed5e 100644 +--- a/src/client/connect/grpc/grpc_volumes_client.cc ++++ b/src/client/connect/grpc/grpc_volumes_client.cc +@@ -16,7 +16,6 @@ + + #include <string> + +-#include "api.grpc.pb.h" + #include "client_base.h" + #include "volumes.grpc.pb.h" + #include "utils.h" +diff --git a/src/daemon/common/events_format.h b/src/daemon/common/events_format.h +index 7e97b2c5..6b8fcfd5 100644 +--- a/src/daemon/common/events_format.h ++++ b/src/daemon/common/events_format.h +@@ -64,6 +64,8 @@ typedef enum { + EVENTS_TYPE_IMAGE_PULL, + EVENTS_TYPE_IMAGE_LOGIN, + EVENTS_TYPE_IMAGE_LOGOUT, ++ EVENTS_TYPE_IMAGE_IMPORT, ++ EVENTS_TYPE_IMAGE_TAG, + EVENTS_TYPE_IMAGE_MAX_STATE + } image_events_type_t; + +diff --git a/src/daemon/entry/cri/v1/v1_cri_image_manager_service_impl.cc b/src/daemon/entry/cri/v1/v1_cri_image_manager_service_impl.cc +index b9cbf24c..066eed5e 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_image_manager_service_impl.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_image_manager_service_impl.cc +@@ -25,7 +25,6 @@ + + #include "v1_cri_helpers.h" + #include "err_msg.h" +-#include "events_sender_api.h" + #include "isula_libutils/log.h" + #include "service_image_api.h" + #include "utils.h" +@@ -277,7 +276,6 @@ auto ImageManagerServiceImpl::PullImage(const runtime::v1::ImageSpec &image, + if (response->image_ref != nullptr) { + out_str = response->image_ref; + } +- (void)isulad_monitor_send_image_event(request->image, IM_PULL); + + cleanup: + DAEMON_CLEAR_ERRMSG(); +diff --git a/src/daemon/entry/cri/v1alpha/cri_image_manager_service_impl.cc b/src/daemon/entry/cri/v1alpha/cri_image_manager_service_impl.cc +index 0b36f007..9015df26 100644 +--- a/src/daemon/entry/cri/v1alpha/cri_image_manager_service_impl.cc ++++ b/src/daemon/entry/cri/v1alpha/cri_image_manager_service_impl.cc +@@ -25,7 +25,6 @@ + + #include "cri_helpers.h" + #include "err_msg.h" +-#include "events_sender_api.h" + #include "isula_libutils/log.h" + #include "service_image_api.h" + #include "utils.h" +@@ -277,7 +276,6 @@ auto ImageManagerServiceImpl::PullImage(const runtime::v1alpha2::ImageSpec &imag + if (response->image_ref != nullptr) { + out_str = response->image_ref; + } +- (void)isulad_monitor_send_image_event(request->image, IM_PULL); + + cleanup: + DAEMON_CLEAR_ERRMSG(); +diff --git a/src/daemon/executor/image_cb/image_cb.c b/src/daemon/executor/image_cb/image_cb.c +index 317cb0a8..60899f2b 100644 +--- a/src/daemon/executor/image_cb/image_cb.c ++++ b/src/daemon/executor/image_cb/image_cb.c +@@ -519,6 +519,7 @@ static int image_tag_cb(const image_tag_image_request *request, image_tag_image_ + } + + EVENT("Image Event: {Object: %s, Type: Tagged}", request->src_name); ++ (void)isulad_monitor_send_image_event(request->src_name, IM_TAG); + + out: + if (*response != NULL) { +@@ -997,6 +998,7 @@ static int image_pull_cb(const image_pull_image_request *request, stream_func_wr + } + + EVENT("Image Event: {Object: %s, Type: Pulled}", request->image_name); ++ (void)isulad_monitor_send_image_event(request->image_name, IM_PULL); + + out: + (*response)->cc = cc; +diff --git a/src/daemon/modules/api/event_type.h b/src/daemon/modules/api/event_type.h +index c3c7951b..4f2aaf28 100644 +--- a/src/daemon/modules/api/event_type.h ++++ b/src/daemon/modules/api/event_type.h +@@ -54,7 +54,9 @@ typedef enum { + MAX_STATE, + } runtime_state_t; + +-typedef enum { IM_LOAD, IM_REMOVE, IM_PULL, IM_LOGIN, IM_LOGOUT, IM_IMPORT } image_state_t; ++// relate to g_isulad_image_event_strtype and image_events_type_t ++// we should keep them consistent ++typedef enum { IM_LOAD, IM_REMOVE, IM_PULL, IM_LOGIN, IM_LOGOUT, IM_IMPORT, IM_TAG } image_state_t; + + typedef enum { CONTAINER_EVENT, IMAGE_EVENT } msg_event_type_t; + typedef enum { MONITORD_MSG_STATE, MONITORD_MSG_PRIORITY, MONITORD_MSG_EXIT_CODE } msg_type_t; +diff --git a/src/daemon/modules/events/collector.c b/src/daemon/modules/events/collector.c +index b82ede81..36aa9299 100644 +--- a/src/daemon/modules/events/collector.c ++++ b/src/daemon/modules/events/collector.c +@@ -157,11 +157,11 @@ static const char *isulad_event_sta2str(container_events_type_t sta) + return g_isulad_event_strtype[sta]; + } + +-static const char * const g_isulad_image_event_strtype[] = { "load", "remove", "pull", "login", "logout" }; ++static const char * const g_isulad_image_event_strtype[] = { "load", "remove", "pull", "login", "logout", "import", "tag" }; + + static const char *isulad_image_event_sta2str(image_events_type_t sta) + { +- if (sta > EVENTS_TYPE_IMAGE_LOGOUT) { ++ if (sta >= EVENTS_TYPE_IMAGE_MAX_STATE) { + return NULL; + } + +diff --git a/src/daemon/modules/image/image.c b/src/daemon/modules/image/image.c +index 8d7e2c1a..4a1950fe 100644 +--- a/src/daemon/modules/image/image.c ++++ b/src/daemon/modules/image/image.c +@@ -784,7 +784,7 @@ int im_merge_image_config(const char *image_type, const char *image_name, contai + int ret = 0; + struct bim *bim = NULL; + +- // there is no need to judge the image name as empty, ++ // there is no need to judge the image name as empty, + // because the image name of external type allows it to be empty. + if (container_spec == NULL || image_type == NULL) { + ERROR("Invalid input arguments"); +diff --git a/src/daemon/modules/image/oci/oci_pull.c b/src/daemon/modules/image/oci/oci_pull.c +index 2706af91..9ad875a5 100644 +--- a/src/daemon/modules/image/oci/oci_pull.c ++++ b/src/daemon/modules/image/oci/oci_pull.c +@@ -75,7 +75,8 @@ out: + return ret; + } + +-static void update_option_insecure_registry(registry_pull_options *options, char **insecure_registries, const char *host) ++static void update_option_insecure_registry(registry_pull_options *options, char **insecure_registries, ++ const char *host) + { + char **registry = NULL; + +@@ -188,83 +189,95 @@ typedef struct status_arg { + stream_func_wrapper *stream; + } status_arg; + ++static int do_get_progress_from_store(progress_status_map *status_store, image_progress *result) ++{ ++ int i = 0; ++ size_t progress_size = progress_status_map_size(status_store); ++ ++ result->progresses = util_smart_calloc_s(sizeof(image_progress_progresses_element *), progress_size); ++ if (result->progresses == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ ++ if (!progress_status_map_lock(status_store)) { ++ WARN("Cannot itorate progress status map for locking failed"); ++ // ignore lock error, retry lock after delay. ++ return 0; ++ } ++ ++ map_itor *itor = map_itor_new(status_store->map); ++ for (i = 0; map_itor_valid(itor) && i < progress_size; map_itor_next(itor), i++) { ++ void *id = map_itor_key(itor); ++ const progress *value = (progress *)map_itor_value(itor); ++ const int ID_LEN = 12; // The last 12 charactos of image digest. ++ ++ result->progresses[i] = util_common_calloc_s(sizeof(image_progress_progresses_element)); ++ if (result->progresses[i] == NULL) { ++ // ignore error, return got progress data ++ WARN("Out of memory"); ++ break; ++ } ++ result->progresses[i]->id = util_strdup_s((char *)id + strlen((char *)id) - ID_LEN); ++ result->progresses[i]->total = value->dltotal; ++ result->progresses[i]->current = value->dlnow; ++ result->progresses_len++; ++ } ++ map_itor_free(itor); ++ progress_status_map_unlock(status_store); ++ ++ return 0; ++} ++ + void *get_progress_status(void *arg) + { + status_arg *status = (status_arg *)arg; +- const int delay = 100; // Sleep for 100 milliseconds +- bool write_ok = false; ++ ++ prctl(PR_SET_NAME, "PullProgress"); + + if (status == NULL || status->status_store == NULL || status->stream == NULL) { + ERROR("Get progress status condition error"); + return NULL; + } + +- for (;;) { +- int i = 0; +- +- usleep(delay * 1000); // Sleep for 100 milliseconds ++ while (!status->should_terminal || status->image != NULL) { ++ bool write_ok = false; ++ image_progress *iprogresses = NULL; + +- if (status->should_terminal && status->image == NULL) { ++ // Step 1: delay 100ms, wait progress update ++ util_usleep_nointerupt(100 * 1000); ++ ++ // Step 2: check client whether is canceled? ++ if (status->stream->is_cancelled(status->stream->context)) { ++ WARN("pull stream is cancelled"); + break; + } +- +- image_progress *progresses; +- size_t progress_size = progress_status_map_size(status->status_store); + +- progresses = util_common_calloc_s(sizeof(image_progress)); +- if (progresses == NULL) { +- ERROR("Out of memory. Skip progress show."); +- break; ++ iprogresses = util_common_calloc_s(sizeof(image_progress)); ++ if (iprogresses == NULL) { ++ ERROR("Out of memory"); ++ break; + } +- +- progresses->progresses = util_smart_calloc_s(sizeof(image_progress_progresses_element *), progress_size); +- if (progresses->progresses == NULL) { +- ERROR("Out of memory. Skip progress show."); +- goto roundend; ++ // Step 3: get progress of pull from progress status store ++ if (do_get_progress_from_store(status->status_store, iprogresses) != 0) { ++ free_image_progress(iprogresses); ++ break; + } ++ ++ // Step 4: check main thread whether is finished, and setted pulled image info + if (status->image != NULL) { +- progresses->image = util_strdup_s(status->image_name); ++ iprogresses->image = util_strdup_s(status->image_name); + status->image = NULL; + } + +- if (!progress_status_map_lock(status->status_store)) { +- ERROR("Cannot itorate progress status map for locking failed"); +- goto roundend; +- } +- map_itor *itor = map_itor_new(status->status_store->map); +- for (i = 0; map_itor_valid(itor) && i < progress_size; map_itor_next(itor), i++) { +- void *id = map_itor_key(itor); +- const progress *value = (progress *)map_itor_value(itor); +- const int ID_LEN = 12; // The last 12 charactos of image digest. +- +- progresses->progresses[i] = util_common_calloc_s(sizeof(image_progress_progresses_element)); +- if (progresses->progresses[i] == NULL) { +- WARN("Out of memory. Skip progress show."); +- map_itor_free(itor); +- progress_status_map_unlock(status->status_store); +- goto roundend; +- } +- progresses->progresses[i]->id = util_strdup_s((char *)id + strlen((char *)id) - ID_LEN); +- progresses->progresses[i]->total = value->dltotal; +- progresses->progresses[i]->current = value->dlnow; +- progresses->progresses_len++; ++ // Step 5: send got progress of pull to client ++ write_ok = status->stream->write_func(status->stream->writer, iprogresses); ++ if (!write_ok) { ++ WARN("Send progress data to client failed, just ignore and retry it"); + } +- map_itor_free(itor); +- progress_status_map_unlock(status->status_store); +- +- /* send to client */ +- write_ok = status->stream->write_func(status->stream->writer, progresses); +- if (write_ok) { +- goto roundend; +- } +- if (status->stream->is_cancelled(status->stream->context)) { +- ERROR("pull stream is cancelled"); +- goto roundend; +- } +- ERROR("Send progress data to client failed"); +-roundend: +- free_image_progress(progresses); ++ free_image_progress(iprogresses); + } ++ + return NULL; + } + +@@ -286,7 +299,7 @@ int oci_do_pull_image(const im_pull_request *request, stream_func_wrapper *strea + if (request->is_progress_visible && stream != NULL) { + progress_status_store = progress_status_map_new(); + if (progress_status_store == NULL) { +- ERROR("Out of memory and will not show the pull progress"); ++ ERROR("Out of memory"); + isulad_set_error_message("Failed to pull image %s with error: out of memory", request->image); + ret = -1; + goto out; +@@ -321,21 +334,28 @@ int oci_do_pull_image(const im_pull_request *request, stream_func_wrapper *strea + arg.image = image; + arg.image_name = dest_image_name; + if (!request->is_progress_visible && stream != NULL) { +- image_progress *progresses; ++ image_progress *progresses = NULL; ++ bool nret = false; + + progresses = util_common_calloc_s(sizeof(image_progress)); + if (progresses == NULL) { +- ERROR("Out of memory. Skip progress show."); +- goto out; ++ ERROR("Out of memory"); ++ isulad_set_error_message("Failed to pull image %s with error: out of memory", request->image); ++ ret = -1; ++ goto out; + } + progresses->image = util_strdup_s(dest_image_name); +- if (stream->write_func(stream->writer, progresses)) { ++ nret = stream->write_func(stream->writer, progresses); ++ free_image_progress(progresses); ++ if (!nret) { + ERROR("Send progress data to client failed"); ++ isulad_set_error_message("Failed to pull image %s with error: send progress data to client failed", request->image); ++ ret = -1; + goto out; + } + } + response->image_ref = util_strdup_s(image->id); +- ++ + out: + arg.should_terminal = true; + if (tid != 0 && pthread_join(tid, NULL) != 0) { +diff --git a/src/daemon/modules/image/oci/progress.c b/src/daemon/modules/image/oci/progress.c +index 110f22c0..7d0c10a4 100644 +--- a/src/daemon/modules/image/oci/progress.c ++++ b/src/daemon/modules/image/oci/progress.c +@@ -34,15 +34,16 @@ size_t progress_status_map_size(progress_status_map *progress_status_map) + } + ret = map_size(progress_status_map->map); + progress_status_map_unlock(progress_status_map); +- ++ + return ret; + } + +-bool progress_status_map_insert(progress_status_map *progress_status_map, char *key, progress *value) ++bool progress_status_map_udpate(progress_status_map *progress_status_map, char *key, int64_t current, int64_t total) + { + bool ret = false; ++ progress *pval = NULL; + +- if (progress_status_map == NULL || key == NULL || value == NULL) { ++ if (progress_status_map == NULL || key == NULL) { + ERROR("Invalid parameter"); + return false; + } +@@ -51,9 +52,26 @@ bool progress_status_map_insert(progress_status_map *progress_status_map, char * + ERROR("Cannot replace the progress status map item for locking failed"); + return false; + } +- ret = map_insert(progress_status_map->map, key, value); +- progress_status_map_unlock(progress_status_map); + ++ // If the item exists, only replace the value. ++ pval = map_search(progress_status_map->map, key); ++ if (pval != NULL) { ++ pval->dlnow = current; ++ pval->dltotal = total; ++ progress_status_map_unlock(progress_status_map); ++ return true; ++ } ++ pval = util_common_calloc_s(sizeof(progress)); ++ if (pval == NULL) { ++ ERROR("Out of memory"); ++ progress_status_map_unlock(progress_status_map); ++ return false; ++ } ++ pval->dlnow = current; ++ pval->dltotal = total; ++ ++ ret = map_insert(progress_status_map->map, key, pval); ++ progress_status_map_unlock(progress_status_map); + return ret; + } + +diff --git a/src/daemon/modules/image/oci/progress.h b/src/daemon/modules/image/oci/progress.h +index 496a32f3..dcc8e144 100644 +--- a/src/daemon/modules/image/oci/progress.h ++++ b/src/daemon/modules/image/oci/progress.h +@@ -29,11 +29,11 @@ typedef struct progress_status_map { + } progress_status_map; + + typedef struct progress { +- int64_t dlnow; +- int64_t dltotal; ++ int64_t dlnow; ++ int64_t dltotal; + } progress; + +-bool progress_status_map_insert(progress_status_map *progress_status_map, char *key, progress *value); ++bool progress_status_map_udpate(progress_status_map *progress_status_map, char *key, int64_t current, int64_t total); + + progress_status_map *progress_status_map_new(); + +diff --git a/src/daemon/modules/image/oci/registry/http_request.c b/src/daemon/modules/image/oci/registry/http_request.c +index 748c9a9b..450fbc41 100644 +--- a/src/daemon/modules/image/oci/registry/http_request.c ++++ b/src/daemon/modules/image/oci/registry/http_request.c +@@ -692,44 +692,22 @@ out: + static int xfer_inner(void *p, int64_t dltotal, int64_t dlnow, int64_t ultotal, int64_t ulnow) + { + progress_arg *arg = (progress_arg *)p; +- progress *progress_value = NULL; + + if (arg == NULL || arg->map_store == NULL) { + ERROR("Wrong progress arg"); + return -1; + } ++ + // When fetch_manifest_list, there's no digest. It's not a layer pulling progress and skip it. + if (arg->digest == NULL) { + return 0; + } + +- if (!progress_status_map_lock(arg->map_store)) { +- ERROR("Cannot update progress status map for locking failed"); ++ if (!progress_status_map_udpate(arg->map_store, arg->digest, dlnow, dltotal)) { ++ ERROR("Failed to update pull progress"); + return -1; + } + +- // If the item exists, only replace the value. +- progress_value = map_search(arg->map_store->map, arg->digest); +- if (progress_value != NULL) { +- progress_value->dlnow = dlnow; +- progress_value->dltotal = dltotal; +- progress_status_map_unlock(arg->map_store); +- +- return 0; +- } +- progress_status_map_unlock(arg->map_store); +- +- progress_value = util_common_calloc_s(sizeof(progress)); +- if (progress_value == NULL) { +- ERROR("Out of memory"); +- return -1; +- } +- +- progress_value->dlnow = dlnow; +- progress_value->dltotal = dltotal; +- +- progress_status_map_insert(arg->map_store, arg->digest, progress_value); +- + return 0; + } + +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.c b/src/daemon/modules/image/oci/storage/image_store/image_store.c +index f49f4707..58baa47a 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_store.c ++++ b/src/daemon/modules/image/oci/storage/image_store/image_store.c +@@ -2824,7 +2824,7 @@ static int implicit_digest(map_t *digests, image_t *img) + + // Find whether the manifest in big_data_digests exists, if not, return 0 directly + if (!get_index_by_key((const char **)img->simage->big_data_digests->keys, img->simage->big_data_digests->len, +- IMAGE_DIGEST_BIG_DATA_KEY, &index)) { ++ IMAGE_DIGEST_BIG_DATA_KEY, &index)) { + return 0; + } + +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c +index 7517dd43..3bc433ae 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c +@@ -1930,7 +1930,7 @@ int overlay2_apply_diff(const char *id, const struct graphdriver *driver, const + goto out; + } + +- ret = archive_unpack(content, layer_diff, &options, root_dir ,&err); ++ ret = archive_unpack(content, layer_diff, &options, root_dir, &err); + if (ret != 0) { + ERROR("Failed to unpack to %s: %s", layer_diff, err); + ret = -1; +diff --git a/src/daemon/modules/image/oci/storage/storage.c b/src/daemon/modules/image/oci/storage/storage.c +index 2e53dbac..0d1a846a 100644 +--- a/src/daemon/modules/image/oci/storage/storage.c ++++ b/src/daemon/modules/image/oci/storage/storage.c +@@ -215,7 +215,7 @@ int storage_inc_hold_refs(const char *layer_id) + int storage_dec_hold_refs(const char *layer_id) + { + int ret = 0; +- ++ + if (layer_id == NULL) { + ERROR("Empty layer id"); + return -1; +@@ -550,7 +550,8 @@ char *storage_img_get_image_id(const char *img_name) + return image_store_lookup(img_name); + } + +-static bool is_top_layer_of_other_image(const char *img_id, const imagetool_images_list *all_images, const char *layer_id) ++static bool is_top_layer_of_other_image(const char *img_id, const imagetool_images_list *all_images, ++ const char *layer_id) + { + size_t i = 0; + +-- +2.42.0 + diff --git a/0028-2230-format-code.patch b/0028-2230-format-code.patch new file mode 100644 index 0000000..e1fb272 --- /dev/null +++ b/0028-2230-format-code.patch @@ -0,0 +1,3476 @@ +From c91b4d223fa590e2d5164b42e12a9bca319831e5 Mon Sep 17 00:00:00 2001 +From: haozi007 <liuhao27@huawei.com> +Date: Thu, 23 Nov 2023 12:31:47 +0000 +Subject: [PATCH 28/64] !2230 format code * format code tools/static_check.sh + +--- + src/daemon/common/cri/v1/v1_cri_helpers.cc | 8 +- + src/daemon/common/cri/v1/v1_cri_helpers.h | 6 +- + .../common/cri/v1/v1_cri_security_context.cc | 6 +- + .../entry/connect/grpc/cri/cri_service.cc | 2 +- + .../cri/v1/cri_v1_runtime_image_service.cc | 20 +- + .../cri/v1/cri_v1_runtime_runtime_service.cc | 91 +-- + .../v1alpha/cri_runtime_runtime_service.cc | 3 +- + src/daemon/entry/connect/grpc/grpc_service.cc | 2 +- + src/daemon/entry/cri/network_plugin.cc | 4 +- + .../v1/v1_cri_container_manager_service.cc | 10 +- + .../v1/v1_cri_image_manager_service_impl.cc | 6 +- + .../v1/v1_cri_image_manager_service_impl.h | 2 +- + .../v1/v1_cri_pod_sandbox_manager_service.cc | 15 +- + .../v1alpha/cri_container_manager_service.cc | 3 +- + .../cri_pod_sandbox_manager_service.cc | 2 +- + .../entry/cri/v1alpha/cri_security_context.cc | 6 +- + .../entry/cri/v1alpha/v1alpha_cri_helpers.cc | 2 +- + .../executor/container_cb/execution_create.c | 8 +- + .../container_cb/execution_information.c | 5 +- + src/daemon/modules/api/container_api.h | 3 +- + .../modules/container/restore/restore.c | 3 +- + .../cni_operator/libcni/invoke/libcni_exec.c | 2 +- + .../network/cni_operator/libcni/libcni_api.c | 9 +- + .../network/cni_operator/libcni/libcni_api.h | 2 +- + .../cni_operator/libcni/libcni_cached.c | 2 +- + .../cni_operator/libcni/libcni_result_type.c | 2 +- + .../modules/runtime/isula/isula_rt_ops.c | 2 +- + .../modules/runtime/shim/shim_rt_monitor.cc | 2 +- + src/daemon/modules/runtime/shim/shim_rt_ops.c | 2 +- + .../modules/service/service_container.c | 3 +- + .../modules/service/vsock_io_handler.cc | 7 +- + src/daemon/modules/service/vsock_io_handler.h | 6 +- + .../sandboxer/client/grpc_async_wait_call.cc | 11 +- + .../sandboxer/client/grpc_async_wait_call.h | 18 +- + .../sandboxer/client/grpc_sandboxer_client.cc | 56 +- + .../sandboxer/client/grpc_sandboxer_client.h | 6 +- + .../client/grpc_sandboxer_monitor.cc | 10 +- + .../sandboxer/client/grpc_sandboxer_monitor.h | 2 +- + .../controller/shim/shim_controller.cc | 7 +- + src/daemon/sandbox/sandbox.cc | 3 +- + src/daemon/sandbox/sandbox_manager.cc | 3 +- + src/daemon/sandbox/sandbox_manager.h | 3 +- + src/daemon/sandbox/sandbox_ops.cc | 2 +- + src/daemon/sandbox/sandbox_ops.h | 3 +- + src/utils/cpputils/url.cc | 5 +- + src/utils/cutils/error.h | 2 +- + src/utils/cutils/map/map.h | 8 - + src/utils/tar/isulad_tar.c | 9 +- + src/utils/tar/isulad_tar.h | 3 +- + test/cutils/utils_file/utils_file_ut.cc | 2 +- + test/mocks/callback_mock.cc | 8 +- + test/mocks/controller_stub_mock.cc | 516 +++++++++++------- + test/mocks/controller_stub_mock.h | 259 +++++++-- + test/mocks/grpc_async_wait_call_mock.cc | 3 +- + test/mocks/grpc_async_wait_call_mock.h | 2 +- + test/mocks/grpc_sandboxer_client_mock.cc | 15 +- + test/mocks/grpc_sandboxer_client_mock.h | 12 +- + test/mocks/grpc_sandboxer_monitor_mock.cc | 2 +- + test/mocks/isulad_config_mock.h | 2 +- + test/mocks/sandbox_mock.cc | 2 +- + test/mocks/sandbox_mock.h | 2 +- + test/mocks/service_container_api_mock.h | 2 +- + test/mocks/shim_controller_mock.cc | 6 +- + test/mocks/shim_controller_mock.h | 8 +- + test/network/cni_operate/cni_operate_ut.cc | 40 +- + test/network/network_mock.cc | 2 +- + test/sandbox/controller/controller_common.cc | 16 +- + test/sandbox/controller/controller_common.h | 3 +- + .../manager/controller_manager_ut.cc | 27 +- + .../async_wait_call/async_wait_call_ut.cc | 9 +- + .../async_wait_call/dummy_monitor_utils.h | 44 +- + .../sandboxer_client/sandboxer_client_ut.cc | 135 +++-- + .../sandboxer_controller_ut.cc | 18 +- + .../controller/shim/shim_controller_ut.cc | 6 +- + 74 files changed, 963 insertions(+), 575 deletions(-) + +diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.cc b/src/daemon/common/cri/v1/v1_cri_helpers.cc +index 4f2660e8..be06eb0a 100644 +--- a/src/daemon/common/cri/v1/v1_cri_helpers.cc ++++ b/src/daemon/common/cri/v1/v1_cri_helpers.cc +@@ -391,7 +391,7 @@ auto GetSecurityOpts(const commonSecurityContext &context, const char &separator + } + + std::vector<std::string> selinuxOpts = CRIHelpersV1::GetSELinuxLabelOpts(context.hasSELinuxOption, +- context.selinuxOption, separator, error); ++ context.selinuxOption, separator, error); + if (error.NotEmpty()) { + error.Errorf("Failed to generate SELinuxLabel options for container %s", error.GetMessage().c_str()); + return securityOpts; +@@ -459,7 +459,8 @@ void AddSecurityOptsToHostConfig(std::vector<std::string> &securityOpts, host_co + } + + } +-void GetContainerSandboxID(const std::string &containerID, std::string &realContainerID, std::string &sandboxID, Errors &error) ++void GetContainerSandboxID(const std::string &containerID, std::string &realContainerID, std::string &sandboxID, ++ Errors &error) + { + std::string PodID; + container_inspect *info = CRIHelpers::InspectContainer(containerID, error, false); +@@ -526,7 +527,8 @@ out: + return sandboxer; + } + +-void ApplySandboxSecurityContextToHostConfig(const runtime::v1::LinuxSandboxSecurityContext &context, host_config *hc, Errors &error) ++void ApplySandboxSecurityContextToHostConfig(const runtime::v1::LinuxSandboxSecurityContext &context, host_config *hc, ++ Errors &error) + { + if (hc == nullptr) { + ERROR("Invalid input arguments: empty hostconfig"); +diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.h b/src/daemon/common/cri/v1/v1_cri_helpers.h +index 0fa1ae91..7085f8d2 100644 +--- a/src/daemon/common/cri/v1/v1_cri_helpers.h ++++ b/src/daemon/common/cri/v1/v1_cri_helpers.h +@@ -73,11 +73,13 @@ auto GetPodSELinuxLabelOpts(const std::string &selinuxLabel, Errors &error) + + void AddSecurityOptsToHostConfig(std::vector<std::string> &securityOpts, host_config *hostconfig, Errors &error); + +-void GetContainerSandboxID(const std::string &containerID, std::string &realContainerID, std::string &sandboxID, Errors &error); ++void GetContainerSandboxID(const std::string &containerID, std::string &realContainerID, std::string &sandboxID, ++ Errors &error); + + std::string CRISandboxerConvert(const std::string &runtime); + +-void ApplySandboxSecurityContextToHostConfig(const runtime::v1::LinuxSandboxSecurityContext &context, host_config *hc, Errors &error); ++void ApplySandboxSecurityContextToHostConfig(const runtime::v1::LinuxSandboxSecurityContext &context, host_config *hc, ++ Errors &error); + + }; // namespace CRIHelpers + +diff --git a/src/daemon/common/cri/v1/v1_cri_security_context.cc b/src/daemon/common/cri/v1/v1_cri_security_context.cc +index 930710e0..7cdbf5fc 100644 +--- a/src/daemon/common/cri/v1/v1_cri_security_context.cc ++++ b/src/daemon/common/cri/v1/v1_cri_security_context.cc +@@ -19,7 +19,8 @@ + #include <memory> + + namespace CRISecurityV1 { +-static void ModifyContainerConfig(const runtime::v1::LinuxContainerSecurityContext &sc, container_config *config, Errors &error) ++static void ModifyContainerConfig(const runtime::v1::LinuxContainerSecurityContext &sc, container_config *config, ++ Errors &error) + { + // none -> ""; username -> username; username, uid -> username; username, uid, gid -> username:gid; + // username, gid -> username:gid; uid -> uid; uid, gid -> uid:gid; gid -> error +@@ -157,7 +158,8 @@ static void ApplyMaskedPathsToHostConfig(const runtime::v1::LinuxContainerSecuri + } + } + +-static void ApplyReadonlyPathsToHostConfig(const runtime::v1::LinuxContainerSecurityContext &sc, host_config *hostConfig, ++static void ApplyReadonlyPathsToHostConfig(const runtime::v1::LinuxContainerSecurityContext &sc, ++ host_config *hostConfig, + Errors &error) + { + if (sc.readonly_paths_size() <= 0) { +diff --git a/src/daemon/entry/connect/grpc/cri/cri_service.cc b/src/daemon/entry/connect/grpc/cri/cri_service.cc +index e33a6d34..c1986c44 100644 +--- a/src/daemon/entry/connect/grpc/cri/cri_service.cc ++++ b/src/daemon/entry/connect/grpc/cri/cri_service.cc +@@ -50,7 +50,7 @@ int CRIService::Init(const isulad_daemon_configs *config) + } + + Errors err; +- /* note: get config from args, now use defaults */ ++ /* note: get config from args, now use defaults */ + Network::NetworkPluginConf mConf; + + if (config != nullptr) { +diff --git a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_image_service.cc b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_image_service.cc +index ac62b6eb..09c8958d 100644 +--- a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_image_service.cc ++++ b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_image_service.cc +@@ -28,8 +28,8 @@ RuntimeV1ImageServiceImpl::RuntimeV1ImageServiceImpl() + } + + grpc::Status RuntimeV1ImageServiceImpl::PullImage(grpc::ServerContext *context, +- const runtime::v1::PullImageRequest *request, +- runtime::v1::PullImageResponse *reply) ++ const runtime::v1::PullImageRequest *request, ++ runtime::v1::PullImageResponse *reply) + { + Errors error; + +@@ -53,8 +53,8 @@ grpc::Status RuntimeV1ImageServiceImpl::PullImage(grpc::ServerContext *context, + } + + grpc::Status RuntimeV1ImageServiceImpl::ListImages(grpc::ServerContext *context, +- const runtime::v1::ListImagesRequest *request, +- runtime::v1::ListImagesResponse *reply) ++ const runtime::v1::ListImagesRequest *request, ++ runtime::v1::ListImagesResponse *reply) + { + std::vector<std::unique_ptr<runtime::v1::Image>> images; + Errors error; +@@ -86,8 +86,8 @@ grpc::Status RuntimeV1ImageServiceImpl::ListImages(grpc::ServerContext *context, + } + + grpc::Status RuntimeV1ImageServiceImpl::ImageStatus(grpc::ServerContext *context, +- const runtime::v1::ImageStatusRequest *request, +- runtime::v1::ImageStatusResponse *reply) ++ const runtime::v1::ImageStatusRequest *request, ++ runtime::v1::ImageStatusResponse *reply) + { + std::unique_ptr<runtime::v1::Image> image_info = nullptr; + Errors error; +@@ -117,8 +117,8 @@ grpc::Status RuntimeV1ImageServiceImpl::ImageStatus(grpc::ServerContext *context + } + + grpc::Status RuntimeV1ImageServiceImpl::ImageFsInfo(grpc::ServerContext *context, +- const runtime::v1::ImageFsInfoRequest *request, +- runtime::v1::ImageFsInfoResponse *reply) ++ const runtime::v1::ImageFsInfoRequest *request, ++ runtime::v1::ImageFsInfoResponse *reply) + { + std::vector<std::unique_ptr<runtime::v1::FilesystemUsage>> usages; + Errors error; +@@ -150,8 +150,8 @@ grpc::Status RuntimeV1ImageServiceImpl::ImageFsInfo(grpc::ServerContext *context + } + + grpc::Status RuntimeV1ImageServiceImpl::RemoveImage(grpc::ServerContext *context, +- const runtime::v1::RemoveImageRequest *request, +- runtime::v1::RemoveImageResponse *reply) ++ const runtime::v1::RemoveImageRequest *request, ++ runtime::v1::RemoveImageResponse *reply) + { + Errors error; + +diff --git a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc +index 1db79307..ba9459f6 100644 +--- a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc ++++ b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc +@@ -25,7 +25,8 @@ + + using namespace CRIV1; + +-void RuntimeV1RuntimeServiceImpl::Init(std::string &podSandboxImage, std::shared_ptr<Network::PluginManager> networkPlugin, Errors &err) ++void RuntimeV1RuntimeServiceImpl::Init(std::string &podSandboxImage, ++ std::shared_ptr<Network::PluginManager> networkPlugin, Errors &err) + { + // Assembly implementation for CRIRuntimeServiceImpl + service_executor_t *cb = get_service_executor(); +@@ -47,8 +48,8 @@ void RuntimeV1RuntimeServiceImpl::Shutdown() + } + + grpc::Status RuntimeV1RuntimeServiceImpl::Version(grpc::ServerContext *context, +- const runtime::v1::VersionRequest *request, +- runtime::v1::VersionResponse *reply) ++ const runtime::v1::VersionRequest *request, ++ runtime::v1::VersionResponse *reply) + { + Errors error; + if (request == nullptr || reply == nullptr) { +@@ -65,8 +66,8 @@ grpc::Status RuntimeV1RuntimeServiceImpl::Version(grpc::ServerContext *context, + } + + grpc::Status RuntimeV1RuntimeServiceImpl::CreateContainer(grpc::ServerContext *context, +- const runtime::v1::CreateContainerRequest *request, +- runtime::v1::CreateContainerResponse *reply) ++ const runtime::v1::CreateContainerRequest *request, ++ runtime::v1::CreateContainerResponse *reply) + { + Errors error; + +@@ -91,8 +92,8 @@ grpc::Status RuntimeV1RuntimeServiceImpl::CreateContainer(grpc::ServerContext *c + } + + grpc::Status RuntimeV1RuntimeServiceImpl::StartContainer(grpc::ServerContext *context, +- const runtime::v1::StartContainerRequest *request, +- runtime::v1::StartContainerResponse *reply) ++ const runtime::v1::StartContainerRequest *request, ++ runtime::v1::StartContainerResponse *reply) + { + Errors error; + +@@ -115,8 +116,8 @@ grpc::Status RuntimeV1RuntimeServiceImpl::StartContainer(grpc::ServerContext *co + } + + grpc::Status RuntimeV1RuntimeServiceImpl::StopContainer(grpc::ServerContext *context, +- const runtime::v1::StopContainerRequest *request, +- runtime::v1::StopContainerResponse *reply) ++ const runtime::v1::StopContainerRequest *request, ++ runtime::v1::StopContainerResponse *reply) + { + Errors error; + +@@ -139,8 +140,8 @@ grpc::Status RuntimeV1RuntimeServiceImpl::StopContainer(grpc::ServerContext *con + } + + grpc::Status RuntimeV1RuntimeServiceImpl::RemoveContainer(grpc::ServerContext *context, +- const runtime::v1::RemoveContainerRequest *request, +- runtime::v1::RemoveContainerResponse *reply) ++ const runtime::v1::RemoveContainerRequest *request, ++ runtime::v1::RemoveContainerResponse *reply) + { + Errors error; + +@@ -163,8 +164,8 @@ grpc::Status RuntimeV1RuntimeServiceImpl::RemoveContainer(grpc::ServerContext *c + } + + grpc::Status RuntimeV1RuntimeServiceImpl::ListContainers(grpc::ServerContext *context, +- const runtime::v1::ListContainersRequest *request, +- runtime::v1::ListContainersResponse *reply) ++ const runtime::v1::ListContainersRequest *request, ++ runtime::v1::ListContainersResponse *reply) + { + Errors error; + +@@ -197,8 +198,8 @@ grpc::Status RuntimeV1RuntimeServiceImpl::ListContainers(grpc::ServerContext *co + } + + grpc::Status RuntimeV1RuntimeServiceImpl::ContainerStats(grpc::ServerContext *context, +- const runtime::v1::ContainerStatsRequest *request, +- runtime::v1::ContainerStatsResponse *reply) ++ const runtime::v1::ContainerStatsRequest *request, ++ runtime::v1::ContainerStatsResponse *reply) + { + Errors error; + +@@ -223,8 +224,8 @@ grpc::Status RuntimeV1RuntimeServiceImpl::ContainerStats(grpc::ServerContext *co + } + + grpc::Status RuntimeV1RuntimeServiceImpl::ListContainerStats(grpc::ServerContext *context, +- const runtime::v1::ListContainerStatsRequest *request, +- runtime::v1::ListContainerStatsResponse *reply) ++ const runtime::v1::ListContainerStatsRequest *request, ++ runtime::v1::ListContainerStatsResponse *reply) + { + Errors error; + +@@ -257,8 +258,8 @@ grpc::Status RuntimeV1RuntimeServiceImpl::ListContainerStats(grpc::ServerContext + } + + grpc::Status RuntimeV1RuntimeServiceImpl::ContainerStatus(grpc::ServerContext *context, +- const runtime::v1::ContainerStatusRequest *request, +- runtime::v1::ContainerStatusResponse *reply) ++ const runtime::v1::ContainerStatusRequest *request, ++ runtime::v1::ContainerStatusResponse *reply) + { + Errors error; + +@@ -283,8 +284,8 @@ grpc::Status RuntimeV1RuntimeServiceImpl::ContainerStatus(grpc::ServerContext *c + } + + grpc::Status RuntimeV1RuntimeServiceImpl::ExecSync(grpc::ServerContext *context, +- const runtime::v1::ExecSyncRequest *request, +- runtime::v1::ExecSyncResponse *reply) ++ const runtime::v1::ExecSyncRequest *request, ++ runtime::v1::ExecSyncResponse *reply) + { + Errors error; + +@@ -307,8 +308,8 @@ grpc::Status RuntimeV1RuntimeServiceImpl::ExecSync(grpc::ServerContext *context, + } + + grpc::Status RuntimeV1RuntimeServiceImpl::RunPodSandbox(grpc::ServerContext *context, +- const runtime::v1::RunPodSandboxRequest *request, +- runtime::v1::RunPodSandboxResponse *reply) ++ const runtime::v1::RunPodSandboxRequest *request, ++ runtime::v1::RunPodSandboxResponse *reply) + { + Errors error; + +@@ -335,8 +336,8 @@ grpc::Status RuntimeV1RuntimeServiceImpl::RunPodSandbox(grpc::ServerContext *con + } + + grpc::Status RuntimeV1RuntimeServiceImpl::StopPodSandbox(grpc::ServerContext *context, +- const runtime::v1::StopPodSandboxRequest *request, +- runtime::v1::StopPodSandboxResponse *reply) ++ const runtime::v1::StopPodSandboxRequest *request, ++ runtime::v1::StopPodSandboxResponse *reply) + { + Errors error; + +@@ -360,8 +361,8 @@ grpc::Status RuntimeV1RuntimeServiceImpl::StopPodSandbox(grpc::ServerContext *co + } + + grpc::Status RuntimeV1RuntimeServiceImpl::RemovePodSandbox(grpc::ServerContext *context, +- const runtime::v1::RemovePodSandboxRequest *request, +- runtime::v1::RemovePodSandboxResponse *reply) ++ const runtime::v1::RemovePodSandboxRequest *request, ++ runtime::v1::RemovePodSandboxResponse *reply) + { + Errors error; + +@@ -385,8 +386,8 @@ grpc::Status RuntimeV1RuntimeServiceImpl::RemovePodSandbox(grpc::ServerContext * + } + + grpc::Status RuntimeV1RuntimeServiceImpl::PodSandboxStatus(grpc::ServerContext *context, +- const runtime::v1::PodSandboxStatusRequest *request, +- runtime::v1::PodSandboxStatusResponse *reply) ++ const runtime::v1::PodSandboxStatusRequest *request, ++ runtime::v1::PodSandboxStatusResponse *reply) + { + Errors error; + +@@ -412,8 +413,8 @@ grpc::Status RuntimeV1RuntimeServiceImpl::PodSandboxStatus(grpc::ServerContext * + } + + grpc::Status RuntimeV1RuntimeServiceImpl::ListPodSandbox(grpc::ServerContext *context, +- const runtime::v1::ListPodSandboxRequest *request, +- runtime::v1::ListPodSandboxResponse *reply) ++ const runtime::v1::ListPodSandboxRequest *request, ++ runtime::v1::ListPodSandboxResponse *reply) + { + Errors error; + +@@ -446,8 +447,8 @@ grpc::Status RuntimeV1RuntimeServiceImpl::ListPodSandbox(grpc::ServerContext *co + } + + grpc::Status RuntimeV1RuntimeServiceImpl::PodSandboxStats(grpc::ServerContext *context, +- const runtime::v1::PodSandboxStatsRequest *request, +- runtime::v1::PodSandboxStatsResponse *reply) ++ const runtime::v1::PodSandboxStatsRequest *request, ++ runtime::v1::PodSandboxStatsResponse *reply) + { + Errors error; + +@@ -474,8 +475,8 @@ grpc::Status RuntimeV1RuntimeServiceImpl::PodSandboxStats(grpc::ServerContext *c + + grpc::Status + RuntimeV1RuntimeServiceImpl::ListPodSandboxStats(grpc::ServerContext *context, +- const runtime::v1::ListPodSandboxStatsRequest *request, +- runtime::v1::ListPodSandboxStatsResponse *reply) ++ const runtime::v1::ListPodSandboxStatsRequest *request, ++ runtime::v1::ListPodSandboxStatsResponse *reply) + { + Errors error; + +@@ -508,8 +509,8 @@ RuntimeV1RuntimeServiceImpl::ListPodSandboxStats(grpc::ServerContext *context, + + grpc::Status + RuntimeV1RuntimeServiceImpl::UpdateContainerResources(grpc::ServerContext *context, +- const runtime::v1::UpdateContainerResourcesRequest *request, +- runtime::v1::UpdateContainerResourcesResponse *reply) ++ const runtime::v1::UpdateContainerResourcesRequest *request, ++ runtime::v1::UpdateContainerResourcesResponse *reply) + { + Errors error; + +@@ -533,8 +534,8 @@ RuntimeV1RuntimeServiceImpl::UpdateContainerResources(grpc::ServerContext *conte + } + + grpc::Status RuntimeV1RuntimeServiceImpl::Exec(grpc::ServerContext *context, +- const runtime::v1::ExecRequest *request, +- runtime::v1::ExecResponse *response) ++ const runtime::v1::ExecRequest *request, ++ runtime::v1::ExecResponse *response) + { + Errors error; + +@@ -558,8 +559,8 @@ grpc::Status RuntimeV1RuntimeServiceImpl::Exec(grpc::ServerContext *context, + } + + grpc::Status RuntimeV1RuntimeServiceImpl::Attach(grpc::ServerContext *context, +- const runtime::v1::AttachRequest *request, +- runtime::v1::AttachResponse *response) ++ const runtime::v1::AttachRequest *request, ++ runtime::v1::AttachResponse *response) + { + Errors error; + +@@ -584,8 +585,8 @@ grpc::Status RuntimeV1RuntimeServiceImpl::Attach(grpc::ServerContext *context, + + grpc::Status + RuntimeV1RuntimeServiceImpl::UpdateRuntimeConfig(grpc::ServerContext *context, +- const runtime::v1::UpdateRuntimeConfigRequest *request, +- runtime::v1::UpdateRuntimeConfigResponse *reply) ++ const runtime::v1::UpdateRuntimeConfigRequest *request, ++ runtime::v1::UpdateRuntimeConfigResponse *reply) + { + Errors error; + +@@ -608,8 +609,8 @@ RuntimeV1RuntimeServiceImpl::UpdateRuntimeConfig(grpc::ServerContext *context, + } + + grpc::Status RuntimeV1RuntimeServiceImpl::Status(grpc::ServerContext *context, +- const runtime::v1::StatusRequest *request, +- runtime::v1::StatusResponse *reply) ++ const runtime::v1::StatusRequest *request, ++ runtime::v1::StatusResponse *reply) + { + Errors error; + +diff --git a/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.cc b/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.cc +index a56b167c..5e85702c 100644 +--- a/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.cc ++++ b/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.cc +@@ -23,7 +23,8 @@ + + using namespace CRI; + +-void RuntimeRuntimeServiceImpl::Init(std::string &podSandboxImage, std::shared_ptr<Network::PluginManager> networkPlugin, Errors &err) ++void RuntimeRuntimeServiceImpl::Init(std::string &podSandboxImage, ++ std::shared_ptr<Network::PluginManager> networkPlugin, Errors &err) + { + // Assembly implementation for CRIRuntimeServiceImpl + service_executor_t *cb = get_service_executor(); +diff --git a/src/daemon/entry/connect/grpc/grpc_service.cc b/src/daemon/entry/connect/grpc/grpc_service.cc +index 4e1ae019..61e284f3 100644 +--- a/src/daemon/entry/connect/grpc/grpc_service.cc ++++ b/src/daemon/entry/connect/grpc/grpc_service.cc +@@ -109,7 +109,7 @@ public: + void Shutdown(void) + { + m_server->Shutdown(); +- ++ + // call CRI to shutdown stream server + m_criService.Shutdown(); + +diff --git a/src/daemon/entry/cri/network_plugin.cc b/src/daemon/entry/cri/network_plugin.cc +index f919a059..f8f9c7e6 100644 +--- a/src/daemon/entry/cri/network_plugin.cc ++++ b/src/daemon/entry/cri/network_plugin.cc +@@ -558,14 +558,14 @@ void NoopNetworkPlugin::SetUpPod(const std::string &ns, const std::string &name, + } + + container_network_settings *network_settings = static_cast<container_network_settings *> +- (util_common_calloc_s(sizeof(container_network_settings))); ++ (util_common_calloc_s(sizeof(container_network_settings))); + if (network_settings == nullptr) { + ERROR("Out of memory"); + error.SetError("Out of memory"); + return; + } + auto settingsWarpper = std::unique_ptr<CStructWrapper<container_network_settings>>(new +- CStructWrapper<container_network_settings>(network_settings, free_container_network_settings)); ++ CStructWrapper<container_network_settings>(network_settings, free_container_network_settings)); + + network_settings->sandbox_key = util_strdup_s(netnsPath.c_str()); + +diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +index f635df2b..067f4210 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +@@ -149,7 +149,7 @@ void ContainerManagerService::DoUsePodLevelSELinuxConfig(const runtime::v1::Cont + auto ContainerManagerService::IsSELinuxLabelEmpty(const ::runtime::v1::SELinuxOption &selinuxOption) -> bool + { + return selinuxOption.user().length() == 0 && selinuxOption.role().length() == 0 && +- selinuxOption.type().length() == 0 && selinuxOption.level().length() == 0; ++ selinuxOption.type().length() == 0 && selinuxOption.level().length() == 0; + } + + auto ContainerManagerService::GenerateCreateContainerHostConfig( +@@ -194,7 +194,7 @@ auto ContainerManagerService::GenerateCreateContainerHostConfig( + DoUsePodLevelSELinuxConfig(containerConfig, hostconfig, sandbox, error); + if (error.NotEmpty()) { + ERROR("Failed to add pod: %s security context to host config for container: %s", +- sandbox.GetName().c_str(), containerConfig.metadata().name().c_str()); ++ sandbox.GetName().c_str(), containerConfig.metadata().name().c_str()); + goto cleanup; + } + } +@@ -539,8 +539,10 @@ void ContainerManagerService::CreateContainerLogSymlink(const std::string &conta + WARN("Deleted previously existing symlink file: %s", path); + } + if (symlink(realPath, path) != 0) { +- SYSERROR("failed to create symbolic link %s to the container log file %s for container %s", path, realPath, containerID.c_str()); +- error.Errorf("failed to create symbolic link %s to the container log file %s for container %s", path, realPath, containerID.c_str()); ++ SYSERROR("failed to create symbolic link %s to the container log file %s for container %s", path, realPath, ++ containerID.c_str()); ++ error.Errorf("failed to create symbolic link %s to the container log file %s for container %s", path, realPath, ++ containerID.c_str()); + goto cleanup; + } + } else { +diff --git a/src/daemon/entry/cri/v1/v1_cri_image_manager_service_impl.cc b/src/daemon/entry/cri/v1/v1_cri_image_manager_service_impl.cc +index 066eed5e..71918706 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_image_manager_service_impl.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_image_manager_service_impl.cc +@@ -202,7 +202,7 @@ auto ImageManagerServiceImpl::status_request_from_grpc(const runtime::v1::ImageS + } + + std::unique_ptr<runtime::v1::Image> ImageManagerServiceImpl::status_image_to_grpc(im_summary_response *response, +- Errors & /*error*/) ++ Errors & /*error*/) + { + imagetool_image_summary *image_info = response->image_summary; + if (image_info == nullptr) { +@@ -220,8 +220,8 @@ std::unique_ptr<runtime::v1::Image> ImageManagerServiceImpl::status_image_to_grp + } + + std::unique_ptr<runtime::v1::Image> ImageManagerServiceImpl::ImageStatus(const runtime::v1::ImageSpec +- &image, +- Errors &error) ++ &image, ++ Errors &error) + { + im_summary_request *request { nullptr }; + im_summary_response *response { nullptr }; +diff --git a/src/daemon/entry/cri/v1/v1_cri_image_manager_service_impl.h b/src/daemon/entry/cri/v1/v1_cri_image_manager_service_impl.h +index 3f13a157..1c276e06 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_image_manager_service_impl.h ++++ b/src/daemon/entry/cri/v1/v1_cri_image_manager_service_impl.h +@@ -33,7 +33,7 @@ public: + std::vector<std::unique_ptr<runtime::v1::Image>> &images, Errors &error) override; + + std::unique_ptr<runtime::v1::Image> ImageStatus(const runtime::v1::ImageSpec &image, +- Errors &error) override; ++ Errors &error) override; + + std::string PullImage(const runtime::v1::ImageSpec &image, const runtime::v1::AuthConfig &auth, + Errors &error) override; +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +index 2c802900..0f6b8508 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +@@ -196,7 +196,8 @@ void PodSandboxManagerService::UpdateSandboxConfig(runtime::v1::PodSandboxConfig + // TODO: Update SecurityContext with default values + } + +-void PodSandboxManagerService::SetupSandboxFiles(const std::string &resolvPath, const runtime::v1::PodSandboxConfig &config, Errors &error) ++void PodSandboxManagerService::SetupSandboxFiles(const std::string &resolvPath, ++ const runtime::v1::PodSandboxConfig &config, Errors &error) + { + if (resolvPath.empty()) { + return; +@@ -423,7 +424,8 @@ void PodSandboxManagerService::ClearCniNetwork(const std::shared_ptr<sandbox::Sa + stdAnnos.insert(std::pair<std::string, std::string>(CRIHelpers::Constants::POD_SANDBOX_KEY, sandboxKey)); + + Errors pluginErr; +- m_pluginManager->TearDownPod(config.metadata().namespace_(), config.metadata().name(), Network::DEFAULT_NETWORK_INTERFACE_NAME, ++ m_pluginManager->TearDownPod(config.metadata().namespace_(), config.metadata().name(), ++ Network::DEFAULT_NETWORK_INTERFACE_NAME, + sandbox->GetId(), stdAnnos, pluginErr); + if (pluginErr.NotEmpty()) { + WARN("TearDownPod cni network failed: %s", pluginErr.GetCMessage()); +@@ -492,7 +494,8 @@ auto PodSandboxManagerService::GetContainerListResponse(const std::string &readS + } + + ret = m_cb->container.list(list_request, &list_response); +- auto list_response_wrapper = makeUniquePtrCStructWrapper<container_list_response>(list_response, free_container_list_response); ++ auto list_response_wrapper = makeUniquePtrCStructWrapper<container_list_response>(list_response, ++ free_container_list_response); + if (list_response_wrapper == nullptr) { + ERROR("Failed to call list container callback"); + errors.push_back("Failed to call list container callback"); +@@ -510,7 +513,7 @@ auto PodSandboxManagerService::GetContainerListResponse(const std::string &readS + } + + return list_response_wrapper; +-} ++} + + auto PodSandboxManagerService::StopAllContainersInSandbox(const std::string &readSandboxID, + Errors &error) -> int +@@ -751,8 +754,8 @@ void PodSandboxManagerService::GetIPs(std::shared_ptr<sandbox::Sandbox> sandbox, + return; + } + +- auto settings = std::unique_ptr<CStructWrapper<container_network_settings>>(new +- CStructWrapper<container_network_settings>(network_settings, free_container_network_settings)); ++ auto settings = std::unique_ptr<CStructWrapper<container_network_settings>>(new ++ CStructWrapper<container_network_settings>(network_settings, free_container_network_settings)); + if (settings == nullptr) { + ERROR("Out of memory"); + return; +diff --git a/src/daemon/entry/cri/v1alpha/cri_container_manager_service.cc b/src/daemon/entry/cri/v1alpha/cri_container_manager_service.cc +index 9da25768..b0abdb52 100644 +--- a/src/daemon/entry/cri/v1alpha/cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/v1alpha/cri_container_manager_service.cc +@@ -547,7 +547,8 @@ void ContainerManagerService::CreateContainerLogSymlink(const std::string &conta + WARN("Deleted previously existing symlink file: %s", path); + } + if (symlink(realPath, path) != 0) { +- SYSERROR("failed to create symbolic link %s to the container log file %s for container %s", path, realPath, containerID.c_str()); ++ SYSERROR("failed to create symbolic link %s to the container log file %s for container %s", path, realPath, ++ containerID.c_str()); + error.Errorf("failed to create symbolic link %s to the container log file %s for container %s: %s", path, + realPath, containerID.c_str()); + goto cleanup; +diff --git a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +index 8eff22ac..bc40cb06 100644 +--- a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +@@ -569,7 +569,7 @@ void PodSandboxManagerService::UpdatePodSandboxNetworkSettings(const std::string + } + + auto req = (container_update_network_settings_request *)util_common_calloc_s( +- sizeof(container_update_network_settings_request)); ++ sizeof(container_update_network_settings_request)); + if (req == nullptr) { + error.Errorf("container update network settings request: Out of memory"); + return; +diff --git a/src/daemon/entry/cri/v1alpha/cri_security_context.cc b/src/daemon/entry/cri/v1alpha/cri_security_context.cc +index 57ec3a63..2b6c42fe 100644 +--- a/src/daemon/entry/cri/v1alpha/cri_security_context.cc ++++ b/src/daemon/entry/cri/v1alpha/cri_security_context.cc +@@ -130,7 +130,8 @@ static void ModifyHostConfigscSupplementalGroups(const runtime::v1alpha2::LinuxC + } + } + +-static void ApplyMaskedPathsToHostConfig(const runtime::v1alpha2::LinuxContainerSecurityContext &sc, host_config *hostConfig, ++static void ApplyMaskedPathsToHostConfig(const runtime::v1alpha2::LinuxContainerSecurityContext &sc, ++ host_config *hostConfig, + Errors &error) + { + if (sc.masked_paths_size() <= 0) { +@@ -159,7 +160,8 @@ static void ApplyMaskedPathsToHostConfig(const runtime::v1alpha2::LinuxContainer + } + } + +-static void ApplyReadonlyPathsToHostConfig(const runtime::v1alpha2::LinuxContainerSecurityContext &sc, host_config *hostConfig, ++static void ApplyReadonlyPathsToHostConfig(const runtime::v1alpha2::LinuxContainerSecurityContext &sc, ++ host_config *hostConfig, + Errors &error) + { + if (sc.readonly_paths_size() <= 0) { +diff --git a/src/daemon/entry/cri/v1alpha/v1alpha_cri_helpers.cc b/src/daemon/entry/cri/v1alpha/v1alpha_cri_helpers.cc +index e55e46f0..b06092a7 100644 +--- a/src/daemon/entry/cri/v1alpha/v1alpha_cri_helpers.cc ++++ b/src/daemon/entry/cri/v1alpha/v1alpha_cri_helpers.cc +@@ -392,7 +392,7 @@ auto GetSecurityOpts(const commonSecurityContext &context, const char &separator + } + + std::vector<std::string> selinuxOpts = CRIHelpersV1Alpha::GetSELinuxLabelOpts(context.hasSELinuxOption, +- context.selinuxOption, separator, error); ++ context.selinuxOption, separator, error); + if (error.NotEmpty()) { + error.Errorf("Failed to generate SELinuxLabel options for container %s", error.GetMessage().c_str()); + return securityOpts; +diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c +index 4b6f62de..6b6c3b75 100644 +--- a/src/daemon/executor/container_cb/execution_create.c ++++ b/src/daemon/executor/container_cb/execution_create.c +@@ -766,8 +766,8 @@ static int maintain_container_id(const container_create_request *request, char * + used_id = container_name_index_get(name); + ERROR("Name %s is in use by container %s", name, used_id); + isulad_set_error_message("Conflict. The name \"%s\" is already in use by container %s. " +- "You have to remove (or rename) that container to be able to reuse that name.", +- name, used_id); ++ "You have to remove (or rename) that container to be able to reuse that name.", ++ name, used_id); + free(used_id); + used_id = NULL; + ret = -1; +@@ -1342,7 +1342,7 @@ static int v2_spec_fill_sandbox_info(const container_sandbox_info *sandbox_info, + return 0; + } + +- if(dup_container_sandbox_info(sandbox_info, &v2_spec->sandbox_info) != 0) { ++ if (dup_container_sandbox_info(sandbox_info, &v2_spec->sandbox_info) != 0) { + ERROR("Failed to dup sandbox info"); + return -1; + } +@@ -1431,7 +1431,7 @@ int container_create_cb(const container_create_request *request, container_creat + v2_spec_fill_basic_info(id, name, image_name, image_type, container_spec, v2_spec); + + #ifdef ENABLE_CRI_API_V1 +- if(v2_spec_fill_sandbox_info(request->sandbox, v2_spec) != 0) { ++ if (v2_spec_fill_sandbox_info(request->sandbox, v2_spec) != 0) { + ERROR("Failed to fill sandbox info"); + cc = ISULAD_ERR_INPUT; + goto clean_container_root_dir; +diff --git a/src/daemon/executor/container_cb/execution_information.c b/src/daemon/executor/container_cb/execution_information.c +index 95f522c6..420f08df 100644 +--- a/src/daemon/executor/container_cb/execution_information.c ++++ b/src/daemon/executor/container_cb/execution_information.c +@@ -250,7 +250,7 @@ static int get_proxy_env(char **proxy, const char *type) + *proxy = NULL; + goto out; + } +- ++ + out: + util_free_sensitive_string(tmp_proxy); + return ret; +@@ -634,7 +634,8 @@ out: + return pid_arg; + } + +-static int get_pids(const char *name, const char *runtime, const char *rootpath, const char *statepath, pid_t **pids, size_t *pids_len, ++static int get_pids(const char *name, const char *runtime, const char *rootpath, const char *statepath, pid_t **pids, ++ size_t *pids_len, + char **pid_args) + { + int ret = 0; +diff --git a/src/daemon/modules/api/container_api.h b/src/daemon/modules/api/container_api.h +index 49849c2d..8e3cd7ac 100644 +--- a/src/daemon/modules/api/container_api.h ++++ b/src/daemon/modules/api/container_api.h +@@ -284,7 +284,8 @@ bool container_is_in_gc_progress(const char *id); + int container_module_init(); + + #ifdef ENABLE_CRI_API_V1 +-static inline bool is_sandbox_container(container_sandbox_info *sandbox) { ++static inline bool is_sandbox_container(container_sandbox_info *sandbox) ++{ + return sandbox != NULL && sandbox->is_sandbox_container; + } + #endif +diff --git a/src/daemon/modules/container/restore/restore.c b/src/daemon/modules/container/restore/restore.c +index a60b1410..fad5b071 100644 +--- a/src/daemon/modules/container/restore/restore.c ++++ b/src/daemon/modules/container/restore/restore.c +@@ -291,7 +291,8 @@ static void restore_state(container_t *cont) + } + } + if (tempret != 0) { +- WARN("Failed to restore container %s, make real status to STOPPED. Due to cannot load container with status %d", id, status); ++ WARN("Failed to restore container %s, make real status to STOPPED. Due to cannot load container with status %d", id, ++ status); + real_status.status = RUNTIME_CONTAINER_STATUS_STOPPED; + } + } +diff --git a/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.c b/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.c +index 4908565e..74d6d74a 100644 +--- a/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.c ++++ b/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.c +@@ -482,7 +482,7 @@ static int do_parse_version_info_stdout_str(int exec_ret, const cni_exec_error * + const char *stdout_str, cni_version_info **result_version) + { + __isula_auto_free char *err_msg = NULL; +- struct parser_context ctx = { OPT_GEN_SIMPLIFY, 0 }; ++ struct parser_context ctx = { OPT_GEN_SIMPLIFY, 0 }; + __isula_auto_free parser_error perr = NULL; + + if (exec_ret != 0) { +diff --git a/src/daemon/modules/network/cni_operator/libcni/libcni_api.c b/src/daemon/modules/network/cni_operator/libcni/libcni_api.c +index 7f62df78..068881be 100644 +--- a/src/daemon/modules/network/cni_operator/libcni/libcni_api.c ++++ b/src/daemon/modules/network/cni_operator/libcni/libcni_api.c +@@ -209,7 +209,7 @@ static int inject_cni_aliases(const struct runtime_conf *rt, cni_net_conf_runtim + } + + for (i = 0; i < rt->aliases_len; i++) { +- rt_config->aliases[i]= util_strdup_s(rt->aliases[i]); ++ rt_config->aliases[i] = util_strdup_s(rt->aliases[i]); + } + rt_config->aliases_len = rt->aliases_len; + return 0; +@@ -472,7 +472,7 @@ static int find_plugin_in_path(const char *plugin, const char * const *paths, si + ERROR("Invalid plugin name: %s", plugin); + return -1; + } +- ++ + for (i = 0; i < len; i++) { + if (do_check_file(plugin, paths[i], find_path) == 0) { + ret = 0; +@@ -689,7 +689,8 @@ int cni_add_network_list(const struct cni_network_list_conf *list, const struct + } + } + +- if (*pret != NULL && version_greater_than_or_equal_to((*pret)->cniversion, SUPPORT_CACHE_AND_CHECK_VERSION, &greater) != 0) { ++ if (*pret != NULL && ++ version_greater_than_or_equal_to((*pret)->cniversion, SUPPORT_CACHE_AND_CHECK_VERSION, &greater) != 0) { + return 0; + } + +@@ -867,7 +868,7 @@ static int version_network(const char *plugin_name, cni_version_info **result_ve + } + + int cni_version_network_list(const struct cni_network_list_conf *list, +- struct cni_version_info_list **result_version_list) ++ struct cni_version_info_list **result_version_list) + { + int ret = 0; + int i; +diff --git a/src/daemon/modules/network/cni_operator/libcni/libcni_api.h b/src/daemon/modules/network/cni_operator/libcni/libcni_api.h +index f94ab3f7..2f10d6e9 100644 +--- a/src/daemon/modules/network/cni_operator/libcni/libcni_api.h ++++ b/src/daemon/modules/network/cni_operator/libcni/libcni_api.h +@@ -84,7 +84,7 @@ int cni_del_network_list(const struct cni_network_list_conf *list, const struct + + int cni_check_network_list(const struct cni_network_list_conf *list, const struct runtime_conf *rc, + struct cni_opt_result **p_result); +- ++ + int cni_version_network_list(const struct cni_network_list_conf *list, + struct cni_version_info_list **result_version_list); + +diff --git a/src/daemon/modules/network/cni_operator/libcni/libcni_cached.c b/src/daemon/modules/network/cni_operator/libcni/libcni_cached.c +index 222511d6..1457c9be 100644 +--- a/src/daemon/modules/network/cni_operator/libcni/libcni_cached.c ++++ b/src/daemon/modules/network/cni_operator/libcni/libcni_cached.c +@@ -205,7 +205,7 @@ static int do_cache_insert_aliases(const struct runtime_conf *rc, cni_cached_inf + return -1; + } + +- for (i = 0; i < rc->aliases_len; i++){ ++ for (i = 0; i < rc->aliases_len; i++) { + tmp_aliases[i] = util_strdup_s(rc->aliases[i]); + } + +diff --git a/src/daemon/modules/network/cni_operator/libcni/libcni_result_type.c b/src/daemon/modules/network/cni_operator/libcni/libcni_result_type.c +index 8a0ce1dd..2365a128 100644 +--- a/src/daemon/modules/network/cni_operator/libcni/libcni_result_type.c ++++ b/src/daemon/modules/network/cni_operator/libcni/libcni_result_type.c +@@ -137,7 +137,7 @@ void free_cni_version_info_list(struct cni_version_info_list *val) + if (val == NULL) { + return; + } +- ++ + for (i = 0; i < val->result_versions_len; i++) { + free_cni_version_info(val->result_versions[i]); + val->result_versions[i] = NULL; +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index e61d1f91..859356e5 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -1495,7 +1495,7 @@ static int get_attach_socketfd(const char *attach_socket, int *socket_fd) + + if (strlen(attach_socket) >= sizeof(addr.sun_path)) { + SYSERROR("Invalid attach socket path: %s", attach_socket); +- return -1; ++ return -1; + } + + tmp_socket = socket(AF_UNIX, SOCK_STREAM, 0); +diff --git a/src/daemon/modules/runtime/shim/shim_rt_monitor.cc b/src/daemon/modules/runtime/shim/shim_rt_monitor.cc +index f5178c26..2547a206 100644 +--- a/src/daemon/modules/runtime/shim/shim_rt_monitor.cc ++++ b/src/daemon/modules/runtime/shim/shim_rt_monitor.cc +@@ -31,7 +31,7 @@ + #include "error.h" + + extern "C" { +- #include <shim_v2.h> ++#include <shim_v2.h> + } + + struct shim_monitor_data { +diff --git a/src/daemon/modules/runtime/shim/shim_rt_ops.c b/src/daemon/modules/runtime/shim/shim_rt_ops.c +index 56fc43c2..1bc9dc54 100644 +--- a/src/daemon/modules/runtime/shim/shim_rt_ops.c ++++ b/src/daemon/modules/runtime/shim/shim_rt_ops.c +@@ -395,7 +395,7 @@ int rt_shim_create(const char *id, const char *runtime, const rt_create_params_t + + /** + * If task address is not set, create a new shim-v2 and get the address. +- * If task address is set, use it directly. ++ * If task address is set, use it directly. + */ + if (params->task_addr == NULL || strlen(params->task_addr) == 0) { + if (shim_bin_v2_create(runtime, id, params->bundle, NULL, response, state_path) != 0) { +diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c +index 01427ddf..97f73768 100644 +--- a/src/daemon/modules/service/service_container.c ++++ b/src/daemon/modules/service/service_container.c +@@ -2197,7 +2197,8 @@ static int exec_prepare_console(const container_t *cont, const container_exec_re + return exec_prepare_vsock(cont, request, stdinfd, stdout_handler, stderr_handler, io_addresses, sync_fd, thread_id); + } + #endif +- return exec_prepare_fifo(cont, request, stdinfd, stdout_handler, stderr_handler, io_addresses, iopath, sync_fd, thread_id); ++ return exec_prepare_fifo(cont, request, stdinfd, stdout_handler, stderr_handler, io_addresses, iopath, sync_fd, ++ thread_id); + } + + static void exec_container_end(container_exec_response *response, const container_t *cont, +diff --git a/src/daemon/modules/service/vsock_io_handler.cc b/src/daemon/modules/service/vsock_io_handler.cc +index efc74bc8..2c73490a 100644 +--- a/src/daemon/modules/service/vsock_io_handler.cc ++++ b/src/daemon/modules/service/vsock_io_handler.cc +@@ -165,7 +165,7 @@ static int vsock_connect(uint32_t cid, uint32_t port) + sa.svm_cid = cid; + sa.svm_port = port; + +- if (connect(fd, (struct sockaddr *)&sa, sizeof(sa)) !=0) { ++ if (connect(fd, (struct sockaddr *)&sa, sizeof(sa)) != 0) { + SYSERROR("Failed to connect vsock socket"); + close(fd); + return -1; +@@ -314,7 +314,7 @@ void delete_daemon_vsockpaths(const char *sandbox_id, const char *vsockpaths[]) + } + } + +-enum IOFlowType{ ++enum IOFlowType { + IO_SRC = 0, + IO_DST, + IO_FLOW_INVALID, +@@ -784,7 +784,8 @@ static void *IOCopyThread(void *arg) + return NULL; + } + +-int start_vsock_io_copy(const char *exec_id, int sync_fd, bool detach, const char *fifoin, const char *fifoout, const char *fifoerr, ++int start_vsock_io_copy(const char *exec_id, int sync_fd, bool detach, const char *fifoin, const char *fifoout, ++ const char *fifoerr, + int stdin_fd, struct io_write_wrapper *stdout_handler, struct io_write_wrapper *stderr_handler, + const char *vsocks[], pthread_t *tid) + { +diff --git a/src/daemon/modules/service/vsock_io_handler.h b/src/daemon/modules/service/vsock_io_handler.h +index cc0c1dd0..8bda7711 100644 +--- a/src/daemon/modules/service/vsock_io_handler.h ++++ b/src/daemon/modules/service/vsock_io_handler.h +@@ -20,8 +20,7 @@ + #include <pthread.h> + + #ifdef __cplusplus +-extern "C" +-{ ++extern "C" { + #endif + + bool is_vsock_path(const char *path); +@@ -35,7 +34,8 @@ int create_daemon_vsockpaths(const char *sandbox_id, uint32_t cid, bool attach_s + + void delete_daemon_vsockpaths(const char *sandbox_id, const char *vsockpaths[]); + +-int start_vsock_io_copy(const char *exec_id, int sync_fd, bool detach, const char *fifoin, const char *fifoout, const char *fifoerr, ++int start_vsock_io_copy(const char *exec_id, int sync_fd, bool detach, const char *fifoin, const char *fifoout, ++ const char *fifoerr, + int stdin_fd, struct io_write_wrapper *stdout_handler, struct io_write_wrapper *stderr_handler, + const char *vsocks[], pthread_t *tid); + +diff --git a/src/daemon/sandbox/controller/sandboxer/client/grpc_async_wait_call.cc b/src/daemon/sandbox/controller/sandboxer/client/grpc_async_wait_call.cc +index d6d94461..c6f97da1 100644 +--- a/src/daemon/sandbox/controller/sandboxer/client/grpc_async_wait_call.cc ++++ b/src/daemon/sandbox/controller/sandboxer/client/grpc_async_wait_call.cc +@@ -36,7 +36,8 @@ SandboxerAsyncWaitCall::SandboxerAsyncWaitCall(std::shared_ptr<SandboxStatusCall + m_remove = false; + } + +-auto SandboxerAsyncWaitCall::Call(containerd::services::sandbox::v1::Controller::StubInterface &stub, grpc::CompletionQueue &cq) -> bool ++auto SandboxerAsyncWaitCall::Call(containerd::services::sandbox::v1::Controller::StubInterface &stub, ++ grpc::CompletionQueue &cq) -> bool + { + containerd::services::sandbox::v1::ControllerWaitRequest request; + m_context = std::unique_ptr<grpc::ClientContext>(new grpc::ClientContext()); +@@ -96,10 +97,11 @@ SandboxerAsyncWaitStatus SandboxerAsyncWaitCall::HandleResponse() + ControllerExitInfo exitInfo; + SandboxerAsyncWaitStatus waitStatus = SANDBOXER_ASYNC_WAIT_STATUS_ERROR; + +- switch(m_status.error_code()) { ++ switch (m_status.error_code()) { + case grpc::StatusCode::UNAVAILABLE: + // If the status is unavailable, connection failed, we should retry +- WARN("Sandboxer controller wait rpc server unavailable, error_code: %d: %s", m_status.error_code(), m_status.error_message().c_str()); ++ WARN("Sandboxer controller wait rpc server unavailable, error_code: %d: %s", m_status.error_code(), ++ m_status.error_message().c_str()); + waitStatus = SANDBOXER_ASYNC_WAIT_STATUS_RETRY; + m_retryTimes++; + // If retried times is more than 10, we should retry every 300 seconds +@@ -128,7 +130,8 @@ SandboxerAsyncWaitStatus SandboxerAsyncWaitCall::HandleResponse() + break; + default: + // TODO: More error code should be handled +- ERROR("Sandboxer controller wait request failed, error_code: %d: %s", m_status.error_code(), m_status.error_message().c_str()); ++ ERROR("Sandboxer controller wait request failed, error_code: %d: %s", m_status.error_code(), ++ m_status.error_message().c_str()); + SandboxExitCallback(false, exitInfo); + break; + } +diff --git a/src/daemon/sandbox/controller/sandboxer/client/grpc_async_wait_call.h b/src/daemon/sandbox/controller/sandboxer/client/grpc_async_wait_call.h +index eb633e99..6e5a6756 100644 +--- a/src/daemon/sandbox/controller/sandboxer/client/grpc_async_wait_call.h ++++ b/src/daemon/sandbox/controller/sandboxer/client/grpc_async_wait_call.h +@@ -49,16 +49,26 @@ public: + void SandboxPendingCallback(); + void SandboxReadyCallback(); + auto GetSandboxId() -> const std::string &; +- auto MarkRemove() -> void { m_remove = true; } +- auto ToRemove() -> bool { return m_remove; } +- auto ResetRetryTimes() -> void { m_retryTimes = 0; } ++ auto MarkRemove() -> void ++ { ++ m_remove = true; ++ } ++ auto ToRemove() -> bool ++ { ++ return m_remove; ++ } ++ auto ResetRetryTimes() -> void ++ { ++ m_retryTimes = 0; ++ } + + protected: + std::shared_ptr<containerd::services::sandbox::v1::Controller::StubInterface> m_stub; + std::shared_ptr<SandboxStatusCallback> m_cb; + std::string m_sandboxId; + std::string m_sandboxer; +- std::unique_ptr<grpc::ClientAsyncResponseReaderInterface<containerd::services::sandbox::v1::ControllerWaitResponse>> m_responseReader; ++ std::unique_ptr<grpc::ClientAsyncResponseReaderInterface<containerd::services::sandbox::v1::ControllerWaitResponse>> ++ m_responseReader; + std::unique_ptr<grpc::ClientContext> m_context; + containerd::services::sandbox::v1::ControllerWaitResponse m_response; + grpc::Status m_status; +diff --git a/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_client.cc b/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_client.cc +index b5dda0ed..11c2b014 100644 +--- a/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_client.cc ++++ b/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_client.cc +@@ -102,14 +102,16 @@ auto SandboxerClient::Create(const std::string &sandboxId, const ControllerCreat + status = m_stub->Create(&context, request, &response); + if (!status.ok()) { + error.SetError(status.error_message()); +- ERROR("Sandboxer controller create request failed, error_code: %d: %s", status.error_code(), status.error_message().c_str()); ++ ERROR("Sandboxer controller create request failed, error_code: %d: %s", status.error_code(), ++ status.error_message().c_str()); + return false; + } + + return true; + } + +-void SandboxerClient::StartResponseToSandboxInfo(const containerd::services::sandbox::v1::ControllerStartResponse &response, ++void SandboxerClient::StartResponseToSandboxInfo(const containerd::services::sandbox::v1::ControllerStartResponse ++ &response, + ControllerSandboxInfo &sandboxInfo) + { + sandboxInfo.id = response.sandbox_id(); +@@ -132,7 +134,8 @@ auto SandboxerClient::Start(const std::string &sandboxId, ControllerSandboxInfo + status = m_stub->Start(&context, request, &response); + if (!status.ok()) { + error.SetError(status.error_message()); +- ERROR("Sandboxer controller start request failed, error_code: %d: %s", status.error_code(), status.error_message().c_str()); ++ ERROR("Sandboxer controller start request failed, error_code: %d: %s", status.error_code(), ++ status.error_message().c_str()); + return false; + } + +@@ -174,7 +177,8 @@ auto SandboxerClient::InitPrepareRequest(containerd::services::sandbox::v1::Prep + return true; + } + +-auto SandboxerClient::Prepare(const std::string &sandboxId, const ControllerPrepareParams ¶ms, std::string &bundle, Errors &error) -> bool ++auto SandboxerClient::Prepare(const std::string &sandboxId, const ControllerPrepareParams ¶ms, std::string &bundle, ++ Errors &error) -> bool + { + grpc::ClientContext context; + containerd::services::sandbox::v1::PrepareRequest request; +@@ -189,7 +193,8 @@ auto SandboxerClient::Prepare(const std::string &sandboxId, const ControllerPrep + status = m_stub->Prepare(&context, request, &response); + if (!status.ok()) { + error.SetError(status.error_message()); +- ERROR("Sandboxer controller prepare request failed, error_code: %d: %s", status.error_code(), status.error_message().c_str()); ++ ERROR("Sandboxer controller prepare request failed, error_code: %d: %s", status.error_code(), ++ status.error_message().c_str()); + return false; + } + +@@ -199,7 +204,7 @@ auto SandboxerClient::Prepare(const std::string &sandboxId, const ControllerPrep + } + + auto SandboxerClient::Purge(const std::string &sandboxId, const std::string &containerId, +- const std::string &execId, Errors &error) -> bool ++ const std::string &execId, Errors &error) -> bool + { + grpc::ClientContext context; + containerd::services::sandbox::v1::PurgeRequest request; +@@ -214,7 +219,8 @@ auto SandboxerClient::Purge(const std::string &sandboxId, const std::string &con + status = m_stub->Purge(&context, request, &response); + if (!status.ok()) { + error.SetError(status.error_message()); +- ERROR("Sandboxer controller purge request failed, error_code: %d: %s", status.error_code(), status.error_message().c_str()); ++ ERROR("Sandboxer controller purge request failed, error_code: %d: %s", status.error_code(), ++ status.error_message().c_str()); + return false; + } + +@@ -237,7 +243,8 @@ auto SandboxerClient::InitUpdateResourcesRequest(containerd::services::sandbox:: + return true; + } + +-auto SandboxerClient::UpdateResources(const std::string &sandboxId, const ControllerUpdateResourcesParams ¶ms, Errors &error) -> bool ++auto SandboxerClient::UpdateResources(const std::string &sandboxId, const ControllerUpdateResourcesParams ¶ms, ++ Errors &error) -> bool + { + grpc::ClientContext context; + containerd::services::sandbox::v1::UpdateResourcesRequest request; +@@ -245,21 +252,24 @@ auto SandboxerClient::UpdateResources(const std::string &sandboxId, const Contro + grpc::Status status; + + if (!InitUpdateResourcesRequest(request, sandboxId, params)) { +- error.SetError("Failed to init update-resources request for sandboxer update-resources request, sandbox id: " + sandboxId); ++ error.SetError("Failed to init update-resources request for sandboxer update-resources request, sandbox id: " + ++ sandboxId); + return false; + } + + status = m_stub->UpdateResources(&context, request, &response); + if (!status.ok()) { + error.SetError(status.error_message()); +- ERROR("Sandboxer controller update resources request failed, error_code: %d: %s", status.error_code(), status.error_message().c_str()); ++ ERROR("Sandboxer controller update resources request failed, error_code: %d: %s", status.error_code(), ++ status.error_message().c_str()); + return false; + } + + return true; + } + +-void SandboxerClient::PlatformResponseToPlatformInfo(const containerd::services::sandbox::v1::ControllerPlatformResponse &response, ++void SandboxerClient::PlatformResponseToPlatformInfo(const containerd::services::sandbox::v1::ControllerPlatformResponse ++ &response, + ControllerPlatformInfo &platformInfo) + { + auto &platform = response.platform(); +@@ -268,7 +278,8 @@ void SandboxerClient::PlatformResponseToPlatformInfo(const containerd::services: + platformInfo.variant = platform.variant(); + } + +-auto SandboxerClient::Platform(const std::string &sandboxId, ControllerPlatformInfo &platformInfo, Errors &error) -> bool ++auto SandboxerClient::Platform(const std::string &sandboxId, ControllerPlatformInfo &platformInfo, ++ Errors &error) -> bool + { + grpc::ClientContext context; + containerd::services::sandbox::v1::ControllerPlatformRequest request; +@@ -281,7 +292,8 @@ auto SandboxerClient::Platform(const std::string &sandboxId, ControllerPlatformI + status = m_stub->Platform(&context, request, &response); + if (!status.ok()) { + error.SetError(status.error_message()); +- ERROR("Sandboxer controller platform request failed, error_code: %d: %s", status.error_code(), status.error_message().c_str()); ++ ERROR("Sandboxer controller platform request failed, error_code: %d: %s", status.error_code(), ++ status.error_message().c_str()); + return false; + } + +@@ -304,14 +316,16 @@ auto SandboxerClient::Stop(const std::string &sandboxId, uint32_t timeoutSecs, E + status = m_stub->Stop(&context, request, &response); + if (!status.ok()) { + error.SetError(status.error_message()); +- ERROR("Sandboxer controller stop request failed, error_code: %d: %s", status.error_code(), status.error_message().c_str()); ++ ERROR("Sandboxer controller stop request failed, error_code: %d: %s", status.error_code(), ++ status.error_message().c_str()); + return false; + } + + return true; + } + +-void SandboxerClient::StatusResponseToSandboxStatus(const containerd::services::sandbox::v1::ControllerStatusResponse &response, ++void SandboxerClient::StatusResponseToSandboxStatus(const containerd::services::sandbox::v1::ControllerStatusResponse ++ &response, + ControllerSandboxStatus &sandboxStatus) + { + sandboxStatus.id = response.sandbox_id(); +@@ -324,7 +338,8 @@ void SandboxerClient::StatusResponseToSandboxStatus(const containerd::services:: + sandboxStatus.extra = response.extra().value(); + } + +-auto SandboxerClient::Status(const std::string &sandboxId, bool verbose, ControllerSandboxStatus &sandboxStatus, Errors &error) -> bool ++auto SandboxerClient::Status(const std::string &sandboxId, bool verbose, ControllerSandboxStatus &sandboxStatus, ++ Errors &error) -> bool + { + grpc::ClientContext context; + containerd::services::sandbox::v1::ControllerStatusRequest request; +@@ -338,7 +353,8 @@ auto SandboxerClient::Status(const std::string &sandboxId, bool verbose, Control + status = m_stub->Status(&context, request, &response); + if (!status.ok()) { + error.SetError(status.error_message()); +- ERROR("Sandboxer controller status request failed, error_code: %d: %s", status.error_code(), status.error_message().c_str()); ++ ERROR("Sandboxer controller status request failed, error_code: %d: %s", status.error_code(), ++ status.error_message().c_str()); + return false; + } + +@@ -360,14 +376,16 @@ auto SandboxerClient::Shutdown(const std::string &sandboxId, Errors &error) -> b + status = m_stub->Shutdown(&context, request, &response); + if (!status.ok()) { + error.SetError(status.error_message()); +- ERROR("Sandboxer controller shutdown request failed, error_code: %d: %s", status.error_code(), status.error_message().c_str()); ++ ERROR("Sandboxer controller shutdown request failed, error_code: %d: %s", status.error_code(), ++ status.error_message().c_str()); + return false; + } + + return true; + } + +-auto SandboxerClient::Wait(std::shared_ptr<SandboxStatusCallback> cb, const std::string &sandboxId, Errors &error) -> bool ++auto SandboxerClient::Wait(std::shared_ptr<SandboxStatusCallback> cb, const std::string &sandboxId, ++ Errors &error) -> bool + { + if (m_monitor == nullptr) { + error.SetError("Cannot wait for sandbox, sandboxer client monitor is not initialized, " +diff --git a/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_client.h b/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_client.h +index 85e1e608..accca16b 100644 +--- a/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_client.h ++++ b/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_client.h +@@ -48,12 +48,14 @@ public: + + auto Platform(const std::string &sandboxId, ControllerPlatformInfo &platformInfo, Errors &error) -> bool; + +- auto Prepare(const std::string &sandboxId, const ControllerPrepareParams ¶ms, std::string &bundle, Errors &error) -> bool; ++ auto Prepare(const std::string &sandboxId, const ControllerPrepareParams ¶ms, std::string &bundle, ++ Errors &error) -> bool; + + auto Purge(const std::string &sandboxId, const std::string &containerId, + const std::string &execId, Errors &error) -> bool; + +- auto UpdateResources(const std::string &sandboxId, const ControllerUpdateResourcesParams ¶ms, Errors &error) -> bool; ++ auto UpdateResources(const std::string &sandboxId, const ControllerUpdateResourcesParams ¶ms, ++ Errors &error) -> bool; + + auto Stop(const std::string &sandboxId, uint32_t timeoutSecs, Errors &error) -> bool; + +diff --git a/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_monitor.cc b/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_monitor.cc +index 1417ee40..485a0b23 100644 +--- a/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_monitor.cc ++++ b/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_monitor.cc +@@ -24,7 +24,7 @@ namespace sandbox { + const int64_t DEFERRED_QUEUE_CHECK_INTERVAL = 200; // milliseconds + + SandboxerClientMonitor::SandboxerClientMonitor(std::shared_ptr<grpc::Channel> channel, const std::string &sandboxer): +- m_channel(channel), m_sandboxer(sandboxer) ,m_teardown(false) ++ m_channel(channel), m_sandboxer(sandboxer), m_teardown(false) + { + m_stub = containerd::services::sandbox::v1::Controller::NewStub(m_channel); + } +@@ -46,7 +46,7 @@ auto SandboxerClientMonitor::Monitor(SandboxerAsyncWaitCall *call) -> bool + } + + // Try to monitor the call, if failed, we should delete it right way +- if (!call->Call(*m_stub ,m_cq)) { ++ if (!call->Call(*m_stub, m_cq)) { + // The failure is most likely due to the fact that the completion queue is shutdown + delete call; + return false; +@@ -114,11 +114,11 @@ void SandboxerClientMonitor::WaitForDeferredCall() + // 2. SandboxerAsyncWaitCall *: The call handled by the future + void SandboxerClientMonitor::InvokeDeferredCall(SandboxerAsyncWaitCall *call) + { +- m_futures.push_back(std::async([this, call](){ ++ m_futures.push_back(std::async([this, call]() { + // Random sleep for 50 ~ 200 milliseconds to avoid thundering herd + std::random_device rd; + std::mt19937 gen(rd()); +- std::uniform_int_distribution<> dis(DEFERRED_QUEUE_CHECK_INTERVAL/4, DEFERRED_QUEUE_CHECK_INTERVAL); ++ std::uniform_int_distribution<> dis(DEFERRED_QUEUE_CHECK_INTERVAL / 4, DEFERRED_QUEUE_CHECK_INTERVAL); + int sleepTime = dis(gen); + std::this_thread::sleep_for(std::chrono::milliseconds(sleepTime)); + +@@ -186,7 +186,7 @@ void SandboxerClientMonitor::CheckCompletedFutures() + // or OnSandboxExit in the HandleResponse function. + // In this case, the OnSandboxReady will overwrite the + // status, but it is ok, because: +- // 1. If OnSandboxPending has been invoked, ++ // 1. If OnSandboxPending has been invoked, + // retry will happen pretty soon, and the + // callback will be invoked again. + // 2. If OnSandboxExit has been invoked, the caller +diff --git a/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_monitor.h b/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_monitor.h +index 32fca934..b5740b44 100644 +--- a/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_monitor.h ++++ b/src/daemon/sandbox/controller/sandboxer/client/grpc_sandboxer_monitor.h +@@ -68,7 +68,7 @@ private: + // Vector for holding all the calls for monitoring + std::vector<SandboxerAsyncWaitCall *> m_calls; + std::mutex m_callsMutex; +- // Use to indicate whether ++ // Use to indicate whether + bool m_teardown; + // Vector for holding all the retry calls + std::vector<SandboxerAsyncWaitCall *> m_deferredCalls; +diff --git a/src/daemon/sandbox/controller/shim/shim_controller.cc b/src/daemon/sandbox/controller/shim/shim_controller.cc +index baddf1aa..39fcf8ea 100644 +--- a/src/daemon/sandbox/controller/shim/shim_controller.cc ++++ b/src/daemon/sandbox/controller/shim/shim_controller.cc +@@ -247,7 +247,8 @@ auto ShimController::GenerateSandboxCreateContainerRequest(const std::string &sa + return nullptr; + } + +- auto requestWrapper = PackCreateContainerRequest(sandboxId, params, hostConfigWrapper->get(), customConfigWrapper->get(), error); ++ auto requestWrapper = PackCreateContainerRequest(sandboxId, params, hostConfigWrapper->get(), ++ customConfigWrapper->get(), error); + if (requestWrapper == nullptr) { + ERROR("Failed to pack create container request"); + error.SetError("Failed to pack create container request"); +@@ -570,7 +571,7 @@ bool ShimController::UpdateNetworkSettings(const std::string &sandboxId, const s + } + + auto requestWrapper = makeUniquePtrCStructWrapper<container_update_network_settings_request>( +- free_container_update_network_settings_request); ++ free_container_update_network_settings_request); + if (requestWrapper == nullptr) { + ERROR("container update network settings request: Out of memory"); + error.Errorf("container update network settings request: Out of memory"); +@@ -583,7 +584,7 @@ bool ShimController::UpdateNetworkSettings(const std::string &sandboxId, const s + container_update_network_settings_response *response { nullptr }; + int ret = m_cb->container.update_network_settings(request, &response); + auto responseWrapper = makeUniquePtrCStructWrapper<container_update_network_settings_response>( +- response, free_container_update_network_settings_response); ++ response, free_container_update_network_settings_response); + + if (ret != 0) { + if (response != nullptr && response->errmsg != nullptr) { +diff --git a/src/daemon/sandbox/sandbox.cc b/src/daemon/sandbox/sandbox.cc +index c8fd30be..b1832265 100644 +--- a/src/daemon/sandbox/sandbox.cc ++++ b/src/daemon/sandbox/sandbox.cc +@@ -942,7 +942,8 @@ auto Sandbox::ParseSandboxMetadataFile() -> std::unique_ptr<CStructWrapper<sandb + if (metadata == NULL) { + return nullptr; + } +- return std::unique_ptr<CStructWrapper<sandbox_metadata>>(new CStructWrapper<sandbox_metadata>(metadata, free_sandbox_metadata)); ++ return std::unique_ptr<CStructWrapper<sandbox_metadata>>(new CStructWrapper<sandbox_metadata>(metadata, ++ free_sandbox_metadata)); + } + + auto Sandbox::isValidMetadata(std::unique_ptr<CStructWrapper<sandbox_metadata>> &metadata) -> bool +diff --git a/src/daemon/sandbox/sandbox_manager.cc b/src/daemon/sandbox/sandbox_manager.cc +index 60ce97d5..d3db4fb4 100644 +--- a/src/daemon/sandbox/sandbox_manager.cc ++++ b/src/daemon/sandbox/sandbox_manager.cc +@@ -60,7 +60,8 @@ auto SandboxManager::Init(Errors &error) -> bool + return true; + } + +-auto SandboxManager::CreateSandbox(const std::string &name, RuntimeInfo &info, std::string &netNsPath, std::string &netMode, ++auto SandboxManager::CreateSandbox(const std::string &name, RuntimeInfo &info, std::string &netNsPath, ++ std::string &netMode, + const runtime::v1::PodSandboxConfig &sandboxConfig, const std::string &image, Errors &error) -> std::shared_ptr<Sandbox> + { + std::shared_ptr<Sandbox> sandbox; +diff --git a/src/daemon/sandbox/sandbox_manager.h b/src/daemon/sandbox/sandbox_manager.h +index 3a6ce3c9..c9ed78ed 100644 +--- a/src/daemon/sandbox/sandbox_manager.h ++++ b/src/daemon/sandbox/sandbox_manager.h +@@ -37,7 +37,8 @@ public: + + // Create meanningful sandbox instance + auto CreateSandbox(const std::string &name, RuntimeInfo &info, std::string &netNsPath, std::string &netMode, +- const runtime::v1::PodSandboxConfig &sandboxConfig, const std::string &image, Errors &error) -> std::shared_ptr<Sandbox>; ++ const runtime::v1::PodSandboxConfig &sandboxConfig, const std::string &image, ++ Errors &error) -> std::shared_ptr<Sandbox>; + + auto GetSandbox(const std::string &idOrName) -> std::shared_ptr<Sandbox>; + auto DeleteSandbox(const std::string &idOrName, Errors &error) -> bool; +diff --git a/src/daemon/sandbox/sandbox_ops.cc b/src/daemon/sandbox/sandbox_ops.cc +index 2f4a46f6..005063c0 100644 +--- a/src/daemon/sandbox/sandbox_ops.cc ++++ b/src/daemon/sandbox/sandbox_ops.cc +@@ -70,7 +70,7 @@ static int do_sandbox_prepare(const container_config_v2_common_config *config, + } + + params.containerId = config->id; +- params.execId = (nullptr == exec_id) ? "" :exec_id; ++ params.execId = (nullptr == exec_id) ? "" : exec_id; + params.spec = std::move(std::unique_ptr<std::string>(new std::string(oci_spec))); + + if (generate_ctrl_rootfs(params, config) != 0) { +diff --git a/src/daemon/sandbox/sandbox_ops.h b/src/daemon/sandbox/sandbox_ops.h +index bcb1e9d2..bef884fb 100644 +--- a/src/daemon/sandbox/sandbox_ops.h ++++ b/src/daemon/sandbox/sandbox_ops.h +@@ -21,8 +21,7 @@ + #include <isula_libutils/oci_runtime_spec.h> + + #ifdef __cplusplus +-extern "C" +-{ ++extern "C" { + #endif + + int sandbox_prepare_container(const container_config_v2_common_config *config, +diff --git a/src/utils/cpputils/url.cc b/src/utils/cpputils/url.cc +index baaded07..c1e5d27f 100644 +--- a/src/utils/cpputils/url.cc ++++ b/src/utils/cpputils/url.cc +@@ -122,8 +122,9 @@ int UnescapeDealWithPercentSign(size_t &i, std::string &s, const EncodeMode &mod + } + // for 3 bit hex, max value is 8 + if (mode == EncodeMode::ENCODE_HOST && s1 < 8 && +- std::string(s.begin() + static_cast<long>(i), s.begin() + static_cast<long>(i+3)) != percentSign) { +- ERROR("invalid URL escape %s", std::string(s.begin() + static_cast<long>(i), s.begin() + static_cast<long>(i + 3)).c_str()); ++ std::string(s.begin() + static_cast<long>(i), s.begin() + static_cast<long>(i + 3)) != percentSign) { ++ ERROR("invalid URL escape %s", std::string(s.begin() + static_cast<long>(i), ++ s.begin() + static_cast<long>(i + 3)).c_str()); + return -1; + } + if (mode == EncodeMode::ENCODE_ZONE) { +diff --git a/src/utils/cutils/error.h b/src/utils/cutils/error.h +index 75eae760..1ad799fa 100644 +--- a/src/utils/cutils/error.h ++++ b/src/utils/cutils/error.h +@@ -47,7 +47,7 @@ extern "C" { + /* info for detach */ \ + XX(INFO_DETACH, "Attach detach") \ + /* err max */ \ +- XX(ERR_UNKNOWN, "Unknown error") ++ XX(ERR_UNKNOWN, "Unknown error") + + #define ISULAD_ERRNO_GEN(n, s) ISULAD_##n, + typedef enum { ISULAD_ERRNO_MAP(ISULAD_ERRNO_GEN) } isulad_errno_t; +diff --git a/src/utils/cutils/map/map.h b/src/utils/cutils/map/map.h +index 45ce26b8..d569a3da 100644 +--- a/src/utils/cutils/map/map.h ++++ b/src/utils/cutils/map/map.h +@@ -20,10 +20,6 @@ + + #include "rb_tree.h" + +-#ifdef __cplusplus +-extern "C" { +-#endif +- + struct _map_t; + + #if defined(__cplusplus) || defined(c_plusplus) +@@ -113,9 +109,5 @@ void map_clear(map_t *map); + } + #endif + +-#ifdef __cplusplus +-} +-#endif +- + #endif // UTILS_CUTILS_MAP_MAP_H + +diff --git a/src/utils/tar/isulad_tar.c b/src/utils/tar/isulad_tar.c +index 2e61d823..bbe4c3b2 100644 +--- a/src/utils/tar/isulad_tar.c ++++ b/src/utils/tar/isulad_tar.c +@@ -97,7 +97,8 @@ cleanup: + return ret; + } + +-static int resolve_host_source_path(const char *path, bool follow_link, char **resolved_path, char **rebase_name, char **err) ++static int resolve_host_source_path(const char *path, bool follow_link, char **resolved_path, char **rebase_name, ++ char **err) + { + int ret = -1; + int nret = 0; +@@ -419,7 +420,8 @@ cleanup: + return ret; + } + +-static int tar_resource_rebase(const char *path, const char *rebase, const char *root_dir, struct io_read_wrapper *archive_reader, char **err) ++static int tar_resource_rebase(const char *path, const char *rebase, const char *root_dir, ++ struct io_read_wrapper *archive_reader, char **err) + { + int ret = -1; + int nret; +@@ -450,7 +452,8 @@ cleanup: + return ret; + } + +-int tar_resource(const struct archive_copy_info *info, const char *root_dir, struct io_read_wrapper *archive_reader, char **err) ++int tar_resource(const struct archive_copy_info *info, const char *root_dir, struct io_read_wrapper *archive_reader, ++ char **err) + { + if (info == NULL || root_dir == NULL || archive_reader == NULL || err == NULL) { + return -1; +diff --git a/src/utils/tar/isulad_tar.h b/src/utils/tar/isulad_tar.h +index 414bb024..401dcf50 100644 +--- a/src/utils/tar/isulad_tar.h ++++ b/src/utils/tar/isulad_tar.h +@@ -43,7 +43,8 @@ struct archive_copy_info *copy_info_source_path(const char *path, bool follow_li + char *prepare_archive_copy(const struct archive_copy_info *srcinfo, const struct archive_copy_info *dstinfo, + char **src_base, char **dst_base, char **err); + +-int tar_resource(const struct archive_copy_info *info, const char *root_dir, struct io_read_wrapper *archive_reader, char **err); ++int tar_resource(const struct archive_copy_info *info, const char *root_dir, struct io_read_wrapper *archive_reader, ++ char **err); + + int archive_copy_to(const struct io_read_wrapper *content, const struct archive_copy_info *srcinfo, + const char *dstpath, const char *root_dir, char **err); +diff --git a/test/cutils/utils_file/utils_file_ut.cc b/test/cutils/utils_file/utils_file_ut.cc +index cacfef45..b3e35744 100644 +--- a/test/cutils/utils_file/utils_file_ut.cc ++++ b/test/cutils/utils_file/utils_file_ut.cc +@@ -200,7 +200,7 @@ TEST(utils_file, test_util_proc_file_line_by_line) + ASSERT_EQ(util_proc_file_line_by_line(fp, nullptr, (void *)checked_layers), -1); + fclose(fp); + ASSERT_EQ(util_path_remove(path.c_str()), 0); +- ++ + ASSERT_EQ(util_proc_file_line_by_line(nullptr, parse_checked_layer_cb, (void *)checked_layers), -1); + } + +diff --git a/test/mocks/callback_mock.cc b/test/mocks/callback_mock.cc +index ef0e347c..153446eb 100644 +--- a/test/mocks/callback_mock.cc ++++ b/test/mocks/callback_mock.cc +@@ -25,7 +25,8 @@ void MockCallback_SetMock(std::shared_ptr<MockContainerCallback> mock) + g_container_callback_mock = mock; + } + +-static int service_executor_container_create(const container_create_request *request, container_create_response **response) ++static int service_executor_container_create(const container_create_request *request, ++ container_create_response **response) + { + if (g_container_callback_mock != nullptr) { + return g_container_callback_mock->ContainerCreate(request, response); +@@ -50,7 +51,8 @@ static int service_executor_container_stop(const container_stop_request *request + return 0; + } + +-static int service_executor_container_remove(const container_delete_request *request, container_delete_response **response) ++static int service_executor_container_remove(const container_delete_request *request, ++ container_delete_response **response) + { + if (g_container_callback_mock != nullptr) { + return g_container_callback_mock->ContainerRemove(request, response); +@@ -67,7 +69,7 @@ static int service_executor_container_wait(const container_wait_request *request + } + + static int service_executor_container_update_network_settings(const container_update_network_settings_request *request, +- container_update_network_settings_response **response) ++ container_update_network_settings_response **response) + { + if (g_container_callback_mock != nullptr) { + return g_container_callback_mock->ContainerUpdateNetworkSettings(request, response); +diff --git a/test/mocks/controller_stub_mock.cc b/test/mocks/controller_stub_mock.cc +index eadf3cb4..712540bb 100644 +--- a/test/mocks/controller_stub_mock.cc ++++ b/test/mocks/controller_stub_mock.cc +@@ -2,221 +2,313 @@ + + static std::shared_ptr<MockControllerStub> g_controller_stub_mock = NULL; + +-std::unique_ptr<DummyControllerStub> NewDummyControllerStub() { +- std::unique_ptr<DummyControllerStub> stub(new DummyControllerStub()); +- return stub; ++std::unique_ptr<DummyControllerStub> NewDummyControllerStub() ++{ ++ std::unique_ptr<DummyControllerStub> stub(new DummyControllerStub()); ++ return stub; + } + +-void MockControllerStub_SetMock(std::shared_ptr<MockControllerStub> mock) { ++void MockControllerStub_SetMock(std::shared_ptr<MockControllerStub> mock) ++{ + g_controller_stub_mock = mock; + } + +-::grpc::Status DummyControllerStub::Create(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerCreateRequest& request, ::containerd::services::sandbox::v1::ControllerCreateResponse* response) { +- if (g_controller_stub_mock == NULL) { +- return ::grpc::Status::OK; +- } +- return g_controller_stub_mock->Create(context, request, response); +-} +- +-::grpc::Status DummyControllerStub::Start(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStartRequest& request, ::containerd::services::sandbox::v1::ControllerStartResponse* response) { +- if (g_controller_stub_mock == NULL) { +- return ::grpc::Status::OK; +- } +- return g_controller_stub_mock->Start(context, request, response); +-} +- +-::grpc::Status DummyControllerStub::Platform(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerPlatformRequest& request, ::containerd::services::sandbox::v1::ControllerPlatformResponse* response) { +- if (g_controller_stub_mock == NULL) { +- return ::grpc::Status::OK; +- } +- return g_controller_stub_mock->Platform(context, request, response); +-} +- +-::grpc::Status DummyControllerStub::Prepare(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PrepareRequest& request, ::containerd::services::sandbox::v1::PrepareResponse* response) { +- if (g_controller_stub_mock == NULL) { +- return ::grpc::Status::OK; +- } +- return g_controller_stub_mock->Prepare(context, request, response); +-} +- +-::grpc::Status DummyControllerStub::Purge(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PurgeRequest& request, ::containerd::services::sandbox::v1::PurgeResponse* response) { +- if (g_controller_stub_mock == NULL) { +- return ::grpc::Status::OK; +- } +- return g_controller_stub_mock->Purge(context, request, response); +-} +- +-::grpc::Status DummyControllerStub::UpdateResources(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::UpdateResourcesRequest& request, ::containerd::services::sandbox::v1::UpdateResourcesResponse* response) { +- if (g_controller_stub_mock == NULL) { +- return ::grpc::Status::OK; +- } +- return g_controller_stub_mock->UpdateResources(context, request, response); +-} +- +-::grpc::Status DummyControllerStub::Stop(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStopRequest& request, ::containerd::services::sandbox::v1::ControllerStopResponse* response) { +- if (g_controller_stub_mock == NULL) { +- return ::grpc::Status::OK; +- } +- return g_controller_stub_mock->Stop(context, request, response); +-} +- +-::grpc::Status DummyControllerStub::Wait(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerWaitRequest& request, ::containerd::services::sandbox::v1::ControllerWaitResponse* response) { +- if (g_controller_stub_mock == NULL) { +- return ::grpc::Status::OK; +- } +- return g_controller_stub_mock->Wait(context, request, response); +-} +- +-::grpc::Status DummyControllerStub::Status(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStatusRequest& request, ::containerd::services::sandbox::v1::ControllerStatusResponse* response) { +- if (g_controller_stub_mock == NULL) { +- return ::grpc::Status::OK; +- } +- return g_controller_stub_mock->Status(context, request, response); +-} +- +-::grpc::Status DummyControllerStub::Shutdown(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerShutdownRequest& request, ::containerd::services::sandbox::v1::ControllerShutdownResponse* response) { +- if (g_controller_stub_mock == NULL) { +- return ::grpc::Status::OK; +- } +- return g_controller_stub_mock->Shutdown(context, request, response); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerCreateResponse>* DummyControllerStub::AsyncCreateRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerCreateRequest& request, ::grpc::CompletionQueue* cq) { +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->AsyncCreateRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerCreateResponse>* DummyControllerStub::PrepareAsyncCreateRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerCreateRequest& request, ::grpc::CompletionQueue* cq) { +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->PrepareAsyncCreateRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStartResponse>* DummyControllerStub::AsyncStartRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStartRequest& request, ::grpc::CompletionQueue* cq) { +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->AsyncStartRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStartResponse>* DummyControllerStub::PrepareAsyncStartRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStartRequest& request, ::grpc::CompletionQueue* cq) { +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->PrepareAsyncStartRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerPlatformResponse>* DummyControllerStub::AsyncPlatformRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerPlatformRequest& request, ::grpc::CompletionQueue* cq) { +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->AsyncPlatformRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerPlatformResponse>* DummyControllerStub::PrepareAsyncPlatformRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerPlatformRequest& request, ::grpc::CompletionQueue* cq) { +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->PrepareAsyncPlatformRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PrepareResponse>* DummyControllerStub::AsyncPrepareRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PrepareRequest& request, ::grpc::CompletionQueue* cq) { +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->AsyncPrepareRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PrepareResponse>* DummyControllerStub::PrepareAsyncPrepareRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PrepareRequest& request, ::grpc::CompletionQueue* cq) { +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->PrepareAsyncPrepareRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PurgeResponse>* DummyControllerStub::AsyncPurgeRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PurgeRequest& request, ::grpc::CompletionQueue* cq) { +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->AsyncPurgeRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PurgeResponse>* DummyControllerStub::PrepareAsyncPurgeRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PurgeRequest& request, ::grpc::CompletionQueue* cq) { +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->PrepareAsyncPurgeRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::UpdateResourcesResponse>* DummyControllerStub::AsyncUpdateResourcesRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::UpdateResourcesRequest& request, ::grpc::CompletionQueue* cq) { +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->AsyncUpdateResourcesRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::UpdateResourcesResponse>* DummyControllerStub::PrepareAsyncUpdateResourcesRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::UpdateResourcesRequest& request, ::grpc::CompletionQueue* cq) { +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->PrepareAsyncUpdateResourcesRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStopResponse>* DummyControllerStub::AsyncStopRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStopRequest& request, ::grpc::CompletionQueue* cq) { +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->AsyncStopRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStopResponse>* DummyControllerStub::PrepareAsyncStopRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStopRequest& request, ::grpc::CompletionQueue* cq) { +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->PrepareAsyncStopRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerWaitResponse>* DummyControllerStub::AsyncWaitRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerWaitRequest& request, ::grpc::CompletionQueue* cq) { +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->AsyncWaitRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerWaitResponse>* DummyControllerStub::PrepareAsyncWaitRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerWaitRequest& request, ::grpc::CompletionQueue* cq) { +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->PrepareAsyncWaitRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStatusResponse>* DummyControllerStub::AsyncStatusRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStatusRequest& request, ::grpc::CompletionQueue* cq) { +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->AsyncStatusRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStatusResponse>* DummyControllerStub::PrepareAsyncStatusRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStatusRequest& request, ::grpc::CompletionQueue* cq) { +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->PrepareAsyncStatusRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerShutdownResponse>* DummyControllerStub::AsyncShutdownRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerShutdownRequest& request, ::grpc::CompletionQueue* cq) { +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->AsyncShutdownRaw(context, request, cq); +-} +- +-::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerShutdownResponse>* DummyControllerStub::PrepareAsyncShutdownRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerShutdownRequest& request, ::grpc::CompletionQueue* cq) { +- if (g_controller_stub_mock == NULL) { +- return NULL; +- } +- return g_controller_stub_mock->PrepareAsyncShutdownRaw(context, request, cq); ++::grpc::Status DummyControllerStub::Create(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerCreateRequest &request, ++ ::containerd::services::sandbox::v1::ControllerCreateResponse* response) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return ::grpc::Status::OK; ++ } ++ return g_controller_stub_mock->Create(context, request, response); ++} ++ ++::grpc::Status DummyControllerStub::Start(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerStartRequest &request, ++ ::containerd::services::sandbox::v1::ControllerStartResponse* response) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return ::grpc::Status::OK; ++ } ++ return g_controller_stub_mock->Start(context, request, response); ++} ++ ++::grpc::Status DummyControllerStub::Platform(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerPlatformRequest &request, ++ ::containerd::services::sandbox::v1::ControllerPlatformResponse* response) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return ::grpc::Status::OK; ++ } ++ return g_controller_stub_mock->Platform(context, request, response); ++} ++ ++::grpc::Status DummyControllerStub::Prepare(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::PrepareRequest &request, ++ ::containerd::services::sandbox::v1::PrepareResponse* response) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return ::grpc::Status::OK; ++ } ++ return g_controller_stub_mock->Prepare(context, request, response); ++} ++ ++::grpc::Status DummyControllerStub::Purge(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::PurgeRequest &request, ++ ::containerd::services::sandbox::v1::PurgeResponse* response) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return ::grpc::Status::OK; ++ } ++ return g_controller_stub_mock->Purge(context, request, response); ++} ++ ++::grpc::Status DummyControllerStub::UpdateResources(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::UpdateResourcesRequest &request, ++ ::containerd::services::sandbox::v1::UpdateResourcesResponse* response) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return ::grpc::Status::OK; ++ } ++ return g_controller_stub_mock->UpdateResources(context, request, response); ++} ++ ++::grpc::Status DummyControllerStub::Stop(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerStopRequest &request, ++ ::containerd::services::sandbox::v1::ControllerStopResponse* response) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return ::grpc::Status::OK; ++ } ++ return g_controller_stub_mock->Stop(context, request, response); ++} ++ ++::grpc::Status DummyControllerStub::Wait(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerWaitRequest &request, ++ ::containerd::services::sandbox::v1::ControllerWaitResponse* response) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return ::grpc::Status::OK; ++ } ++ return g_controller_stub_mock->Wait(context, request, response); ++} ++ ++::grpc::Status DummyControllerStub::Status(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerStatusRequest &request, ++ ::containerd::services::sandbox::v1::ControllerStatusResponse* response) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return ::grpc::Status::OK; ++ } ++ return g_controller_stub_mock->Status(context, request, response); ++} ++ ++::grpc::Status DummyControllerStub::Shutdown(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerShutdownRequest &request, ++ ::containerd::services::sandbox::v1::ControllerShutdownResponse* response) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return ::grpc::Status::OK; ++ } ++ return g_controller_stub_mock->Shutdown(context, request, response); ++} ++ ++::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerCreateResponse>* ++DummyControllerStub::AsyncCreateRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerCreateRequest &request, ::grpc::CompletionQueue* cq) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return NULL; ++ } ++ return g_controller_stub_mock->AsyncCreateRaw(context, request, cq); ++} ++ ++::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerCreateResponse>* ++DummyControllerStub::PrepareAsyncCreateRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerCreateRequest &request, ::grpc::CompletionQueue* cq) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return NULL; ++ } ++ return g_controller_stub_mock->PrepareAsyncCreateRaw(context, request, cq); ++} ++ ++::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStartResponse>* ++DummyControllerStub::AsyncStartRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerStartRequest &request, ::grpc::CompletionQueue* cq) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return NULL; ++ } ++ return g_controller_stub_mock->AsyncStartRaw(context, request, cq); ++} ++ ++::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStartResponse>* ++DummyControllerStub::PrepareAsyncStartRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerStartRequest &request, ::grpc::CompletionQueue* cq) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return NULL; ++ } ++ return g_controller_stub_mock->PrepareAsyncStartRaw(context, request, cq); ++} ++ ++::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerPlatformResponse>* ++DummyControllerStub::AsyncPlatformRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerPlatformRequest &request, ::grpc::CompletionQueue* cq) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return NULL; ++ } ++ return g_controller_stub_mock->AsyncPlatformRaw(context, request, cq); ++} ++ ++::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerPlatformResponse>* ++DummyControllerStub::PrepareAsyncPlatformRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerPlatformRequest &request, ::grpc::CompletionQueue* cq) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return NULL; ++ } ++ return g_controller_stub_mock->PrepareAsyncPlatformRaw(context, request, cq); ++} ++ ++::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PrepareResponse>* ++DummyControllerStub::AsyncPrepareRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::PrepareRequest &request, ::grpc::CompletionQueue* cq) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return NULL; ++ } ++ return g_controller_stub_mock->AsyncPrepareRaw(context, request, cq); ++} ++ ++::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PrepareResponse>* ++DummyControllerStub::PrepareAsyncPrepareRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::PrepareRequest &request, ::grpc::CompletionQueue* cq) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return NULL; ++ } ++ return g_controller_stub_mock->PrepareAsyncPrepareRaw(context, request, cq); ++} ++ ++::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PurgeResponse>* ++DummyControllerStub::AsyncPurgeRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::PurgeRequest &request, ::grpc::CompletionQueue* cq) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return NULL; ++ } ++ return g_controller_stub_mock->AsyncPurgeRaw(context, request, cq); ++} ++ ++::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PurgeResponse>* ++DummyControllerStub::PrepareAsyncPurgeRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::PurgeRequest &request, ::grpc::CompletionQueue* cq) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return NULL; ++ } ++ return g_controller_stub_mock->PrepareAsyncPurgeRaw(context, request, cq); ++} ++ ++::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::UpdateResourcesResponse>* ++DummyControllerStub::AsyncUpdateResourcesRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::UpdateResourcesRequest &request, ::grpc::CompletionQueue* cq) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return NULL; ++ } ++ return g_controller_stub_mock->AsyncUpdateResourcesRaw(context, request, cq); ++} ++ ++::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::UpdateResourcesResponse>* ++DummyControllerStub::PrepareAsyncUpdateResourcesRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::UpdateResourcesRequest &request, ::grpc::CompletionQueue* cq) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return NULL; ++ } ++ return g_controller_stub_mock->PrepareAsyncUpdateResourcesRaw(context, request, cq); ++} ++ ++::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStopResponse>* ++DummyControllerStub::AsyncStopRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerStopRequest &request, ::grpc::CompletionQueue* cq) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return NULL; ++ } ++ return g_controller_stub_mock->AsyncStopRaw(context, request, cq); ++} ++ ++::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStopResponse>* ++DummyControllerStub::PrepareAsyncStopRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerStopRequest &request, ::grpc::CompletionQueue* cq) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return NULL; ++ } ++ return g_controller_stub_mock->PrepareAsyncStopRaw(context, request, cq); ++} ++ ++::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerWaitResponse>* ++DummyControllerStub::AsyncWaitRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerWaitRequest &request, ::grpc::CompletionQueue* cq) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return NULL; ++ } ++ return g_controller_stub_mock->AsyncWaitRaw(context, request, cq); ++} ++ ++::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerWaitResponse>* ++DummyControllerStub::PrepareAsyncWaitRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerWaitRequest &request, ::grpc::CompletionQueue* cq) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return NULL; ++ } ++ return g_controller_stub_mock->PrepareAsyncWaitRaw(context, request, cq); ++} ++ ++::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStatusResponse>* ++DummyControllerStub::AsyncStatusRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerStatusRequest &request, ::grpc::CompletionQueue* cq) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return NULL; ++ } ++ return g_controller_stub_mock->AsyncStatusRaw(context, request, cq); ++} ++ ++::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStatusResponse>* ++DummyControllerStub::PrepareAsyncStatusRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerStatusRequest &request, ::grpc::CompletionQueue* cq) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return NULL; ++ } ++ return g_controller_stub_mock->PrepareAsyncStatusRaw(context, request, cq); ++} ++ ++::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerShutdownResponse>* ++DummyControllerStub::AsyncShutdownRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerShutdownRequest &request, ::grpc::CompletionQueue* cq) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return NULL; ++ } ++ return g_controller_stub_mock->AsyncShutdownRaw(context, request, cq); ++} ++ ++::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerShutdownResponse>* ++DummyControllerStub::PrepareAsyncShutdownRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerShutdownRequest &request, ::grpc::CompletionQueue* cq) ++{ ++ if (g_controller_stub_mock == NULL) { ++ return NULL; ++ } ++ return g_controller_stub_mock->PrepareAsyncShutdownRaw(context, request, cq); + } +diff --git a/test/mocks/controller_stub_mock.h b/test/mocks/controller_stub_mock.h +index b3920bf8..85cb82bb 100644 +--- a/test/mocks/controller_stub_mock.h ++++ b/test/mocks/controller_stub_mock.h +@@ -23,73 +23,212 @@ + // MockControllerStub is a mock implementation of the Controller::StubInterface interface. + class MockControllerStub { + public: +- MOCK_METHOD3(Create, ::grpc::Status(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerCreateRequest& request, ::containerd::services::sandbox::v1::ControllerCreateResponse* response)); +- MOCK_METHOD3(Start, ::grpc::Status(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStartRequest& request, ::containerd::services::sandbox::v1::ControllerStartResponse* response)); +- MOCK_METHOD3(Platform, ::grpc::Status(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerPlatformRequest& request, ::containerd::services::sandbox::v1::ControllerPlatformResponse* response)); +- MOCK_METHOD3(Prepare, ::grpc::Status(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PrepareRequest& request, ::containerd::services::sandbox::v1::PrepareResponse* response)); +- MOCK_METHOD3(Purge, ::grpc::Status(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PurgeRequest& request, ::containerd::services::sandbox::v1::PurgeResponse* response)); +- MOCK_METHOD3(UpdateResources, ::grpc::Status(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::UpdateResourcesRequest& request, ::containerd::services::sandbox::v1::UpdateResourcesResponse* response)); +- MOCK_METHOD3(Stop, ::grpc::Status(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStopRequest& request, ::containerd::services::sandbox::v1::ControllerStopResponse* response)); +- MOCK_METHOD3(Wait, ::grpc::Status(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerWaitRequest& request, ::containerd::services::sandbox::v1::ControllerWaitResponse* response)); +- MOCK_METHOD3(Status, ::grpc::Status(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStatusRequest& request, ::containerd::services::sandbox::v1::ControllerStatusResponse* response)); +- MOCK_METHOD3(Shutdown, ::grpc::Status(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerShutdownRequest& request, ::containerd::services::sandbox::v1::ControllerShutdownResponse* response)); +- MOCK_METHOD3(AsyncCreateRaw, ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerCreateResponse>*(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerCreateRequest& request, ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(PrepareAsyncCreateRaw, ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerCreateResponse>*(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerCreateRequest& request, ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(AsyncStartRaw, ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStartResponse>*(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStartRequest& request, ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(PrepareAsyncStartRaw, ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStartResponse>*(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStartRequest& request, ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(AsyncPlatformRaw, ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerPlatformResponse>*(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerPlatformRequest& request, ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(PrepareAsyncPlatformRaw, ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerPlatformResponse>*(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerPlatformRequest& request, ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(AsyncPrepareRaw, ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PrepareResponse>*(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PrepareRequest& request, ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(PrepareAsyncPrepareRaw, ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PrepareResponse>*(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PrepareRequest& request, ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(AsyncPurgeRaw, ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PurgeResponse>*(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PurgeRequest& request, ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(PrepareAsyncPurgeRaw, ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PurgeResponse>*(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PurgeRequest& request, ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(AsyncUpdateResourcesRaw, ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::UpdateResourcesResponse>*(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::UpdateResourcesRequest& request, ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(PrepareAsyncUpdateResourcesRaw, ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::UpdateResourcesResponse>*(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::UpdateResourcesRequest& request, ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(AsyncStopRaw, ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStopResponse>*(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStopRequest& request, ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(PrepareAsyncStopRaw, ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStopResponse>*(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStopRequest& request, ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(AsyncWaitRaw, ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerWaitResponse>*(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerWaitRequest& request, ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(PrepareAsyncWaitRaw, ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerWaitResponse>*(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerWaitRequest& request, ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(AsyncStatusRaw, ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStatusResponse>*(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStatusRequest& request, ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(PrepareAsyncStatusRaw, ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStatusResponse>*(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStatusRequest& request, ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(AsyncShutdownRaw, ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerShutdownResponse>*(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerShutdownRequest& request, ::grpc::CompletionQueue* cq)); +- MOCK_METHOD3(PrepareAsyncShutdownRaw, ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerShutdownResponse>*(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerShutdownRequest& request, ::grpc::CompletionQueue* cq)); ++ MOCK_METHOD3(Create, ::grpc::Status(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerCreateRequest &request, ++ ::containerd::services::sandbox::v1::ControllerCreateResponse* response)); ++ MOCK_METHOD3(Start, ::grpc::Status(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerStartRequest &request, ++ ::containerd::services::sandbox::v1::ControllerStartResponse* response)); ++ MOCK_METHOD3(Platform, ::grpc::Status(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerPlatformRequest &request, ++ ::containerd::services::sandbox::v1::ControllerPlatformResponse* response)); ++ MOCK_METHOD3(Prepare, ::grpc::Status(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::PrepareRequest &request, ++ ::containerd::services::sandbox::v1::PrepareResponse* response)); ++ MOCK_METHOD3(Purge, ::grpc::Status(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::PurgeRequest &request, ++ ::containerd::services::sandbox::v1::PurgeResponse* response)); ++ MOCK_METHOD3(UpdateResources, ::grpc::Status(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::UpdateResourcesRequest &request, ++ ::containerd::services::sandbox::v1::UpdateResourcesResponse* response)); ++ MOCK_METHOD3(Stop, ::grpc::Status(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerStopRequest &request, ++ ::containerd::services::sandbox::v1::ControllerStopResponse* response)); ++ MOCK_METHOD3(Wait, ::grpc::Status(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerWaitRequest &request, ++ ::containerd::services::sandbox::v1::ControllerWaitResponse* response)); ++ MOCK_METHOD3(Status, ::grpc::Status(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerStatusRequest &request, ++ ::containerd::services::sandbox::v1::ControllerStatusResponse* response)); ++ MOCK_METHOD3(Shutdown, ::grpc::Status(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerShutdownRequest &request, ++ ::containerd::services::sandbox::v1::ControllerShutdownResponse* response)); ++ MOCK_METHOD3(AsyncCreateRaw, ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerCreateResponse>* ++ (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerCreateRequest &request, ++ ::grpc::CompletionQueue* cq)); ++ MOCK_METHOD3(PrepareAsyncCreateRaw, ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerCreateResponse>* ++ (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerCreateRequest &request, ++ ::grpc::CompletionQueue* cq)); ++ MOCK_METHOD3(AsyncStartRaw, ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStartResponse>* ++ (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStartRequest &request, ++ ::grpc::CompletionQueue* cq)); ++ MOCK_METHOD3(PrepareAsyncStartRaw, ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStartResponse>* ++ (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStartRequest &request, ++ ::grpc::CompletionQueue* cq)); ++ MOCK_METHOD3(AsyncPlatformRaw, ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerPlatformResponse>* ++ (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerPlatformRequest &request, ++ ::grpc::CompletionQueue* cq)); ++ MOCK_METHOD3(PrepareAsyncPlatformRaw, ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerPlatformResponse>* ++ (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerPlatformRequest &request, ++ ::grpc::CompletionQueue* cq)); ++ MOCK_METHOD3(AsyncPrepareRaw, ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PrepareResponse>* ++ (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PrepareRequest &request, ++ ::grpc::CompletionQueue* cq)); ++ MOCK_METHOD3(PrepareAsyncPrepareRaw, ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PrepareResponse>* ++ (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PrepareRequest &request, ++ ::grpc::CompletionQueue* cq)); ++ MOCK_METHOD3(AsyncPurgeRaw, ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PurgeResponse>* ++ (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PurgeRequest &request, ++ ::grpc::CompletionQueue* cq)); ++ MOCK_METHOD3(PrepareAsyncPurgeRaw, ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PurgeResponse>* ++ (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PurgeRequest &request, ++ ::grpc::CompletionQueue* cq)); ++ MOCK_METHOD3(AsyncUpdateResourcesRaw, ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::UpdateResourcesResponse>* ++ (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::UpdateResourcesRequest &request, ++ ::grpc::CompletionQueue* cq)); ++ MOCK_METHOD3(PrepareAsyncUpdateResourcesRaw, ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::UpdateResourcesResponse>* ++ (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::UpdateResourcesRequest &request, ++ ::grpc::CompletionQueue* cq)); ++ MOCK_METHOD3(AsyncStopRaw, ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStopResponse>* ++ (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStopRequest &request, ++ ::grpc::CompletionQueue* cq)); ++ MOCK_METHOD3(PrepareAsyncStopRaw, ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStopResponse>* ++ (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStopRequest &request, ++ ::grpc::CompletionQueue* cq)); ++ MOCK_METHOD3(AsyncWaitRaw, ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerWaitResponse>* ++ (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerWaitRequest &request, ++ ::grpc::CompletionQueue* cq)); ++ MOCK_METHOD3(PrepareAsyncWaitRaw, ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerWaitResponse>* ++ (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerWaitRequest &request, ++ ::grpc::CompletionQueue* cq)); ++ MOCK_METHOD3(AsyncStatusRaw, ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStatusResponse>* ++ (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStatusRequest &request, ++ ::grpc::CompletionQueue* cq)); ++ MOCK_METHOD3(PrepareAsyncStatusRaw, ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStatusResponse>* ++ (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStatusRequest &request, ++ ::grpc::CompletionQueue* cq)); ++ MOCK_METHOD3(AsyncShutdownRaw, ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerShutdownResponse>* ++ (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerShutdownRequest &request, ++ ::grpc::CompletionQueue* cq)); ++ MOCK_METHOD3(PrepareAsyncShutdownRaw, ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerShutdownResponse>* ++ (::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerShutdownRequest &request, ++ ::grpc::CompletionQueue* cq)); + }; + + class DummyControllerStub: public containerd::services::sandbox::v1::Controller::StubInterface { + public: + DummyControllerStub() = default; + ~DummyControllerStub() = default; +- ::grpc::Status Create(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerCreateRequest& request, ::containerd::services::sandbox::v1::ControllerCreateResponse* response) override; +- ::grpc::Status Start(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStartRequest& request, ::containerd::services::sandbox::v1::ControllerStartResponse* response) override; +- ::grpc::Status Platform(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerPlatformRequest& request, ::containerd::services::sandbox::v1::ControllerPlatformResponse* response) override; +- ::grpc::Status Prepare(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PrepareRequest& request, ::containerd::services::sandbox::v1::PrepareResponse* response) override; +- ::grpc::Status Purge(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PurgeRequest& request, ::containerd::services::sandbox::v1::PurgeResponse* response) override; +- ::grpc::Status UpdateResources(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::UpdateResourcesRequest& request, ::containerd::services::sandbox::v1::UpdateResourcesResponse* response) override; +- ::grpc::Status Stop(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStopRequest& request, ::containerd::services::sandbox::v1::ControllerStopResponse* response) override; +- ::grpc::Status Wait(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerWaitRequest& request, ::containerd::services::sandbox::v1::ControllerWaitResponse* response) override; +- ::grpc::Status Status(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStatusRequest& request, ::containerd::services::sandbox::v1::ControllerStatusResponse* response) override; +- ::grpc::Status Shutdown(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerShutdownRequest& request, ::containerd::services::sandbox::v1::ControllerShutdownResponse* response) override; ++ ::grpc::Status Create(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerCreateRequest &request, ++ ::containerd::services::sandbox::v1::ControllerCreateResponse* response) override; ++ ::grpc::Status Start(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerStartRequest &request, ++ ::containerd::services::sandbox::v1::ControllerStartResponse* response) override; ++ ::grpc::Status Platform(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerPlatformRequest &request, ++ ::containerd::services::sandbox::v1::ControllerPlatformResponse* response) override; ++ ::grpc::Status Prepare(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::PrepareRequest &request, ++ ::containerd::services::sandbox::v1::PrepareResponse* response) override; ++ ::grpc::Status Purge(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PurgeRequest &request, ++ ::containerd::services::sandbox::v1::PurgeResponse* response) override; ++ ::grpc::Status UpdateResources(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::UpdateResourcesRequest &request, ++ ::containerd::services::sandbox::v1::UpdateResourcesResponse* response) override; ++ ::grpc::Status Stop(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerStopRequest &request, ++ ::containerd::services::sandbox::v1::ControllerStopResponse* response) override; ++ ::grpc::Status Wait(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerWaitRequest &request, ++ ::containerd::services::sandbox::v1::ControllerWaitResponse* response) override; ++ ::grpc::Status Status(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerStatusRequest &request, ++ ::containerd::services::sandbox::v1::ControllerStatusResponse* response) override; ++ ::grpc::Status Shutdown(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerShutdownRequest &request, ++ ::containerd::services::sandbox::v1::ControllerShutdownResponse* response) override; + private: +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerCreateResponse>* AsyncCreateRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerCreateRequest& request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerCreateResponse>* PrepareAsyncCreateRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerCreateRequest& request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStartResponse>* AsyncStartRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStartRequest& request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStartResponse>* PrepareAsyncStartRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStartRequest& request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerPlatformResponse>* AsyncPlatformRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerPlatformRequest& request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerPlatformResponse>* PrepareAsyncPlatformRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerPlatformRequest& request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PrepareResponse>* AsyncPrepareRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PrepareRequest& request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PrepareResponse>* PrepareAsyncPrepareRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PrepareRequest& request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PurgeResponse>* AsyncPurgeRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PurgeRequest& request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PurgeResponse>* PrepareAsyncPurgeRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PurgeRequest& request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::UpdateResourcesResponse>* AsyncUpdateResourcesRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::UpdateResourcesRequest& request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::UpdateResourcesResponse>* PrepareAsyncUpdateResourcesRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::UpdateResourcesRequest& request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStopResponse>* AsyncStopRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStopRequest& request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStopResponse>* PrepareAsyncStopRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStopRequest& request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerWaitResponse>* AsyncWaitRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerWaitRequest& request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerWaitResponse>* PrepareAsyncWaitRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerWaitRequest& request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStatusResponse>* AsyncStatusRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStatusRequest& request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStatusResponse>* PrepareAsyncStatusRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStatusRequest& request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerShutdownResponse>* AsyncShutdownRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerShutdownRequest& request, ::grpc::CompletionQueue* cq) override; +- ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerShutdownResponse>* PrepareAsyncShutdownRaw(::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerShutdownRequest& request, ::grpc::CompletionQueue* cq) override; ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerCreateResponse>* ++ AsyncCreateRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerCreateRequest &request, ::grpc::CompletionQueue* cq) override; ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerCreateResponse>* ++ PrepareAsyncCreateRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerCreateRequest &request, ::grpc::CompletionQueue* cq) override; ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStartResponse>* ++ AsyncStartRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerStartRequest &request, ::grpc::CompletionQueue* cq) override; ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStartResponse>* ++ PrepareAsyncStartRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerStartRequest &request, ::grpc::CompletionQueue* cq) override; ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerPlatformResponse>* ++ AsyncPlatformRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerPlatformRequest &request, ::grpc::CompletionQueue* cq) override; ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerPlatformResponse>* ++ PrepareAsyncPlatformRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerPlatformRequest &request, ::grpc::CompletionQueue* cq) override; ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PrepareResponse>* AsyncPrepareRaw( ++ ::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PrepareRequest &request, ++ ::grpc::CompletionQueue* cq) override; ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PrepareResponse>* ++ PrepareAsyncPrepareRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::PrepareRequest &request, ::grpc::CompletionQueue* cq) override; ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PurgeResponse>* AsyncPurgeRaw( ++ ::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PurgeRequest &request, ++ ::grpc::CompletionQueue* cq) override; ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::PurgeResponse>* PrepareAsyncPurgeRaw( ++ ::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::PurgeRequest &request, ++ ::grpc::CompletionQueue* cq) override; ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::UpdateResourcesResponse>* ++ AsyncUpdateResourcesRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::UpdateResourcesRequest &request, ::grpc::CompletionQueue* cq) override; ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::UpdateResourcesResponse>* ++ PrepareAsyncUpdateResourcesRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::UpdateResourcesRequest &request, ::grpc::CompletionQueue* cq) override; ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStopResponse>* AsyncStopRaw( ++ ::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerStopRequest &request, ++ ::grpc::CompletionQueue* cq) override; ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStopResponse>* ++ PrepareAsyncStopRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerStopRequest &request, ::grpc::CompletionQueue* cq) override; ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerWaitResponse>* AsyncWaitRaw( ++ ::grpc::ClientContext* context, const ::containerd::services::sandbox::v1::ControllerWaitRequest &request, ++ ::grpc::CompletionQueue* cq) override; ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerWaitResponse>* ++ PrepareAsyncWaitRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerWaitRequest &request, ::grpc::CompletionQueue* cq) override; ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStatusResponse>* ++ AsyncStatusRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerStatusRequest &request, ::grpc::CompletionQueue* cq) override; ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerStatusResponse>* ++ PrepareAsyncStatusRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerStatusRequest &request, ::grpc::CompletionQueue* cq) override; ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerShutdownResponse>* ++ AsyncShutdownRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerShutdownRequest &request, ::grpc::CompletionQueue* cq) override; ++ ::grpc::ClientAsyncResponseReaderInterface< ::containerd::services::sandbox::v1::ControllerShutdownResponse>* ++ PrepareAsyncShutdownRaw(::grpc::ClientContext* context, ++ const ::containerd::services::sandbox::v1::ControllerShutdownRequest &request, ::grpc::CompletionQueue* cq) override; + }; + + std::unique_ptr<DummyControllerStub> NewDummyControllerStub(); +diff --git a/test/mocks/grpc_async_wait_call_mock.cc b/test/mocks/grpc_async_wait_call_mock.cc +index 034cc65c..5eef1794 100644 +--- a/test/mocks/grpc_async_wait_call_mock.cc ++++ b/test/mocks/grpc_async_wait_call_mock.cc +@@ -34,7 +34,8 @@ SandboxerAsyncWaitCall::SandboxerAsyncWaitCall(std::shared_ptr<SandboxStatusCall + m_remove = false; + } + +-auto SandboxerAsyncWaitCall::Call(containerd::services::sandbox::v1::Controller::StubInterface &stub, grpc::CompletionQueue &cq) -> bool ++auto SandboxerAsyncWaitCall::Call(containerd::services::sandbox::v1::Controller::StubInterface &stub, ++ grpc::CompletionQueue &cq) -> bool + { + if (g_sandboxer_async_wait_call_mock == NULL) { + return true; +diff --git a/test/mocks/grpc_async_wait_call_mock.h b/test/mocks/grpc_async_wait_call_mock.h +index eb890ced..c79f998e 100644 +--- a/test/mocks/grpc_async_wait_call_mock.h ++++ b/test/mocks/grpc_async_wait_call_mock.h +@@ -29,7 +29,7 @@ public: + MOCK_METHOD2(SandboxExitCallback, void(bool statusOK, const ControllerExitInfo &exitInfo)); + MOCK_METHOD0(SandboxPendingCallback, void()); + MOCK_METHOD0(SandboxReadyCallback, void()); +- MOCK_METHOD0(GetSandboxId, const std::string &()); ++ MOCK_METHOD0(GetSandboxId, const std::string & ()); + }; + + void MockSandboxerAsyncWaitCall_SetMock(std::shared_ptr<SandboxerAsyncWaitCallMock> mock); +diff --git a/test/mocks/grpc_sandboxer_client_mock.cc b/test/mocks/grpc_sandboxer_client_mock.cc +index 0e57cfe5..03df9048 100644 +--- a/test/mocks/grpc_sandboxer_client_mock.cc ++++ b/test/mocks/grpc_sandboxer_client_mock.cc +@@ -58,7 +58,8 @@ auto SandboxerClient::Start(const std::string &sandboxId, ControllerSandboxInfo + return g_sandboxer_client_mock->Start(sandboxId, sandboxInfo, error); + } + +-auto SandboxerClient::Platform(const std::string &sandboxId, ControllerPlatformInfo &platformInfo, Errors &error) -> bool ++auto SandboxerClient::Platform(const std::string &sandboxId, ControllerPlatformInfo &platformInfo, ++ Errors &error) -> bool + { + if (g_sandboxer_client_mock == NULL) { + return true; +@@ -66,7 +67,8 @@ auto SandboxerClient::Platform(const std::string &sandboxId, ControllerPlatformI + return g_sandboxer_client_mock->Platform(sandboxId, platformInfo, error); + } + +-auto SandboxerClient::Prepare(const std::string &sandboxId, const ControllerPrepareParams ¶ms, std::string &bundle, Errors &error) -> bool ++auto SandboxerClient::Prepare(const std::string &sandboxId, const ControllerPrepareParams ¶ms, std::string &bundle, ++ Errors &error) -> bool + { + if (g_sandboxer_client_mock == NULL) { + return true; +@@ -83,7 +85,8 @@ auto SandboxerClient::Purge(const std::string &sandboxId, const std::string &con + return g_sandboxer_client_mock->Purge(sandboxId, containerId, execId, error); + } + +-auto SandboxerClient::UpdateResources(const std::string &sandboxId, const ControllerUpdateResourcesParams ¶ms, Errors &error) -> bool ++auto SandboxerClient::UpdateResources(const std::string &sandboxId, const ControllerUpdateResourcesParams ¶ms, ++ Errors &error) -> bool + { + if (g_sandboxer_client_mock == NULL) { + return true; +@@ -99,7 +102,8 @@ auto SandboxerClient::Stop(const std::string &sandboxId, uint32_t timeoutSecs, E + return g_sandboxer_client_mock->Stop(sandboxId, timeoutSecs, error); + } + +-auto SandboxerClient::Wait(std::shared_ptr<SandboxStatusCallback> cb, const std::string &sandboxId, Errors &error) -> bool ++auto SandboxerClient::Wait(std::shared_ptr<SandboxStatusCallback> cb, const std::string &sandboxId, ++ Errors &error) -> bool + { + if (g_sandboxer_client_mock == NULL) { + return true; +@@ -107,7 +111,8 @@ auto SandboxerClient::Wait(std::shared_ptr<SandboxStatusCallback> cb, const std: + return g_sandboxer_client_mock->Wait(cb, sandboxId, error); + } + +-auto SandboxerClient::Status(const std::string &sandboxId, bool verbose, ControllerSandboxStatus &sandboxStatus, Errors &error) -> bool ++auto SandboxerClient::Status(const std::string &sandboxId, bool verbose, ControllerSandboxStatus &sandboxStatus, ++ Errors &error) -> bool + { + if (g_sandboxer_client_mock == NULL) { + return true; +diff --git a/test/mocks/grpc_sandboxer_client_mock.h b/test/mocks/grpc_sandboxer_client_mock.h +index a3dcd690..ac06462a 100644 +--- a/test/mocks/grpc_sandboxer_client_mock.h ++++ b/test/mocks/grpc_sandboxer_client_mock.h +@@ -28,12 +28,16 @@ public: + MOCK_METHOD3(Create, bool(const std::string &sandboxId, const ControllerCreateParams ¶ms, Errors &error)); + MOCK_METHOD3(Start, bool(const std::string &sandboxId, ControllerSandboxInfo &sandboxInfo, Errors &error)); + MOCK_METHOD3(Platform, bool(const std::string &sandboxId, ControllerPlatformInfo &platformInfo, Errors &error)); +- MOCK_METHOD4(Prepare, bool(const std::string &sandboxId, const ControllerPrepareParams ¶ms, std::string &bundle, Errors &error)); +- MOCK_METHOD4(Purge, bool(const std::string &sandboxId, const std::string &containerId, const std::string &execId, Errors &error)); +- MOCK_METHOD3(UpdateResources, bool(const std::string &sandboxId, const ControllerUpdateResourcesParams ¶ms, Errors &error)); ++ MOCK_METHOD4(Prepare, bool(const std::string &sandboxId, const ControllerPrepareParams ¶ms, std::string &bundle, ++ Errors &error)); ++ MOCK_METHOD4(Purge, bool(const std::string &sandboxId, const std::string &containerId, const std::string &execId, ++ Errors &error)); ++ MOCK_METHOD3(UpdateResources, bool(const std::string &sandboxId, const ControllerUpdateResourcesParams ¶ms, ++ Errors &error)); + MOCK_METHOD3(Stop, bool(const std::string &sandboxId, uint32_t timeoutSecs, Errors &error)); + MOCK_METHOD3(Wait, bool(std::shared_ptr<SandboxStatusCallback> cb, const std::string &sandboxId, Errors &error)); +- MOCK_METHOD4(Status, bool(const std::string &sandboxId, bool verbose, ControllerSandboxStatus &sandboxStatus, Errors &error)); ++ MOCK_METHOD4(Status, bool(const std::string &sandboxId, bool verbose, ControllerSandboxStatus &sandboxStatus, ++ Errors &error)); + MOCK_METHOD2(Shutdown, bool(const std::string &sandboxId, Errors &error)); + }; + +diff --git a/test/mocks/grpc_sandboxer_monitor_mock.cc b/test/mocks/grpc_sandboxer_monitor_mock.cc +index e307e0ae..b3ddda87 100644 +--- a/test/mocks/grpc_sandboxer_monitor_mock.cc ++++ b/test/mocks/grpc_sandboxer_monitor_mock.cc +@@ -19,7 +19,7 @@ static std::shared_ptr<SandboxerClientMonitorMock> g_sandboxer_client_monitor_mo + + + SandboxerClientMonitor::SandboxerClientMonitor(std::shared_ptr<grpc::Channel> channel, const std::string &sandboxer): +- m_channel(channel), m_sandboxer(sandboxer) ,m_teardown(false) {} ++ m_channel(channel), m_sandboxer(sandboxer), m_teardown(false) {} + + void SandboxerClientMonitor::Start() + { +diff --git a/test/mocks/isulad_config_mock.h b/test/mocks/isulad_config_mock.h +index 6a92fc13..6c6ff7f1 100644 +--- a/test/mocks/isulad_config_mock.h ++++ b/test/mocks/isulad_config_mock.h +@@ -40,7 +40,7 @@ public: + MOCK_METHOD0(InitIsuladDaemonConstants, int (void)); + MOCK_METHOD0(GetIsuladDaemonConstants, isulad_daemon_constants * (void)); + MOCK_METHOD0(ConfGetIsuladUsernsRemap, char *(void)); +- MOCK_METHOD0(ConfGetServerConf, struct service_arguments *(void)); ++ MOCK_METHOD0(ConfGetServerConf, struct service_arguments * (void)); + MOCK_METHOD0(ConfGetSandboxRootPath, char *(void)); + MOCK_METHOD0(ConfGetSandboxStatePath, char *(void)); + }; +diff --git a/test/mocks/sandbox_mock.cc b/test/mocks/sandbox_mock.cc +index e5aefdda..9db57a93 100644 +--- a/test/mocks/sandbox_mock.cc ++++ b/test/mocks/sandbox_mock.cc +@@ -77,7 +77,7 @@ const std::string &Sandbox::GetRuntimeHandle() const + return defaultStr; + } + +-const runtime::v1::PodSandboxConfig & Sandbox::GetSandboxConfig() const ++const runtime::v1::PodSandboxConfig &Sandbox::GetSandboxConfig() const + { + if (g_sandbox_mock != nullptr) { + return g_sandbox_mock->GetSandboxConfig(); +diff --git a/test/mocks/sandbox_mock.h b/test/mocks/sandbox_mock.h +index 341042e9..98f40ad2 100644 +--- a/test/mocks/sandbox_mock.h ++++ b/test/mocks/sandbox_mock.h +@@ -31,7 +31,7 @@ public: + MOCK_METHOD0(GetName, const std::string & ()); + MOCK_METHOD0(GetSandboxer, const std::string & ()); + MOCK_METHOD0(GetRuntimeHandle, const std::string & ()); +- MOCK_METHOD0(GetSandboxConfig, const runtime::v1::PodSandboxConfig &()); ++ MOCK_METHOD0(GetSandboxConfig, const runtime::v1::PodSandboxConfig & ()); + MOCK_METHOD0(GetMutableSandboxConfig, std::shared_ptr<runtime::v1::PodSandboxConfig>()); + MOCK_METHOD0(GetRootDir, const std::string & ()); + MOCK_METHOD0(GetStateDir, std::string & ()); +diff --git a/test/mocks/service_container_api_mock.h b/test/mocks/service_container_api_mock.h +index 350a2fff..9a39f483 100644 +--- a/test/mocks/service_container_api_mock.h ++++ b/test/mocks/service_container_api_mock.h +@@ -23,7 +23,7 @@ + + class MockServiceContainerApi { + public: +- MOCK_METHOD3(InspectContainer, container_inspect *(const char *id, int timeout, bool with_host_config)); ++ MOCK_METHOD3(InspectContainer, container_inspect * (const char *id, int timeout, bool with_host_config)); + }; + + void MockServiceContainerApi_SetMock(std::shared_ptr<MockServiceContainerApi> mock); +diff --git a/test/mocks/shim_controller_mock.cc b/test/mocks/shim_controller_mock.cc +index 88694dbe..e0ffc563 100644 +--- a/test/mocks/shim_controller_mock.cc ++++ b/test/mocks/shim_controller_mock.cc +@@ -117,7 +117,8 @@ bool ShimController::Wait(std::shared_ptr<SandboxStatusCallback> cb, const std:: + return true; + } + +-std::unique_ptr<ControllerSandboxStatus> ShimController::Status(const std::string &sandboxId, bool verbose, Errors &error) ++std::unique_ptr<ControllerSandboxStatus> ShimController::Status(const std::string &sandboxId, bool verbose, ++ Errors &error) + { + if (g_shim_controller_mock != nullptr) { + return g_shim_controller_mock->Status(sandboxId, verbose, error); +@@ -133,7 +134,8 @@ bool ShimController::Shutdown(const std::string &sandboxId, Errors &error) + return true; + } + +-bool ShimController::UpdateNetworkSettings(const std::string &sandboxId, const std::string &networkSettings, Errors &error) ++bool ShimController::UpdateNetworkSettings(const std::string &sandboxId, const std::string &networkSettings, ++ Errors &error) + { + if (g_shim_controller_mock != nullptr) { + return g_shim_controller_mock->UpdateNetworkSettings(sandboxId, networkSettings, error); +diff --git a/test/mocks/shim_controller_mock.h b/test/mocks/shim_controller_mock.h +index 3be05246..6d0de591 100644 +--- a/test/mocks/shim_controller_mock.h ++++ b/test/mocks/shim_controller_mock.h +@@ -39,15 +39,17 @@ public: + const ControllerPrepareParams ¶ms, + Errors &error)); + MOCK_METHOD4(Purge, bool(const std::string &sandboxId, const std::string &containerId, +- const std::string &execId, Errors &error)); ++ const std::string &execId, Errors &error)); + MOCK_METHOD3(UpdateResources, bool(const std::string &sandboxId, + const ControllerUpdateResourcesParams ¶ms, + Errors &error)); + MOCK_METHOD3(Stop, bool(const std::string &sandboxId, uint32_t timeoutSecs, Errors &error)); + MOCK_METHOD3(Wait, bool(std::shared_ptr<SandboxStatusCallback> cb, const std::string &sandboxId, Errors &error)); +- MOCK_METHOD3(Status, std::unique_ptr<ControllerSandboxStatus>(const std::string &sandboxId, bool verbose, Errors &error)); ++ MOCK_METHOD3(Status, std::unique_ptr<ControllerSandboxStatus>(const std::string &sandboxId, bool verbose, ++ Errors &error)); + MOCK_METHOD2(Shutdown, bool(const std::string &sandboxId, Errors &error)); +- MOCK_METHOD3(UpdateNetworkSettings, bool(const std::string &sandboxId, const std::string &networkSettings, Errors &error)); ++ MOCK_METHOD3(UpdateNetworkSettings, bool(const std::string &sandboxId, const std::string &networkSettings, ++ Errors &error)); + }; + + void MockShimController_SetMock(std::shared_ptr<MockShimController> mock); +diff --git a/test/network/cni_operate/cni_operate_ut.cc b/test/network/cni_operate/cni_operate_ut.cc +index 4194641b..f61fee57 100644 +--- a/test/network/cni_operate/cni_operate_ut.cc ++++ b/test/network/cni_operate/cni_operate_ut.cc +@@ -47,28 +47,28 @@ using namespace std; + + extern "C" { + DECLARE_WRAPPER(cni_cache_read, cni_cached_info *, +- (const char *cache_dir, const char *net_name, const struct runtime_conf *rc)); ++ (const char *cache_dir, const char *net_name, const struct runtime_conf *rc)); + DEFINE_WRAPPER(cni_cache_read, cni_cached_info *, +- (const char *cache_dir, const char *net_name, const struct runtime_conf *rc), +- (cache_dir, net_name, rc)); ++ (const char *cache_dir, const char *net_name, const struct runtime_conf *rc), ++ (cache_dir, net_name, rc)); + +- DECLARE_WRAPPER(cni_check_network_list, int, +- (const struct cni_network_list_conf *list, const struct runtime_conf *rc, struct cni_opt_result **p_result)); ++ DECLARE_WRAPPER(cni_check_network_list, int, ++ (const struct cni_network_list_conf *list, const struct runtime_conf *rc, struct cni_opt_result **p_result)); + DEFINE_WRAPPER(cni_check_network_list, int, +- (const struct cni_network_list_conf *list, const struct runtime_conf *rc, struct cni_opt_result **p_result), +- (list, rc, p_result)); ++ (const struct cni_network_list_conf *list, const struct runtime_conf *rc, struct cni_opt_result **p_result), ++ (list, rc, p_result)); + +- DECLARE_WRAPPER(util_atomic_write_file, int, +- (const char *fname, const char *content, size_t content_len, mode_t mode, bool sync)); ++ DECLARE_WRAPPER(util_atomic_write_file, int, ++ (const char *fname, const char *content, size_t content_len, mode_t mode, bool sync)); + DEFINE_WRAPPER(util_atomic_write_file, int, +- (const char *fname, const char *content, size_t content_len, mode_t mode, bool sync), +- (fname, content, content_len, mode, sync)); +- +- DECLARE_WRAPPER(cni_del_network_list, int, +- (const struct cni_network_list_conf *list, const struct runtime_conf *rc, struct cni_opt_result **p_result)); ++ (const char *fname, const char *content, size_t content_len, mode_t mode, bool sync), ++ (fname, content, content_len, mode, sync)); ++ ++ DECLARE_WRAPPER(cni_del_network_list, int, ++ (const struct cni_network_list_conf *list, const struct runtime_conf *rc, struct cni_opt_result **p_result)); + DEFINE_WRAPPER(cni_del_network_list, int, +- (const struct cni_network_list_conf *list, const struct runtime_conf *rc, struct cni_opt_result **p_result), +- (list, rc, p_result)); ++ (const struct cni_network_list_conf *list, const struct runtime_conf *rc, struct cni_opt_result **p_result), ++ (list, rc, p_result)); + + DECLARE_WRAPPER(calloc, void *, (size_t nmemb, size_t size)); + DEFINE_WRAPPER(calloc, void *, (size_t nmemb, size_t size), (nmemb, size)); +@@ -90,19 +90,19 @@ public: + m_list.bytes = cni_net_conf_list_generate_json(m_list.list, &ctx, &jerr); + m_aliases_array = invoke_network_get_aliases_from_cached_info(m_info); + m_manager = { +- .id = (char *)"827bdd4b0b4e28d24dbaf3c563687ff6ffd23cd8fda38cadf818ac324fe5de3e", ++ .id = (char *)"827bdd4b0b4e28d24dbaf3c563687ff6ffd23cd8fda38cadf818ac324fe5de3e", + .netns_path = (char *)"/var/run/netns/isulacni-7dbc2c7d85279d5a", + .ifname = (char *)"eth0" + }; + m_manager.annotations = map_new(MAP_STR_STR, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); +- ++ + ctx = { OPT_PARSE_STRICT, 0 }; + aliases_json = cni_array_of_strings_container_generate_json(m_aliases_array, &ctx, &jerr); + if (aliases_json == nullptr) { + printf("Parse aliases_json failed: %s", jerr); + } + (void)map_replace(m_manager.annotations, (void *)aliases_str, (void *)aliases_json); +- ++ + free(aliases_json); + } + +@@ -140,7 +140,7 @@ TEST_F(CniOperateUnitTest, test_check_network_plane) + MOCK_CLEAR(calloc); + } + +- { ++ { + // cached info will be free in check_network_plane + MOCK_SET(cni_cache_read, invoke_network_get_cached_info((char *)CNI_CACHE_INFO)); + MOCK_SET(cni_check_network_list, 0); +diff --git a/test/network/network_mock.cc b/test/network/network_mock.cc +index 1fa1cc2e..27422b5f 100644 +--- a/test/network/network_mock.cc ++++ b/test/network/network_mock.cc +@@ -64,7 +64,7 @@ cni_array_of_strings_container *invoke_network_get_aliases_from_cached_info(cni_ + aliases_array->items = (char **)isula_smart_calloc_s(sizeof(char *), info->aliases_len); + EXPECT_THAT(aliases_array->items, testing::NotNull()) << "Out of memory" << std::endl; + for (size_t i = 0; i < info->aliases_len; i++) { +- aliases_array->items[i]= util_strdup_s(info->aliases[i]); ++ aliases_array->items[i] = util_strdup_s(info->aliases[i]); + aliases_array->len += 1; + } + +diff --git a/test/sandbox/controller/controller_common.cc b/test/sandbox/controller/controller_common.cc +index ed9c84d7..5f870c34 100644 +--- a/test/sandbox/controller/controller_common.cc ++++ b/test/sandbox/controller/controller_common.cc +@@ -15,7 +15,8 @@ + + #include "controller_common.h" + +-std::unique_ptr<sandbox::ControllerMountInfo> CreateTestMountInfo() { ++std::unique_ptr<sandbox::ControllerMountInfo> CreateTestMountInfo() ++{ + std::unique_ptr<sandbox::ControllerMountInfo> mountInfo(new sandbox::ControllerMountInfo()); + mountInfo->source = "/rootfs"; + mountInfo->destination = "/rootfs"; +@@ -23,7 +24,8 @@ std::unique_ptr<sandbox::ControllerMountInfo> CreateTestMountInfo() { + return mountInfo; + } + +-std::unique_ptr<sandbox::ControllerCreateParams> CreateTestCreateParams() { ++std::unique_ptr<sandbox::ControllerCreateParams> CreateTestCreateParams() ++{ + std::unique_ptr<sandbox::ControllerCreateParams> params(new sandbox::ControllerCreateParams()); + params->config = std::make_shared<runtime::v1::PodSandboxConfig>(); + params->netNSPath = "/proc/1/ns/net"; +@@ -31,7 +33,8 @@ std::unique_ptr<sandbox::ControllerCreateParams> CreateTestCreateParams() { + return params; + } + +-std::unique_ptr<sandbox::ControllerStreamInfo> CreateTestStreamInfo() { ++std::unique_ptr<sandbox::ControllerStreamInfo> CreateTestStreamInfo() ++{ + std::unique_ptr<sandbox::ControllerStreamInfo> streamInfo(new sandbox::ControllerStreamInfo()); + streamInfo->stdin = "/tmp/stdin"; + streamInfo->stdout = "/tmp/stdout"; +@@ -40,7 +43,8 @@ std::unique_ptr<sandbox::ControllerStreamInfo> CreateTestStreamInfo() { + return streamInfo; + } + +-std::unique_ptr<sandbox::ControllerPrepareParams> CreateTestPrepareParams() { ++std::unique_ptr<sandbox::ControllerPrepareParams> CreateTestPrepareParams() ++{ + std::unique_ptr<sandbox::ControllerPrepareParams> params(new sandbox::ControllerPrepareParams()); + params->containerId = DUMMY_CONTAINER_ID; + params->execId = DUMMY_EXEC_ID; +@@ -51,7 +55,9 @@ std::unique_ptr<sandbox::ControllerPrepareParams> CreateTestPrepareParams() { + return params; + } + +-std::unique_ptr<sandbox::ControllerUpdateResourcesParams> CreateTestUpdateResourcesParams(google::protobuf::Map<std::string, std::string> &annotations) { ++std::unique_ptr<sandbox::ControllerUpdateResourcesParams> CreateTestUpdateResourcesParams( ++ google::protobuf::Map<std::string, std::string> &annotations) ++{ + std::unique_ptr<std::string> resources(new std::string("{cpu: 12}")); + std::unique_ptr<sandbox::ControllerUpdateResourcesParams> params( + new sandbox::ControllerUpdateResourcesParams{DUMMY_SANDBOX_ID, std::move(resources), annotations} +diff --git a/test/sandbox/controller/controller_common.h b/test/sandbox/controller/controller_common.h +index e5a22e34..c01ae83c 100644 +--- a/test/sandbox/controller/controller_common.h ++++ b/test/sandbox/controller/controller_common.h +@@ -33,6 +33,7 @@ std::unique_ptr<sandbox::ControllerStreamInfo> CreateTestStreamInfo(); + + std::unique_ptr<sandbox::ControllerPrepareParams> CreateTestPrepareParams(); + +-std::unique_ptr<sandbox::ControllerUpdateResourcesParams> CreateTestUpdateResourcesParams(google::protobuf::Map<std::string, std::string> &annotations); ++std::unique_ptr<sandbox::ControllerUpdateResourcesParams> CreateTestUpdateResourcesParams( ++ google::protobuf::Map<std::string, std::string> &annotations); + + #endif // _ISULAD_TEST_SANDBOX_CONTROLLER_CONTROLLER_COMMON_H +\ No newline at end of file +diff --git a/test/sandbox/controller/manager/controller_manager_ut.cc b/test/sandbox/controller/manager/controller_manager_ut.cc +index 8467fbd4..705baaca 100644 +--- a/test/sandbox/controller/manager/controller_manager_ut.cc ++++ b/test/sandbox/controller/manager/controller_manager_ut.cc +@@ -33,11 +33,13 @@ public: + + class ControllerManagerTest : public testing::Test { + protected: +- void SetUp() override { ++ void SetUp() override ++ { + MockIsuladConf_SetMock(isuladConfMock.get()); + } + +- void TearDown() override { ++ void TearDown() override ++ { + MockIsuladConf_SetMock(nullptr); + static_cast<ControllerManagerWrapper*>(ControllerManagerWrapper::GetInstance())->Clear(); + } +@@ -45,7 +47,8 @@ protected: + std::unique_ptr<MockIsuladConf> isuladConfMock = std::unique_ptr<MockIsuladConf>(new MockIsuladConf()); + }; + +-static struct service_arguments *CreateDummyServerConf(const std::string &conf) { ++static struct service_arguments *CreateDummyServerConf(const std::string &conf) ++{ + parser_error err = nullptr; + struct service_arguments *args = (struct service_arguments *)util_common_calloc_s(sizeof(struct service_arguments)); + if (args == nullptr) { +@@ -59,7 +62,8 @@ static struct service_arguments *CreateDummyServerConf(const std::string &conf) + return args; + } + +-static void FreeDummyServerconf(struct service_arguments *args) { ++static void FreeDummyServerconf(struct service_arguments *args) ++{ + if (args != nullptr) { + free_isulad_daemon_configs(args->json_confs); + free(args); +@@ -70,7 +74,8 @@ static void FreeDummyServerconf(struct service_arguments *args) { + TEST_F(ControllerManagerTest, InitTestSucceed) + { + Errors err; +- const std::string daemonConfig = "{\"cri-sandboxers\": {\"kuasar\": {\"name\": \"vmm\",\"address\": \"/run/vmm-sandboxer.sock\"}}}"; ++ const std::string daemonConfig = ++ "{\"cri-sandboxers\": {\"kuasar\": {\"name\": \"vmm\",\"address\": \"/run/vmm-sandboxer.sock\"}}}"; + struct service_arguments *args = CreateDummyServerConf(daemonConfig); + ASSERT_NE(args, nullptr); + EXPECT_CALL(*isuladConfMock, ConfGetServerConf()).Times(1).WillOnce(testing::Return(args)); +@@ -130,7 +135,8 @@ TEST_F(ControllerManagerTest, InitTestSucceedWithNullConfig) + TEST_F(ControllerManagerTest, InitTestFailedWithDupShimConfig) + { + Errors err; +- const std::string daemonConfig = "{\"cri-sandboxers\": {\"kuasar\": {\"name\": \"shim\",\"address\": \"/run/vmm-sandboxer.sock\"}}}"; ++ const std::string daemonConfig = ++ "{\"cri-sandboxers\": {\"kuasar\": {\"name\": \"shim\",\"address\": \"/run/vmm-sandboxer.sock\"}}}"; + struct service_arguments *args = CreateDummyServerConf(daemonConfig); + ASSERT_NE(args, nullptr); + EXPECT_CALL(*isuladConfMock, ConfGetServerConf()).Times(1).WillOnce(testing::Return(args)); +@@ -148,7 +154,8 @@ TEST_F(ControllerManagerTest, InitTestFailedWithDupShimConfig) + TEST_F(ControllerManagerTest, InitTestFailedWithDupKuasarConfig) + { + Errors err; +- const std::string daemonConfig = "{\"cri-sandboxers\": {\"kuasar\": {\"name\": \"vmm1\",\"address\": \"/run/vmm1-sandboxer.sock\"},\"kuasar\": {\"name\": \"vmm2\",\"address\": \"/run/vmm2-sandboxer.sock\"}}}"; ++ const std::string daemonConfig = ++ "{\"cri-sandboxers\": {\"kuasar\": {\"name\": \"vmm1\",\"address\": \"/run/vmm1-sandboxer.sock\"},\"kuasar\": {\"name\": \"vmm2\",\"address\": \"/run/vmm2-sandboxer.sock\"}}}"; + struct service_arguments *args = CreateDummyServerConf(daemonConfig); + ASSERT_NE(args, nullptr); + EXPECT_CALL(*isuladConfMock, ConfGetServerConf()).Times(1).WillOnce(testing::Return(args)); +@@ -162,7 +169,8 @@ TEST_F(ControllerManagerTest, InitTestFailedWithDupKuasarConfig) + TEST_F(ControllerManagerTest, InitTestFailedWithDupNameConfig) + { + Errors err; +- const std::string daemonConfig = "{\"cri-sandboxers\": {\"kuasar1\": {\"name\": \"vmm\",\"address\": \"/run/vmm1-sandboxer.sock\"},\"kuasar2\": {\"name\": \"vmm\",\"address\": \"/run/vmm2-sandboxer.sock\"}}}"; ++ const std::string daemonConfig = ++ "{\"cri-sandboxers\": {\"kuasar1\": {\"name\": \"vmm\",\"address\": \"/run/vmm1-sandboxer.sock\"},\"kuasar2\": {\"name\": \"vmm\",\"address\": \"/run/vmm2-sandboxer.sock\"}}}"; + struct service_arguments *args = CreateDummyServerConf(daemonConfig); + ASSERT_NE(args, nullptr); + EXPECT_CALL(*isuladConfMock, ConfGetServerConf()).Times(1).WillOnce(testing::Return(args)); +@@ -176,7 +184,8 @@ TEST_F(ControllerManagerTest, InitTestFailedWithDupNameConfig) + TEST_F(ControllerManagerTest, InitTestFailedWithDupInit) + { + Errors err; +- const std::string daemonConfig = "{\"cri-sandboxers\": {\"kuasar\": {\"name\": \"vmm\",\"address\": \"/run/vmm-sandboxer.sock\"}}}"; ++ const std::string daemonConfig = ++ "{\"cri-sandboxers\": {\"kuasar\": {\"name\": \"vmm\",\"address\": \"/run/vmm-sandboxer.sock\"}}}"; + struct service_arguments *args = CreateDummyServerConf(daemonConfig); + ASSERT_NE(args, nullptr); + EXPECT_CALL(*isuladConfMock, ConfGetServerConf()).Times(2).WillRepeatedly(testing::Return(args)); +diff --git a/test/sandbox/controller/sandboxer/async_wait_call/async_wait_call_ut.cc b/test/sandbox/controller/sandboxer/async_wait_call/async_wait_call_ut.cc +index 0596771a..1a58344c 100644 +--- a/test/sandbox/controller/sandboxer/async_wait_call/async_wait_call_ut.cc ++++ b/test/sandbox/controller/sandboxer/async_wait_call/async_wait_call_ut.cc +@@ -35,17 +35,20 @@ public: + + class AsyncWaitCallTest : public testing::Test { + protected: +- void SetUp() override { ++ void SetUp() override ++ { + m_sandboxId = "8040f13d54889ad4cd"; + m_sandboxer = "test_sandboxer"; + m_callback = std::shared_ptr<DummyCallback>(new DummyCallback()); +- m_call = std::unique_ptr<sandbox::SandboxerAsyncWaitCall>(new sandbox::SandboxerAsyncWaitCall(m_callback, m_sandboxId, m_sandboxer)); ++ m_call = std::unique_ptr<sandbox::SandboxerAsyncWaitCall>(new sandbox::SandboxerAsyncWaitCall(m_callback, m_sandboxId, ++ m_sandboxer)); + m_stub = std::unique_ptr<DummyControllerStub>(NewDummyControllerStub()); + m_stub_mock = std::make_shared<MockControllerStub>(); + MockControllerStub_SetMock(m_stub_mock); + } + +- void TearDown() override { ++ void TearDown() override ++ { + MockControllerStub_SetMock(nullptr); + } + +diff --git a/test/sandbox/controller/sandboxer/async_wait_call/dummy_monitor_utils.h b/test/sandbox/controller/sandboxer/async_wait_call/dummy_monitor_utils.h +index 3a12d042..7e98d844 100644 +--- a/test/sandbox/controller/sandboxer/async_wait_call/dummy_monitor_utils.h ++++ b/test/sandbox/controller/sandboxer/async_wait_call/dummy_monitor_utils.h +@@ -21,7 +21,8 @@ + + #include "controller.h" + +-class DummyClientAsyncResponseReader: public grpc::ClientAsyncResponseReaderInterface<containerd::services::sandbox::v1::ControllerWaitResponse> { ++class DummyClientAsyncResponseReader: public ++ grpc::ClientAsyncResponseReaderInterface<containerd::services::sandbox::v1::ControllerWaitResponse> { + public: + DummyClientAsyncResponseReader() = default; + ~DummyClientAsyncResponseReader() = default; +@@ -30,26 +31,32 @@ public: + + void ReadInitialMetadata(void *tag) override {} + +- void Finish(containerd::services::sandbox::v1::ControllerWaitResponse *response, grpc::Status *status, void *tag) override { ++ void Finish(containerd::services::sandbox::v1::ControllerWaitResponse *response, grpc::Status *status, ++ void *tag) override ++ { + response->set_exit_status(m_exitStatus); + response->mutable_exited_at()->CopyFrom(m_exitedAt); + *status = m_status; + m_tag = tag; + } + +- void SetExitAt(const google::protobuf::Timestamp &exitAt) { ++ void SetExitAt(const google::protobuf::Timestamp &exitAt) ++ { + m_exitedAt = exitAt; + } + +- void SetExitStatus(uint32_t status) { ++ void SetExitStatus(uint32_t status) ++ { + m_exitStatus = status; + } + +- void SetStatus(grpc::Status status) { ++ void SetStatus(grpc::Status status) ++ { + m_status = status; + } + +- void *GetTag() { ++ void *GetTag() ++ { + return m_tag; + } + +@@ -70,28 +77,39 @@ enum AsyncWaitCallStatus { + + class DummyCallback: public sandbox::SandboxStatusCallback { + public: +- DummyCallback() { ++ DummyCallback() ++ { + m_status = ASYNC_WAIT_CALL_STATUS_UNKNOWN; + } + ~DummyCallback() = default; + +- void OnSandboxReady() override { m_status = ASYNC_WAIT_CALL_STATUS_READY; } +- void OnSandboxPending() override { m_status = ASYNC_WAIT_CALL_STATUS_PENDING; } +- void OnSandboxExit(const sandbox::ControllerExitInfo &exitInfo) override { ++ void OnSandboxReady() override ++ { ++ m_status = ASYNC_WAIT_CALL_STATUS_READY; ++ } ++ void OnSandboxPending() override ++ { ++ m_status = ASYNC_WAIT_CALL_STATUS_PENDING; ++ } ++ void OnSandboxExit(const sandbox::ControllerExitInfo &exitInfo) override ++ { + m_status = ASYNC_WAIT_CALL_STATUS_EXIT; + m_exitStatus = exitInfo.exitStatus; + m_exitedAt = exitInfo.exitedAt; + } + +- AsyncWaitCallStatus GetStatus() { ++ AsyncWaitCallStatus GetStatus() ++ { + return m_status; + } + +- uint32_t GetExitStatus() { ++ uint32_t GetExitStatus() ++ { + return m_exitStatus; + } + +- uint64_t GetExitedAt() { ++ uint64_t GetExitedAt() ++ { + return m_exitedAt; + } + private: +diff --git a/test/sandbox/controller/sandboxer/sandboxer_client/sandboxer_client_ut.cc b/test/sandbox/controller/sandboxer/sandboxer_client/sandboxer_client_ut.cc +index 0804b38b..b0f4758f 100644 +--- a/test/sandbox/controller/sandboxer/sandboxer_client/sandboxer_client_ut.cc ++++ b/test/sandbox/controller/sandboxer/sandboxer_client/sandboxer_client_ut.cc +@@ -21,7 +21,8 @@ + + class SandboxerClientWrapper : public sandbox::SandboxerClient { + public: +- SandboxerClientWrapper(const std::string &sandboxer, const std::string &address) : SandboxerClient(sandboxer, address) { ++ SandboxerClientWrapper(const std::string &sandboxer, const std::string &address) : SandboxerClient(sandboxer, address) ++ { + m_stub = NewDummyControllerStub(); + } + +@@ -30,7 +31,8 @@ public: + + class ControllerSandboxerClientTest : public testing::Test { + protected: +- void SetUp() override { ++ void SetUp() override ++ { + m_sandboxer = "sandboxer"; + m_address = "/tmp/sandboxer.sock"; + +@@ -39,7 +41,8 @@ protected: + MockControllerStub_SetMock(m_stub); + } + +- void TearDown() override { ++ void TearDown() override ++ { + MockControllerStub_SetMock(nullptr); + } + +@@ -50,18 +53,21 @@ protected: + std::shared_ptr<SandboxerClientWrapper> m_sandboxerClient; + }; + +-static std::unique_ptr<containerd::services::sandbox::v1::ControllerStartResponse> CreateTestGrpcStartResponse() { +- std::unique_ptr<containerd::services::sandbox::v1::ControllerStartResponse> response(new containerd::services::sandbox::v1::ControllerStartResponse()); ++static std::unique_ptr<containerd::services::sandbox::v1::ControllerStartResponse> CreateTestGrpcStartResponse() ++{ ++ std::unique_ptr<containerd::services::sandbox::v1::ControllerStartResponse> response( ++ new containerd::services::sandbox::v1::ControllerStartResponse()); + response->set_sandbox_id(DUMMY_SANDBOX_ID); + response->set_pid(1); +- response->mutable_created_at()->set_seconds(DUMMY_CREATE_AT/SECOND_TO_NANOS); +- response->mutable_created_at()->set_nanos(DUMMY_CREATE_AT%SECOND_TO_NANOS); ++ response->mutable_created_at()->set_seconds(DUMMY_CREATE_AT / SECOND_TO_NANOS); ++ response->mutable_created_at()->set_nanos(DUMMY_CREATE_AT % SECOND_TO_NANOS); + response->mutable_labels()->insert({"label1", "value1"}); + return response; + } + + // Create platform response for test. +-static std::unique_ptr<containerd::services::sandbox::v1::ControllerPlatformResponse> CreateTestPlatformResponse() { ++static std::unique_ptr<containerd::services::sandbox::v1::ControllerPlatformResponse> CreateTestPlatformResponse() ++{ + std::unique_ptr<containerd::services::sandbox::v1::ControllerPlatformResponse> response( + new containerd::services::sandbox::v1::ControllerPlatformResponse() + ); +@@ -72,7 +78,8 @@ static std::unique_ptr<containerd::services::sandbox::v1::ControllerPlatformResp + } + + // Create status response for test +-static std::unique_ptr<containerd::services::sandbox::v1::ControllerStatusResponse> CreateTestStatusResponse() { ++static std::unique_ptr<containerd::services::sandbox::v1::ControllerStatusResponse> CreateTestStatusResponse() ++{ + std::unique_ptr<containerd::services::sandbox::v1::ControllerStatusResponse> response( + new containerd::services::sandbox::v1::ControllerStatusResponse() + ); +@@ -80,29 +87,32 @@ static std::unique_ptr<containerd::services::sandbox::v1::ControllerStatusRespon + response->set_state("running"); + response->set_pid(1); + response->set_task_address(DUMMY_TASK_ADDRESS); +- response->mutable_created_at()->set_seconds(DUMMY_CREATE_AT/SECOND_TO_NANOS); +- response->mutable_created_at()->set_nanos(DUMMY_CREATE_AT%SECOND_TO_NANOS); +- response->mutable_exited_at()->set_seconds(DUMMY_CREATE_AT/SECOND_TO_NANOS); +- response->mutable_exited_at()->set_nanos(DUMMY_CREATE_AT%SECOND_TO_NANOS); ++ response->mutable_created_at()->set_seconds(DUMMY_CREATE_AT / SECOND_TO_NANOS); ++ response->mutable_created_at()->set_nanos(DUMMY_CREATE_AT % SECOND_TO_NANOS); ++ response->mutable_exited_at()->set_seconds(DUMMY_CREATE_AT / SECOND_TO_NANOS); ++ response->mutable_exited_at()->set_nanos(DUMMY_CREATE_AT % SECOND_TO_NANOS); + response->mutable_info()->insert({"info1", "value1"}); + response->mutable_extra()->set_value("{extra: test}"); + return response; + } + + /************* Unit tests for Create *************/ +-TEST_F(ControllerSandboxerClientTest, CreateTestSucceed) { ++TEST_F(ControllerSandboxerClientTest, CreateTestSucceed) ++{ + Errors err; + std::unique_ptr<sandbox::ControllerCreateParams> params = CreateTestCreateParams(); + // Fake a grpc create response. + containerd::services::sandbox::v1::ControllerCreateResponse response; + response.set_sandbox_id(DUMMY_SANDBOX_ID); + // Set response to return sandbox_id, and return OK for stub_->Create(). +- EXPECT_CALL(*m_stub, Create).Times(1).WillOnce(testing::DoAll(testing::SetArgPointee<2>(response), testing::Return(grpc::Status::OK))); ++ EXPECT_CALL(*m_stub, Create).Times(1).WillOnce(testing::DoAll(testing::SetArgPointee<2>(response), ++ testing::Return(grpc::Status::OK))); + EXPECT_TRUE(m_sandboxerClient->Create(DUMMY_SANDBOX_ID, *params, err)); + EXPECT_TRUE(err.Empty()); + } + +-TEST_F(ControllerSandboxerClientTest, CreateTestNullConfig) { ++TEST_F(ControllerSandboxerClientTest, CreateTestNullConfig) ++{ + Errors err; + std::unique_ptr<sandbox::ControllerCreateParams> params(new sandbox::ControllerCreateParams()); + params->config = nullptr; +@@ -113,20 +123,23 @@ TEST_F(ControllerSandboxerClientTest, CreateTestNullConfig) { + EXPECT_THAT(err.GetCMessage(), testing::HasSubstr("Failed to init create request for sandboxer create request")); + } + +-TEST_F(ControllerSandboxerClientTest, CreateTestNullMount) { ++TEST_F(ControllerSandboxerClientTest, CreateTestNullMount) ++{ + Errors err; + std::unique_ptr<sandbox::ControllerCreateParams> params = CreateTestCreateParams(); + params->mounts.push_back(nullptr); + containerd::services::sandbox::v1::ControllerCreateRequest request; + // Save request to check mount size. +- EXPECT_CALL(*m_stub, Create).Times(1).WillOnce(testing::DoAll(testing::SaveArg<1>(&request), testing::Return(grpc::Status::OK))); ++ EXPECT_CALL(*m_stub, Create).Times(1).WillOnce(testing::DoAll(testing::SaveArg<1>(&request), ++ testing::Return(grpc::Status::OK))); + EXPECT_TRUE(m_sandboxerClient->Create(DUMMY_SANDBOX_ID, *params, err)); + // The nullptr pushed in params should not be counted. + EXPECT_EQ(request.rootfs_size(), 1); + EXPECT_TRUE(err.Empty()); + } + +-TEST_F(ControllerSandboxerClientTest, CreateTestStatusNotOK) { ++TEST_F(ControllerSandboxerClientTest, CreateTestStatusNotOK) ++{ + Errors err; + std::unique_ptr<sandbox::ControllerCreateParams> params = CreateTestCreateParams(); + // Fake a grpc create response. +@@ -138,11 +151,13 @@ TEST_F(ControllerSandboxerClientTest, CreateTestStatusNotOK) { + } + + /************* Unit tests for Start *************/ +-TEST_F(ControllerSandboxerClientTest, StartTestSucceed) { ++TEST_F(ControllerSandboxerClientTest, StartTestSucceed) ++{ + Errors err; + sandbox::ControllerSandboxInfo sandboxInfo; + std::unique_ptr<containerd::services::sandbox::v1::ControllerStartResponse> response = CreateTestGrpcStartResponse(); +- EXPECT_CALL(*m_stub, Start).Times(1).WillOnce(testing::DoAll(testing::SetArgPointee<2>(*response), testing::Return(grpc::Status::OK))); ++ EXPECT_CALL(*m_stub, Start).Times(1).WillOnce(testing::DoAll(testing::SetArgPointee<2>(*response), ++ testing::Return(grpc::Status::OK))); + EXPECT_TRUE(m_sandboxerClient->Start(DUMMY_SANDBOX_ID, sandboxInfo, err)); + EXPECT_TRUE(err.Empty()); + EXPECT_EQ(sandboxInfo.id, DUMMY_SANDBOX_ID); +@@ -152,7 +167,8 @@ TEST_F(ControllerSandboxerClientTest, StartTestSucceed) { + EXPECT_EQ(sandboxInfo.labels["label1"], "value1"); + } + +-TEST_F(ControllerSandboxerClientTest, StartTestStatusNotOK) { ++TEST_F(ControllerSandboxerClientTest, StartTestStatusNotOK) ++{ + Errors err; + sandbox::ControllerSandboxInfo sandboxInfo; + EXPECT_CALL(*m_stub, Start).Times(1).WillOnce(testing::Return(grpc::Status(grpc::StatusCode::ABORTED, "gRPC Abort"))); +@@ -161,7 +177,8 @@ TEST_F(ControllerSandboxerClientTest, StartTestStatusNotOK) { + } + + /************* Unit tests for Prepare *************/ +-TEST_F(ControllerSandboxerClientTest, PrepareTestSucceed) { ++TEST_F(ControllerSandboxerClientTest, PrepareTestSucceed) ++{ + Errors err; + std::string bundle; + std::unique_ptr<sandbox::ControllerPrepareParams> params = CreateTestPrepareParams(); +@@ -169,13 +186,15 @@ TEST_F(ControllerSandboxerClientTest, PrepareTestSucceed) { + containerd::services::sandbox::v1::PrepareResponse response; + response.set_bundle("/tmp/bundle"); + // Set response to return bundle, and return OK for stub_->Prepare(). +- EXPECT_CALL(*m_stub, Prepare).Times(1).WillOnce(testing::DoAll(testing::SetArgPointee<2>(response), testing::Return(grpc::Status::OK))); ++ EXPECT_CALL(*m_stub, Prepare).Times(1).WillOnce(testing::DoAll(testing::SetArgPointee<2>(response), ++ testing::Return(grpc::Status::OK))); + EXPECT_TRUE(m_sandboxerClient->Prepare(DUMMY_SANDBOX_ID, *params, bundle, err)); + EXPECT_TRUE(err.Empty()); + EXPECT_EQ(bundle, "/tmp/bundle"); + } + +-TEST_F(ControllerSandboxerClientTest, PrepareTestNullSpec) { ++TEST_F(ControllerSandboxerClientTest, PrepareTestNullSpec) ++{ + Errors err; + std::string bundle; + std::unique_ptr<sandbox::ControllerPrepareParams> params = CreateTestPrepareParams(); +@@ -186,21 +205,24 @@ TEST_F(ControllerSandboxerClientTest, PrepareTestNullSpec) { + EXPECT_THAT(err.GetCMessage(), testing::HasSubstr("Failed to init prepare request for sandboxer prepare request")); + } + +-TEST_F(ControllerSandboxerClientTest, PrepareTestNullMount) { ++TEST_F(ControllerSandboxerClientTest, PrepareTestNullMount) ++{ + Errors err; + std::string bundle; + std::unique_ptr<sandbox::ControllerPrepareParams> params = CreateTestPrepareParams(); + params->rootfs.push_back(nullptr); + containerd::services::sandbox::v1::PrepareRequest request; + // Save request to check mount size. +- EXPECT_CALL(*m_stub, Prepare).Times(1).WillOnce(testing::DoAll(testing::SaveArg<1>(&request), testing::Return(grpc::Status::OK))); ++ EXPECT_CALL(*m_stub, Prepare).Times(1).WillOnce(testing::DoAll(testing::SaveArg<1>(&request), ++ testing::Return(grpc::Status::OK))); + EXPECT_TRUE(m_sandboxerClient->Prepare(DUMMY_SANDBOX_ID, *params, bundle, err)); + // The nullptr pushed in params should not be counted. + EXPECT_EQ(request.rootfs_size(), 2); + EXPECT_TRUE(err.Empty()); + } + +-TEST_F(ControllerSandboxerClientTest, PrepareTestStatusNotOK) { ++TEST_F(ControllerSandboxerClientTest, PrepareTestStatusNotOK) ++{ + Errors err; + std::string bundle; + std::unique_ptr<sandbox::ControllerPrepareParams> params = CreateTestPrepareParams(); +@@ -210,7 +232,8 @@ TEST_F(ControllerSandboxerClientTest, PrepareTestStatusNotOK) { + } + + /************* Unit tests for Purge *************/ +-TEST_F(ControllerSandboxerClientTest, PurgeTestSucceed) { ++TEST_F(ControllerSandboxerClientTest, PurgeTestSucceed) ++{ + Errors err; + // Set response to return OK for stub_->Purge(). + EXPECT_CALL(*m_stub, Purge).Times(1).WillOnce(testing::Return(grpc::Status::OK)); +@@ -218,7 +241,8 @@ TEST_F(ControllerSandboxerClientTest, PurgeTestSucceed) { + EXPECT_TRUE(err.Empty()); + } + +-TEST_F(ControllerSandboxerClientTest, PurgeTestStatusNotOK) { ++TEST_F(ControllerSandboxerClientTest, PurgeTestStatusNotOK) ++{ + Errors err; + EXPECT_CALL(*m_stub, Purge).Times(1).WillOnce(testing::Return(grpc::Status(grpc::StatusCode::ABORTED, "gRPC Abort"))); + EXPECT_FALSE(m_sandboxerClient->Purge(DUMMY_SANDBOX_ID, DUMMY_CONTAINER_ID, DUMMY_EXEC_ID, err)); +@@ -226,7 +250,8 @@ TEST_F(ControllerSandboxerClientTest, PurgeTestStatusNotOK) { + } + + /************* Unit tests for UpdateResources *************/ +-TEST_F(ControllerSandboxerClientTest, UpdateResourcesTestSucceed) { ++TEST_F(ControllerSandboxerClientTest, UpdateResourcesTestSucceed) ++{ + Errors err; + google::protobuf::Map<std::string, std::string> annotations; + std::unique_ptr<sandbox::ControllerUpdateResourcesParams> params = CreateTestUpdateResourcesParams(annotations); +@@ -236,7 +261,8 @@ TEST_F(ControllerSandboxerClientTest, UpdateResourcesTestSucceed) { + EXPECT_TRUE(err.Empty()); + } + +-TEST_F(ControllerSandboxerClientTest, UpdateResourcesTestNullResources) { ++TEST_F(ControllerSandboxerClientTest, UpdateResourcesTestNullResources) ++{ + Errors err; + google::protobuf::Map<std::string, std::string> annotations; + std::unique_ptr<sandbox::ControllerUpdateResourcesParams> params = CreateTestUpdateResourcesParams(annotations); +@@ -244,24 +270,29 @@ TEST_F(ControllerSandboxerClientTest, UpdateResourcesTestNullResources) { + // Stub should not be called + EXPECT_CALL(*m_stub, UpdateResources).Times(0); + EXPECT_FALSE(m_sandboxerClient->UpdateResources(DUMMY_SANDBOX_ID, *params, err)); +- EXPECT_THAT(err.GetCMessage(), testing::HasSubstr("Failed to init update-resources request for sandboxer update-resources request")); ++ EXPECT_THAT(err.GetCMessage(), ++ testing::HasSubstr("Failed to init update-resources request for sandboxer update-resources request")); + } + +-TEST_F(ControllerSandboxerClientTest, UpdateResourcesTestStatusNotOK) { ++TEST_F(ControllerSandboxerClientTest, UpdateResourcesTestStatusNotOK) ++{ + Errors err; + google::protobuf::Map<std::string, std::string> annotations; + std::unique_ptr<sandbox::ControllerUpdateResourcesParams> params = CreateTestUpdateResourcesParams(annotations); +- EXPECT_CALL(*m_stub, UpdateResources).Times(1).WillOnce(testing::Return(grpc::Status(grpc::StatusCode::ABORTED, "gRPC Abort"))); ++ EXPECT_CALL(*m_stub, UpdateResources).Times(1).WillOnce(testing::Return(grpc::Status(grpc::StatusCode::ABORTED, ++ "gRPC Abort"))); + EXPECT_FALSE(m_sandboxerClient->UpdateResources(DUMMY_SANDBOX_ID, *params, err)); + EXPECT_THAT(err.GetCMessage(), testing::HasSubstr("gRPC Abort")); + } + + /************* Unit tests for Platform *************/ +-TEST_F(ControllerSandboxerClientTest, PlatformTestSucceed) { ++TEST_F(ControllerSandboxerClientTest, PlatformTestSucceed) ++{ + Errors err; + sandbox::ControllerPlatformInfo platformInfo; + std::unique_ptr<containerd::services::sandbox::v1::ControllerPlatformResponse> response = CreateTestPlatformResponse(); +- EXPECT_CALL(*m_stub, Platform).Times(1).WillOnce(testing::DoAll(testing::SetArgPointee<2>(*response), testing::Return(grpc::Status::OK))); ++ EXPECT_CALL(*m_stub, Platform).Times(1).WillOnce(testing::DoAll(testing::SetArgPointee<2>(*response), ++ testing::Return(grpc::Status::OK))); + EXPECT_TRUE(m_sandboxerClient->Platform(DUMMY_SANDBOX_ID, platformInfo, err)); + EXPECT_TRUE(err.Empty()); + EXPECT_EQ(platformInfo.os, "linux"); +@@ -269,16 +300,19 @@ TEST_F(ControllerSandboxerClientTest, PlatformTestSucceed) { + EXPECT_EQ(platformInfo.variant, "ubuntu"); + } + +-TEST_F(ControllerSandboxerClientTest, PlatformTestStatusNotOK) { ++TEST_F(ControllerSandboxerClientTest, PlatformTestStatusNotOK) ++{ + Errors err; + sandbox::ControllerPlatformInfo platformInfo; +- EXPECT_CALL(*m_stub, Platform).Times(1).WillOnce(testing::Return(grpc::Status(grpc::StatusCode::ABORTED, "gRPC Abort"))); ++ EXPECT_CALL(*m_stub, Platform).Times(1).WillOnce(testing::Return(grpc::Status(grpc::StatusCode::ABORTED, ++ "gRPC Abort"))); + EXPECT_FALSE(m_sandboxerClient->Platform(DUMMY_SANDBOX_ID, platformInfo, err)); + EXPECT_THAT(err.GetCMessage(), testing::HasSubstr("gRPC Abort")); + } + + /************* Unit tests for Stop *************/ +-TEST_F(ControllerSandboxerClientTest, StopTestSucceed) { ++TEST_F(ControllerSandboxerClientTest, StopTestSucceed) ++{ + Errors err; + // Set response to return OK for stub_->Stop(). + EXPECT_CALL(*m_stub, Stop).Times(1).WillOnce(testing::Return(grpc::Status::OK)); +@@ -286,7 +320,8 @@ TEST_F(ControllerSandboxerClientTest, StopTestSucceed) { + EXPECT_TRUE(err.Empty()); + } + +-TEST_F(ControllerSandboxerClientTest, StopTestStatusNotOK) { ++TEST_F(ControllerSandboxerClientTest, StopTestStatusNotOK) ++{ + Errors err; + EXPECT_CALL(*m_stub, Stop).Times(1).WillOnce(testing::Return(grpc::Status(grpc::StatusCode::ABORTED, "gRPC Abort"))); + EXPECT_FALSE(m_sandboxerClient->Stop(DUMMY_SANDBOX_ID, 0, err)); +@@ -294,11 +329,13 @@ TEST_F(ControllerSandboxerClientTest, StopTestStatusNotOK) { + } + + /************* Unit tests for Status *************/ +-TEST_F(ControllerSandboxerClientTest, StatusTestSucceed) { ++TEST_F(ControllerSandboxerClientTest, StatusTestSucceed) ++{ + Errors err; + sandbox::ControllerSandboxStatus sandboxStatus; + std::unique_ptr<containerd::services::sandbox::v1::ControllerStatusResponse> response = CreateTestStatusResponse(); +- EXPECT_CALL(*m_stub, Status).Times(1).WillOnce(testing::DoAll(testing::SetArgPointee<2>(*response), testing::Return(grpc::Status::OK))); ++ EXPECT_CALL(*m_stub, Status).Times(1).WillOnce(testing::DoAll(testing::SetArgPointee<2>(*response), ++ testing::Return(grpc::Status::OK))); + EXPECT_TRUE(m_sandboxerClient->Status(DUMMY_SANDBOX_ID, false, sandboxStatus, err)); + EXPECT_TRUE(err.Empty()); + EXPECT_EQ(sandboxStatus.id, DUMMY_SANDBOX_ID); +@@ -312,7 +349,8 @@ TEST_F(ControllerSandboxerClientTest, StatusTestSucceed) { + EXPECT_EQ(sandboxStatus.extra, "{extra: test}"); + } + +-TEST_F(ControllerSandboxerClientTest, StatusTestStatusNotOK) { ++TEST_F(ControllerSandboxerClientTest, StatusTestStatusNotOK) ++{ + Errors err; + sandbox::ControllerSandboxStatus sandboxStatus; + EXPECT_CALL(*m_stub, Status).Times(1).WillOnce(testing::Return(grpc::Status(grpc::StatusCode::ABORTED, "gRPC Abort"))); +@@ -321,7 +359,8 @@ TEST_F(ControllerSandboxerClientTest, StatusTestStatusNotOK) { + } + + /************* Unit tests for Shutdown *************/ +-TEST_F(ControllerSandboxerClientTest, ShutdownTestSucceed) { ++TEST_F(ControllerSandboxerClientTest, ShutdownTestSucceed) ++{ + Errors err; + // Set response to return OK for stub_->Shutdown(). + EXPECT_CALL(*m_stub, Shutdown).Times(1).WillOnce(testing::Return(grpc::Status::OK)); +@@ -329,9 +368,11 @@ TEST_F(ControllerSandboxerClientTest, ShutdownTestSucceed) { + EXPECT_TRUE(err.Empty()); + } + +-TEST_F(ControllerSandboxerClientTest, ShutdownTestStatusNotOK) { ++TEST_F(ControllerSandboxerClientTest, ShutdownTestStatusNotOK) ++{ + Errors err; +- EXPECT_CALL(*m_stub, Shutdown).Times(1).WillOnce(testing::Return(grpc::Status(grpc::StatusCode::ABORTED, "gRPC Abort"))); ++ EXPECT_CALL(*m_stub, Shutdown).Times(1).WillOnce(testing::Return(grpc::Status(grpc::StatusCode::ABORTED, ++ "gRPC Abort"))); + EXPECT_FALSE(m_sandboxerClient->Shutdown(DUMMY_SANDBOX_ID, err)); + EXPECT_THAT(err.GetCMessage(), testing::HasSubstr("gRPC Abort")); + } +diff --git a/test/sandbox/controller/sandboxer/sandboxer_controller/sandboxer_controller_ut.cc b/test/sandbox/controller/sandboxer/sandboxer_controller/sandboxer_controller_ut.cc +index 52c3f28a..f49d7cc5 100644 +--- a/test/sandbox/controller/sandboxer/sandboxer_controller/sandboxer_controller_ut.cc ++++ b/test/sandbox/controller/sandboxer/sandboxer_controller/sandboxer_controller_ut.cc +@@ -21,7 +21,8 @@ + + class SandboxerControllerTest : public testing::Test { + protected: +- void SetUp() override { ++ void SetUp() override ++ { + Errors err; + m_contoller = std::move(std::unique_ptr<SandboxerController>(new SandboxerController(m_sandboxer, m_address))); + m_sandboxerClientMock = std::make_shared<SandboxerClientMock>(); +@@ -30,7 +31,8 @@ protected: + m_contoller->Init(err); + } + +- void TearDown() override { ++ void TearDown() override ++ { + m_contoller.reset(nullptr); + } + +@@ -74,7 +76,8 @@ TEST_F(SandboxerControllerTest, StartTestSucceed) + Errors err; + std::unique_ptr<sandbox::ControllerSandboxInfo> sandboxInfo = CreateTestSandboxInfo(); + // Set response to return sandbox_id, and return OK for stub_->Start(). +- EXPECT_CALL(*m_sandboxerClientMock, Start).Times(1).WillOnce(testing::DoAll(testing::SetArgReferee<1>(*sandboxInfo), testing::Return(true))); ++ EXPECT_CALL(*m_sandboxerClientMock, Start).Times(1).WillOnce(testing::DoAll(testing::SetArgReferee<1>(*sandboxInfo), ++ testing::Return(true))); + std::unique_ptr<sandbox::ControllerSandboxInfo> ret = m_contoller->Start(DUMMY_SANDBOX_ID, err); + EXPECT_EQ(ret->id, DUMMY_SANDBOX_ID); + EXPECT_EQ(ret->pid, 1234); +@@ -99,7 +102,8 @@ TEST_F(SandboxerControllerTest, PlatformTestSucceed) + platformInfo->arch = "amd64"; + platformInfo->variant = "openEuler"; + // Set response to return sandbox_id, and return OK for stub_->Platform(). +- EXPECT_CALL(*m_sandboxerClientMock, Platform).Times(1).WillOnce(testing::DoAll(testing::SetArgReferee<1>(*platformInfo), testing::Return(true))); ++ EXPECT_CALL(*m_sandboxerClientMock, Platform).Times(1).WillOnce(testing::DoAll(testing::SetArgReferee<1>(*platformInfo), ++ testing::Return(true))); + std::unique_ptr<sandbox::ControllerPlatformInfo> ret = m_contoller->Platform(DUMMY_SANDBOX_ID, err); + EXPECT_EQ(ret->os, "linux"); + EXPECT_EQ(ret->arch, "amd64"); +@@ -121,7 +125,8 @@ TEST_F(SandboxerControllerTest, PrepareTestSucceed) + Errors err; + std::string bundle = "/tmp/bundle"; + // Set response to return sandbox_id, and return OK for stub_->Prepare(). +- EXPECT_CALL(*m_sandboxerClientMock, Prepare).Times(1).WillOnce(testing::DoAll(testing::SetArgReferee<2>(bundle), testing::Return(true))); ++ EXPECT_CALL(*m_sandboxerClientMock, Prepare).Times(1).WillOnce(testing::DoAll(testing::SetArgReferee<2>(bundle), ++ testing::Return(true))); + std::string ret = m_contoller->Prepare(DUMMY_SANDBOX_ID, *CreateTestPrepareParams(), err); + EXPECT_EQ(ret, bundle); + } +@@ -201,7 +206,8 @@ TEST_F(SandboxerControllerTest, StatusTestSucceed) + sandboxStatus->info["test"] = "test"; + sandboxStatus->exitedAt = DUMMY_EXITED_AT; + // Set response to return sandbox_id, and return OK for stub_->Status(). +- EXPECT_CALL(*m_sandboxerClientMock, Status).Times(1).WillOnce(testing::DoAll(testing::SetArgReferee<2>(*sandboxStatus), testing::Return(true))); ++ EXPECT_CALL(*m_sandboxerClientMock, Status).Times(1).WillOnce(testing::DoAll(testing::SetArgReferee<2>(*sandboxStatus), ++ testing::Return(true))); + std::unique_ptr<sandbox::ControllerSandboxStatus> ret = m_contoller->Status(DUMMY_SANDBOX_ID, false, err); + EXPECT_EQ(ret->id, DUMMY_SANDBOX_ID); + EXPECT_EQ(ret->state, "created"); +diff --git a/test/sandbox/controller/shim/shim_controller_ut.cc b/test/sandbox/controller/shim/shim_controller_ut.cc +index 978e2c36..e43cc645 100644 +--- a/test/sandbox/controller/shim/shim_controller_ut.cc ++++ b/test/sandbox/controller/shim/shim_controller_ut.cc +@@ -24,7 +24,8 @@ + + class ShimControllerTest : public testing::Test { + protected: +- void SetUp() override { ++ void SetUp() override ++ { + Errors err; + m_contoller = std::move(std::unique_ptr<sandbox::ShimController>(new sandbox::ShimController(m_sandboxer))); + m_containerCallbackMock = std::make_shared<MockContainerCallback>(); +@@ -37,7 +38,8 @@ protected: + service_callback_init(); + } + +- void TearDown() override { ++ void TearDown() override ++ { + m_contoller.reset(nullptr); + } + +-- +2.42.0 + diff --git a/0029-2255-Fix-cpusets-offline-issue.patch b/0029-2255-Fix-cpusets-offline-issue.patch new file mode 100644 index 0000000..92f7b63 --- /dev/null +++ b/0029-2255-Fix-cpusets-offline-issue.patch @@ -0,0 +1,445 @@ +From a6f1ff360dded79ce5139a8b97a51c37d2fbd403 Mon Sep 17 00:00:00 2001 +From: xuxuepeng <xuxuepeng1@huawei.com> +Date: Thu, 23 Nov 2023 13:18:13 +0000 +Subject: [PATCH 29/64] !2255 Fix cpusets offline issue * Fix cpusets offline + issue + +--- + src/daemon/common/sysinfo.c | 1 + + src/daemon/common/sysinfo.h | 3 + + src/daemon/modules/spec/verify.c | 24 ++--- + test/mocks/image_mock.cc | 16 +++ + test/mocks/image_mock.h | 2 + + test/specs/CMakeLists.txt | 1 + + test/specs/verify/CMakeLists.txt | 85 +++++++++++++++ + test/specs/verify/verify_ut.cc | 173 +++++++++++++++++++++++++++++++ + 8 files changed, 289 insertions(+), 16 deletions(-) + create mode 100644 test/specs/verify/CMakeLists.txt + create mode 100644 test/specs/verify/verify_ut.cc + +diff --git a/src/daemon/common/sysinfo.c b/src/daemon/common/sysinfo.c +index 957b370b..39338925 100644 +--- a/src/daemon/common/sysinfo.c ++++ b/src/daemon/common/sysinfo.c +@@ -393,6 +393,7 @@ sysinfo_t *get_sys_info(bool quiet) + } + + sysinfo->ncpus = get_nprocs(); ++ sysinfo->ncpus_conf = get_nprocs_conf(); + + cgroup_version = common_get_cgroup_version(); + if (cgroup_version < 0) { +diff --git a/src/daemon/common/sysinfo.h b/src/daemon/common/sysinfo.h +index 4ac65df6..363576a9 100644 +--- a/src/daemon/common/sysinfo.h ++++ b/src/daemon/common/sysinfo.h +@@ -25,7 +25,10 @@ extern "C" { + #include "cgroup.h" + + typedef struct { ++ // Number of processors currently online (i.e., available). + int ncpus; ++ // Number of processors configured. ++ int ncpus_conf; + cgroup_mem_info_t cgmeminfo; + cgroup_cpu_info_t cgcpuinfo; + cgroup_hugetlb_info_t hugetlbinfo; +diff --git a/src/daemon/modules/spec/verify.c b/src/daemon/modules/spec/verify.c +index 850595ed..2a8b3259 100644 +--- a/src/daemon/modules/spec/verify.c ++++ b/src/daemon/modules/spec/verify.c +@@ -25,6 +25,7 @@ + #include <errno.h> + #include <fcntl.h> + #include <isula_libutils/auto_cleanup.h> ++#include <isula_libutils/utils_macro.h> + #include <isula_libutils/container_config.h> + #include <isula_libutils/defs.h> + #include <isula_libutils/host_config.h> +@@ -560,7 +561,7 @@ static bool check_cpu(const char *provided, const char *available) + } + + /* parse unit list */ +-int parse_unit_list(const char *val, bool *available_list, int cpu_num) ++STATIC int parse_unit_list(const char *val, bool *available_list, int cpu_num) + { + int ret = -1; + char *str = NULL; +@@ -612,22 +613,13 @@ out: + } + + /* is cpuset list available */ +-static bool is_cpuset_list_available(const char *provided, const char *available) ++STATIC bool is_cpuset_list_available(const char *provided, const char *available, int cpu_num) + { +- int cpu_num = 0; + int i = 0; + bool ret = false; + bool *parsed_provided = NULL; + bool *parsed_available = NULL; +- sysinfo_t *sysinfo = NULL; +- +- sysinfo = get_sys_info(true); +- if (sysinfo == NULL) { +- ERROR("get sysinfo failed"); +- return false; +- } + +- cpu_num = sysinfo->ncpus; + parsed_provided = util_smart_calloc_s(sizeof(bool), (unsigned int)cpu_num); + if (parsed_provided == NULL) { + ERROR("memory alloc failed!"); +@@ -661,10 +653,10 @@ out: + } + + /* is cpuset cpus available */ +-bool is_cpuset_cpus_available(const sysinfo_t *sysinfo, const char *cpus) ++STATIC bool is_cpuset_cpus_available(const sysinfo_t *sysinfo, const char *cpus) + { + bool ret = false; +- ret = is_cpuset_list_available(cpus, sysinfo->cpusetinfo.cpus); ++ ret = is_cpuset_list_available(cpus, sysinfo->cpusetinfo.cpus, sysinfo->ncpus_conf); + if (!ret) { + ERROR("Checking cpuset.cpus got invalid format: %s.", cpus); + isulad_set_error_message("Checking cpuset.cpus got invalid format: %s.", cpus); +@@ -673,10 +665,10 @@ bool is_cpuset_cpus_available(const sysinfo_t *sysinfo, const char *cpus) + } + + /* is cpuset mems available */ +-bool is_cpuset_mems_available(const sysinfo_t *sysinfo, const char *mems) ++STATIC bool is_cpuset_mems_available(const sysinfo_t *sysinfo, const char *mems) + { + bool ret = false; +- ret = is_cpuset_list_available(mems, sysinfo->cpusetinfo.mems); ++ ret = is_cpuset_list_available(mems, sysinfo->cpusetinfo.mems, sysinfo->ncpus_conf); + if (!ret) { + ERROR("Checking cpuset.mems got invalid format: %s.", mems); + isulad_set_error_message("Checking cpuset.mems got invalid format: %s.", mems); +@@ -685,7 +677,7 @@ bool is_cpuset_mems_available(const sysinfo_t *sysinfo, const char *mems) + } + + // cpuset subsystem checks and adjustments +-static int verify_resources_cpuset(const sysinfo_t *sysinfo, const char *cpus, const char *mems) ++STATIC int verify_resources_cpuset(const sysinfo_t *sysinfo, const char *cpus, const char *mems) + { + int ret = 0; + bool cpus_available = false; +diff --git a/test/mocks/image_mock.cc b/test/mocks/image_mock.cc +index 7114080c..cebe418d 100644 +--- a/test/mocks/image_mock.cc ++++ b/test/mocks/image_mock.cc +@@ -54,3 +54,19 @@ int im_umount_container_rootfs(const char *image_type, const char *image_name, c + } + return 0; + } ++ ++struct graphdriver_status *im_graphdriver_get_status(void) ++{ ++ if (g_image_mock != nullptr) { ++ return g_image_mock->ImGraphdriverGetStatus(); ++ } ++ return nullptr; ++} ++ ++void im_free_graphdriver_status(struct graphdriver_status *status) ++{ ++ if (g_image_mock != nullptr) { ++ g_image_mock->ImFreeGraphdriverStatus(status); ++ } ++ return; ++} +diff --git a/test/mocks/image_mock.h b/test/mocks/image_mock.h +index 0c7c1e51..f05be516 100644 +--- a/test/mocks/image_mock.h ++++ b/test/mocks/image_mock.h +@@ -28,6 +28,8 @@ public: + const char *container_id)); + MOCK_METHOD3(ImUmountContainerRootfs, int(const char *image_type, const char *image_name, + const char *container_id)); ++ MOCK_METHOD0(ImGraphdriverGetStatus, struct graphdriver_status *()); ++ MOCK_METHOD1(ImFreeGraphdriverStatus, void(struct graphdriver_status *status)); + }; + + void MockImage_SetMock(MockImage *mock); +diff --git a/test/specs/CMakeLists.txt b/test/specs/CMakeLists.txt +index 7acd68a1..bf5ed535 100644 +--- a/test/specs/CMakeLists.txt ++++ b/test/specs/CMakeLists.txt +@@ -2,3 +2,4 @@ project(iSulad_UT) + + add_subdirectory(specs) + add_subdirectory(specs_extend) ++add_subdirectory(verify) +diff --git a/test/specs/verify/CMakeLists.txt b/test/specs/verify/CMakeLists.txt +new file mode 100644 +index 00000000..0e60a39e +--- /dev/null ++++ b/test/specs/verify/CMakeLists.txt +@@ -0,0 +1,85 @@ ++project(iSulad_UT) ++ ++SET(EXE specs_verify_ut) ++ ++add_definitions(-DUNIT_TEST=ON) ++ ++add_executable(${EXE} ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_regex.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_verify.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_array.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_string.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_convert.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_file.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_timestamp.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/util_atomic.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_mount_spec.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_fs.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_cap.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/sha256/sha256.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/path.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map/map.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map/rb_tree.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/sysinfo.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup_v1.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/spec/verify.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/image/oci/oci_ut_common.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/containers_store_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/namespace_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/container_unix_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/engine_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/selinux_label_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/isulad_config_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/image_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc ++ verify_ut.cc) ++ ++target_include_directories(${EXE} PUBLIC ++ ${GTEST_INCLUDE_DIR} ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../include ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/image/oci ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/image ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/image/external ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/api ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/volume ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/runtime ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/container ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/container/restart_manager ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/container/health_check ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/image/oci/storage ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/services ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/services/execution ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/spec/ ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/services/execution/manager ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/events ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/services/execution/execute ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/tar ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/plugin ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/http ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/runtime/engines ++ ${ENGINES_INCS} ++ ${RUNTIME_INCS} ++ ${IMAGE_INCS} ++ ${CMAKE_BINARY_DIR}/conf ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/sha256 ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/config ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/services/graphdriver ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/console ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/image/oci ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks ++ ) ++ ++target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) ++add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/specs/verify/verify_ut.cc b/test/specs/verify/verify_ut.cc +new file mode 100644 +index 00000000..e764e476 +--- /dev/null ++++ b/test/specs/verify/verify_ut.cc +@@ -0,0 +1,173 @@ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Description: specs verify ut ++ * Author: xuxuepeng ++ * Create: 2023-11-16 ++ */ ++ ++#include <stdlib.h> ++#include <stdio.h> ++#include <gtest/gtest.h> ++#include "mock.h" ++#include <gtest/gtest.h> ++#include <gmock/gmock.h> ++#include "sysinfo.h" ++#include "utils.h" ++ ++using namespace std; ++ ++#define HOST_CONFIG_FILE "../../../../test/specs/verify/hostconfig.json" ++#define OCI_RUNTIME_SPEC_FILE "../../../../test/specs/verify/oci_runtime_spec.json" ++ ++extern "C" { ++ int verify_resources_cpuset(const sysinfo_t *sysinfo, const char *cpus, const char *mems); ++} ++ ++/* get sys info */ ++sysinfo_t *create_sys_info_for_cpuset_test(const char *cpus, const char *mems, int ncpus_conf, int ncpus) ++{ ++ sysinfo_t *sysinfo = NULL; ++ ++ sysinfo = (sysinfo_t *)util_common_calloc_s(sizeof(sysinfo_t)); ++ if (sysinfo == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ sysinfo->ncpus = ncpus; ++ sysinfo->ncpus_conf = ncpus_conf; ++ ++ sysinfo->cpusetinfo.cpuset = true; ++ sysinfo->cpusetinfo.cpus = util_strdup_s(cpus); ++ sysinfo->cpusetinfo.mems = util_strdup_s(mems); ++ ++ return sysinfo; ++} ++ ++void test_different_provided_cpus_mems(sysinfo_t *sysinfo, const char *provided_cpus, const char *provided_mems, ++ int expected) ++{ ++ int ret = 0; ++ ret = verify_resources_cpuset(sysinfo, provided_cpus, provided_mems); ++ ASSERT_EQ(ret, expected); ++} ++ ++// Test the case when provided is null, and available is 0-7 ++TEST(test_verify_resources_cpuset, test_0_7) ++{ ++ sysinfo_t *sysinfo = create_sys_info_for_cpuset_test("0-7", "0-7", 8, 8); ++ test_different_provided_cpus_mems(sysinfo, nullptr, nullptr, 0); ++ ++ test_different_provided_cpus_mems(sysinfo, "0", "0", 0); ++ test_different_provided_cpus_mems(sysinfo, "2", "2", 0); ++ test_different_provided_cpus_mems(sysinfo, "7", "7", 0); ++ test_different_provided_cpus_mems(sysinfo, "8", "8", -1); ++ ++ test_different_provided_cpus_mems(sysinfo, "1,2", "1,2", 0); ++ test_different_provided_cpus_mems(sysinfo, "1,3,5", "1,3,5", 0); ++ ++ test_different_provided_cpus_mems(sysinfo, "0-7", "0-7", 0); ++ test_different_provided_cpus_mems(sysinfo, "0-8", "0-8", -1); ++ test_different_provided_cpus_mems(sysinfo, "0-1,3-7", "0-1,3-7", 0); ++ test_different_provided_cpus_mems(sysinfo, "0-1,3,5-7", "0-1,3,5-7", 0); ++ ++ free_sysinfo(sysinfo); ++} ++ ++// Test the case when provided is null, and available is 0-1,3-7 ++TEST(test_verify_resources_cpuset, test_0_1_3_7) ++{ ++ sysinfo_t *sysinfo = create_sys_info_for_cpuset_test("0-1,3-7", "0-1,3-7", 8, 7); ++ test_different_provided_cpus_mems(sysinfo, nullptr, nullptr, 0); ++ ++ test_different_provided_cpus_mems(sysinfo, "0", "0", 0); ++ test_different_provided_cpus_mems(sysinfo, "2", "2", -1); ++ test_different_provided_cpus_mems(sysinfo, "7", "7", 0); ++ test_different_provided_cpus_mems(sysinfo, "8", "8", -1); ++ ++ test_different_provided_cpus_mems(sysinfo, "1,2", "1,2", -1); ++ test_different_provided_cpus_mems(sysinfo, "1,3,5", "1,3,5", 0); ++ ++ test_different_provided_cpus_mems(sysinfo, "0-7", "0-7", -1); ++ test_different_provided_cpus_mems(sysinfo, "0-8", "0-8", -1); ++ test_different_provided_cpus_mems(sysinfo, "0-1,3-7", "0-1,3-7", 0); ++ test_different_provided_cpus_mems(sysinfo, "0-1,3,5-7", "0-1,3,5-7", 0); ++ ++ free_sysinfo(sysinfo); ++} ++ ++// Test the case when provided is null, and available is 0-6 ++TEST(test_verify_resources_cpuset, test_0_6) ++{ ++ sysinfo_t *sysinfo = create_sys_info_for_cpuset_test("0-6", "0-6", 8, 7); ++ ++ test_different_provided_cpus_mems(sysinfo, nullptr, nullptr, 0); ++ ++ test_different_provided_cpus_mems(sysinfo, "0", "0", 0); ++ test_different_provided_cpus_mems(sysinfo, "2", "2", 0); ++ test_different_provided_cpus_mems(sysinfo, "7", "7", -1); ++ test_different_provided_cpus_mems(sysinfo, "8", "8", -1); ++ ++ test_different_provided_cpus_mems(sysinfo, "1,2", "1,2", 0); ++ test_different_provided_cpus_mems(sysinfo, "1,3,5", "1,3,5", 0); ++ ++ test_different_provided_cpus_mems(sysinfo, "0-7", "0-7", -1); ++ test_different_provided_cpus_mems(sysinfo, "0-8", "0-8", -1); ++ test_different_provided_cpus_mems(sysinfo, "0-1,3-7", "0-1,3-7", -1); ++ test_different_provided_cpus_mems(sysinfo, "0-1,3,5-7", "0-1,3,5-7", -1); ++ ++ free_sysinfo(sysinfo); ++} ++ ++// Test the case when provided is null, and available is 1-7 ++TEST(test_verify_resources_cpuset, test_1_7) ++{ ++ sysinfo_t *sysinfo = create_sys_info_for_cpuset_test("1-7", "1-7", 8, 7); ++ ++ test_different_provided_cpus_mems(sysinfo, nullptr, nullptr, 0); ++ ++ test_different_provided_cpus_mems(sysinfo, "0", "0", -1); ++ test_different_provided_cpus_mems(sysinfo, "2", "2", 0); ++ test_different_provided_cpus_mems(sysinfo, "7", "7", 0); ++ test_different_provided_cpus_mems(sysinfo, "8", "8", -1); ++ ++ test_different_provided_cpus_mems(sysinfo, "1,2", "1,2", 0); ++ test_different_provided_cpus_mems(sysinfo, "1,3,5", "1,3,5", 0); ++ ++ test_different_provided_cpus_mems(sysinfo, "0-7", "0-7", -1); ++ test_different_provided_cpus_mems(sysinfo, "0-8", "0-8", -1); ++ test_different_provided_cpus_mems(sysinfo, "0-1,3-7", "0-1,3-7", -1); ++ test_different_provided_cpus_mems(sysinfo, "0-1,3,5-7", "0-1,3,5-7", -1); ++ ++ free_sysinfo(sysinfo); ++} ++ ++// Test the case when provided is null, and available is 0,3 ++TEST(test_verify_resources_cpuset, test_null_03) ++{ ++ sysinfo_t *sysinfo = create_sys_info_for_cpuset_test("0,3", "0,3", 8, 2); ++ test_different_provided_cpus_mems(sysinfo, nullptr, nullptr, 0); ++ ++ test_different_provided_cpus_mems(sysinfo, "0", "0", 0); ++ test_different_provided_cpus_mems(sysinfo, "2", "2", -1); ++ test_different_provided_cpus_mems(sysinfo, "7", "7", -1); ++ test_different_provided_cpus_mems(sysinfo, "8", "8", -1); ++ ++ test_different_provided_cpus_mems(sysinfo, "1,2", "1,2", -1); ++ test_different_provided_cpus_mems(sysinfo, "1,3,5", "1,3,5", -1); ++ ++ test_different_provided_cpus_mems(sysinfo, "0-7", "0-7", -1); ++ test_different_provided_cpus_mems(sysinfo, "0-8", "0-8", -1); ++ test_different_provided_cpus_mems(sysinfo, "0-1,3-7", "0-1,3-7", -1); ++ test_different_provided_cpus_mems(sysinfo, "0-1,3,5-7", "0-1,3,5-7", -1); ++ ++ free_sysinfo(sysinfo); ++} +-- +2.42.0 + diff --git a/0030-modify-daemon-json-default-runtime-to-runc.patch b/0030-modify-daemon-json-default-runtime-to-runc.patch new file mode 100644 index 0000000..dd09a65 --- /dev/null +++ b/0030-modify-daemon-json-default-runtime-to-runc.patch @@ -0,0 +1,52 @@ +From b94f36b3d06abd711449b2e91303dfdd33f9c979 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Tue, 21 Nov 2023 21:31:48 +0800 +Subject: [PATCH 30/64] modify daemon json default runtime to runc + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/cmd/isula/base/create.h | 2 +- + src/cmd/isula/extend/stats.c | 1 - + src/contrib/config/daemon.json | 2 +- + 3 files changed, 2 insertions(+), 3 deletions(-) + +diff --git a/src/cmd/isula/base/create.h b/src/cmd/isula/base/create.h +index 986be41f..9eb471b4 100644 +--- a/src/cmd/isula/base/create.h ++++ b/src/cmd/isula/base/create.h +@@ -332,7 +332,7 @@ extern "C" { + "runtime", \ + 'R', \ + &(cmdargs).runtime, \ +- "Runtime to use for containers(default: lcr)", \ ++ "Runtime to use for containers", \ + NULL }, \ + { CMD_OPT_TYPE_STRING_DUP, \ + false, \ +diff --git a/src/cmd/isula/extend/stats.c b/src/cmd/isula/extend/stats.c +index c11fe218..04485608 100644 +--- a/src/cmd/isula/extend/stats.c ++++ b/src/cmd/isula/extend/stats.c +@@ -41,7 +41,6 @@ struct client_arguments g_cmd_stats_args = { + .showall = false, + .nostream = false, + .original = false, +- .runtime = "lcr", + }; + + static struct isula_stats_response *g_oldstats = NULL; +diff --git a/src/contrib/config/daemon.json b/src/contrib/config/daemon.json +index 4faf4057..966e016a 100644 +--- a/src/contrib/config/daemon.json ++++ b/src/contrib/config/daemon.json +@@ -1,6 +1,6 @@ + { + "group": "isula", +- "default-runtime": "lcr", ++ "default-runtime": "runc", + "graph": "/var/lib/isulad", + "state": "/var/run/isulad", + "log-level": "ERROR", +-- +2.42.0 + diff --git a/0031-modify-CI-for-default-runtime-to-runc.patch b/0031-modify-CI-for-default-runtime-to-runc.patch new file mode 100644 index 0000000..9b8f8d8 --- /dev/null +++ b/0031-modify-CI-for-default-runtime-to-runc.patch @@ -0,0 +1,815 @@ +From c0d86490ba53bf9a33f7569dc31c4ec1ba54f073 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Tue, 21 Nov 2023 21:32:08 +0800 +Subject: [PATCH 31/64] modify CI for default runtime to runc + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + CI/test_cases/container_cases/cni_test.sh | 103 ++++++++++-------- + .../container_cases/exec_additional_gids.sh | 26 +++-- + CI/test_cases/container_cases/export.sh | 10 +- + .../hook_ignore_poststart_error.sh | 10 +- + .../container_cases/hook_spec_test.sh | 12 +- + ...igdata_stream.sh => lcr_bigdata_stream.sh} | 32 +----- + .../container_cases/{exec.sh => lcr_exec.sh} | 2 +- + CI/test_cases/container_cases/nano_cpus.sh | 8 +- + CI/test_cases/container_cases/restart.sh | 14 ++- + CI/test_cases/container_cases/run.sh | 49 +++++---- + ..._stream_runc.sh => runc_bigdata_stream.sh} | 0 + .../{exec_runc.sh => runc_exec.sh} | 0 + CI/test_cases/container_cases/seccomp.sh | 12 +- + CI/test_cases/container_cases/stop.sh | 19 +++- + CI/test_cases/critest.sh | 6 +- + 15 files changed, 166 insertions(+), 137 deletions(-) + rename CI/test_cases/container_cases/{bigdata_stream.sh => lcr_bigdata_stream.sh} (93%) + rename CI/test_cases/container_cases/{exec.sh => lcr_exec.sh} (97%) + rename CI/test_cases/container_cases/{bigdata_stream_runc.sh => runc_bigdata_stream.sh} (100%) + rename CI/test_cases/container_cases/{exec_runc.sh => runc_exec.sh} (100%) + +diff --git a/CI/test_cases/container_cases/cni_test.sh b/CI/test_cases/container_cases/cni_test.sh +index bbc381dd..114cf2a3 100755 +--- a/CI/test_cases/container_cases/cni_test.sh ++++ b/CI/test_cases/container_cases/cni_test.sh +@@ -37,6 +37,10 @@ function do_post() + start_isulad_with_valgrind + } + ++# $1: pod runtime; ++# $2: pod config; ++# $3: eth0 ip; ++# $4: eth1 ip; + function do_test_help() + { + msg_info "this is $0 do_test" +@@ -53,7 +57,7 @@ function do_test_help() + TC_RET_T=$(($TC_RET_T+1)) + fi + +- sid=`crictl runp ${data_path}/$1` ++ sid=`crictl runp --runtime $1 ${data_path}/$2` + if [ $? -ne 0 ]; then + msg_err "Failed to run sandbox" + TC_RET_T=$(($TC_RET_T+1)) +@@ -61,7 +65,7 @@ function do_test_help() + + cnt=`ls /var/lib/cni/results/* | wc -l` + target_cnt=1 +- if [ "x$3" != "x" ];then ++ if [ "x$4" != "x" ];then + target_cnt=2 + fi + +@@ -77,7 +81,7 @@ function do_test_help() + TC_RET_T=$(($TC_RET_T+1)) + fi + +- cid=`crictl create $sid ${data_path}/container-config.json ${data_path}/$1` ++ cid=`crictl create $sid ${data_path}/container-config.json ${data_path}/$2` + if [ $? -ne 0 ];then + msg_err "create container failed" + TC_RET_T=$(($TC_RET_T+1)) +@@ -107,29 +111,29 @@ function do_test_help() + nsenter -t $con_pid -n ifconfig eth0 + TC_RET_T=$(($TC_RET_T+1)) + fi +- nsenter -t $pod_pid -n ifconfig eth0 | grep "$2" ++ nsenter -t $pod_pid -n ifconfig eth0 | grep "$3" + if [ $? -ne 0 ];then +- msg_err "expect ip: $1, get: " ++ msg_err "expect ip: $3, get: " + nsenter -t $pod_pid -n ifconfig eth0 + TC_RET_T=$(($TC_RET_T+1)) + fi +- crictl inspectp $sid | grep "$2" ++ crictl inspectp $sid | grep "$3" + if [ $? -ne 0 ];then +- msg_err "inspectp: expect ip: $1, get: " ++ msg_err "inspectp: expect ip: $3, get: " + crictl inspectp $sid + TC_RET_T=$(($TC_RET_T+1)) + fi + +- if [ "x$3" != "x" ];then +- nsenter -t $pod_pid -n ifconfig eth1 | grep "$3" ++ if [ "x$4" != "x" ];then ++ nsenter -t $pod_pid -n ifconfig eth1 | grep "$4" + if [ $? -ne 0 ];then +- msg_err "expect ip: $2, get: " ++ msg_err "expect ip: $4, get: " + nsenter -t $pod_pid -n ifconfig eth1 + TC_RET_T=$(($TC_RET_T+1)) + fi +- crictl inspectp $sid | grep "$3" ++ crictl inspectp $sid | grep "$4" + if [ $? -ne 0 ];then +- msg_err "inspectp expect ip: $2, get: " ++ msg_err "inspectp expect ip: $4, get: " + crictl inspectp $sid + TC_RET_T=$(($TC_RET_T+1)) + fi +@@ -170,7 +174,7 @@ function do_test_help() + + function default_cni_config() + { +- do_test_help "sandbox-config.json" "10\.1\." ++ do_test_help $1 "sandbox-config.json" "10\.1\." + } + + function new_cni_config() +@@ -189,12 +193,12 @@ function new_cni_config() + fi + done + tail $ISUALD_LOG +- do_test_help "mutlnet_pod.json" "10\.2\." "10\.1\." ++ do_test_help $1 "mutlnet_pod.json" "10\.2\." "10\.1\." + } + + function check_annotation_extension() + { +- sid=`crictl runp ${data_path}/sandbox-config.json` ++ sid=`crictl runp --runtime $1 ${data_path}/sandbox-config.json` + if [ $? -ne 0 ]; then + msg_err "Failed to run sandbox" + TC_RET_T=$(($TC_RET_T+1)) +@@ -253,7 +257,7 @@ function check_rollback() + done + tail $ISUALD_LOG + +- crictl runp ${data_path}/mutl_wrong_net_pod.json ++ crictl runp --runtime $1 ${data_path}/mutl_wrong_net_pod.json + if [ $? -eq 0 ]; then + msg_err "Run sandbox success with invalid cni configs" + TC_RET_T=$(($TC_RET_T+1)) +@@ -302,13 +306,14 @@ function check_rollback() + # $2: expect ingress rate; + # $3: input egress rate; + # $4: expect egress rate; ++# $5: pod runtime; + function check_annotation_valid_bandwidth() + { + rm bandwidth.json + cp ${data_path}/mock_sandbox.json bandwidth.json + sed -i "s#ingressholder#$1#g" bandwidth.json + sed -i "s#engressholder#$3#g" bandwidth.json +- sid=`crictl runp bandwidth.json` ++ sid=`crictl runp --runtime $5 bandwidth.json` + if [ $? -ne 0 ]; then + msg_err "Failed to run sandbox" + TC_RET_T=$(($TC_RET_T+1)) +@@ -345,6 +350,7 @@ function check_annotation_valid_bandwidth() + return $TC_RET_T + } + ++# function not called + function check_annotation_invalid_bandwidth() + { + rm bandwidth.json +@@ -386,44 +392,51 @@ function check_annotation() + done + tail $ISUALD_LOG + +- check_annotation_extension ++ check_annotation_extension $1 + +- check_annotation_valid_bandwidth "10.24k" "10240" "-1.024k" "-1024" +- check_annotation_valid_bandwidth "1024m" "2" "-1024m" "-1" +- check_annotation_valid_bandwidth "1.000001Ki" "1025" "-1.00001Ki" "-1024" +- check_annotation_valid_bandwidth "0.1Mi" "104858" "-0.01Mi" "-10485" +- check_annotation_valid_bandwidth "1.00001e2" "101" "-1.0001e2" "-100" ++ check_annotation_valid_bandwidth "10.24k" "10240" "-1.024k" "-1024" $1 ++ check_annotation_valid_bandwidth "1024m" "2" "-1024m" "-1" $1 ++ check_annotation_valid_bandwidth "1.000001Ki" "1025" "-1.00001Ki" "-1024" $1 ++ check_annotation_valid_bandwidth "0.1Mi" "104858" "-0.01Mi" "-10485" $1 ++ check_annotation_valid_bandwidth "1.00001e2" "101" "-1.0001e2" "-100" $1 + + return $TC_RET_T + } + +-ret=0 ++function do_test_t() ++{ ++ local ret=0 ++ local runtime=$1 ++ local test="cni_test => (${runtime})" ++ msg_info "${test} starting..." ++ ++ default_cni_config $runtime || ((ret++)) ++ ++ new_cni_config $runtime || ((ret++)) ++ ++ check_annotation $runtime || ((ret++)) + +-do_pre +-if [ $? -ne 0 ];then +- let "ret=$ret + 1" +-fi ++ check_rollback $runtime || ((ret++)) + +-default_cni_config +-if [ $? -ne 0 ];then +- let "ret=$ret + 1" +-fi ++ msg_info "${test} finished with return ${ret}..." + +-new_cni_config +-if [ $? -ne 0 ];then +- let "ret=$ret + 1" +-fi ++ return $ret ++} + +-check_annotation +-if [ $? -ne 0 ];then +- let "ret=$ret + 1" +-fi ++ret=0 + +-check_rollback +-if [ $? -ne 0 ];then +- let "ret=$ret + 1" +-fi ++for element in ${RUNTIME_LIST[@]}; ++do ++ do_pre ++ if [ $? -ne 0 ];then ++ let "ret=$ret + 1" ++ fi + +-do_post ++ do_test_t $element ++ if [ $? -ne 0 ];then ++ let "ret=$ret + 1" ++ fi ++ do_post ++done + + show_result $ret "cni base test" +diff --git a/CI/test_cases/container_cases/exec_additional_gids.sh b/CI/test_cases/container_cases/exec_additional_gids.sh +index f24678d3..2edfd750 100755 +--- a/CI/test_cases/container_cases/exec_additional_gids.sh ++++ b/CI/test_cases/container_cases/exec_additional_gids.sh +@@ -22,7 +22,6 @@ + curr_path=$(dirname $(readlink -f "$0")) + data_path=$(realpath $curr_path/../data) + source ../helpers.sh +-test="exec additional gids test => test_exec_additional_gids" + test_log=$(mktemp /tmp/additional_gids_test_XXX) + + USERNAME="user" +@@ -37,10 +36,14 @@ file_info="Keep it secret, keep it safe" + function additional_gids_test() + { + local ret=0 ++ local runtime=$1 ++ test="exec additional gids test => test_exec_additional_gids => $runtime" ++ ++ msg_info "${test} starting..." + + isula rm -f `isula ps -a -q` + +- isula run -tid -n $cont_name ubuntu bash ++ isula run -tid --runtime $runtime -n $cont_name ubuntu bash + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container" && ((ret++)) + + isula exec $cont_name bash -c "groupadd --gid $USER_GID $USERNAME \ +@@ -52,10 +55,13 @@ function additional_gids_test() + && chmod 606 /app/sekrit.txt" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - create user and group failed" && ((ret++)) + ++ # runc is not support exec --workdir + /usr/bin/expect <<- EOF > ${test_log} 2>&1 + set timeout 10 +-spawn isula exec -it --workdir /app -u $USERNAME $cont_name bash ++spawn isula exec -it -u $USERNAME $cont_name bash + expect "${USERNAME}*" ++send "cd /app\n" ++expect "*" + send "newgrp ${ADDITIONAL_GROUP}\n" + expect "*" + send "groups\n" +@@ -75,18 +81,18 @@ EOF + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - read error message failed" && ((ret++)) + + isula rm -f `isula ps -a -q` ++ rm -rf ${test_log} ++ ++ msg_info "${test} finished with return ${ret}..." + + return ${ret} + } + + declare -i ans=0 + +-msg_info "${test} starting..." +- +-additional_gids_test || ((ans++)) +- +-rm -rf ${test_log} +- +-msg_info "${test} finished with return ${ret}..." ++for element in ${RUNTIME_LIST[@]}; ++do ++ additional_gids_test $element || ((ans++)) ++done + + show_result ${ans} "${curr_path}/${0}" +diff --git a/CI/test_cases/container_cases/export.sh b/CI/test_cases/container_cases/export.sh +index eeef2809..1cff873d 100755 +--- a/CI/test_cases/container_cases/export.sh ++++ b/CI/test_cases/container_cases/export.sh +@@ -26,7 +26,8 @@ function test_image_export() + { + local ret=0 + local image="busybox" +- local test="export container test => (${FUNCNAME[@]})" ++ local runtime=$1 ++ local test="export container test => (${FUNCNAME[@]}) => $runtime" + + msg_info "${test} starting..." + +@@ -36,7 +37,7 @@ function test_image_export() + isula images | grep busybox + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++)) + +- CONT=`isula run -itd busybox` ++ CONT=`isula run --runtime $runtime -itd busybox` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + + isula export -o export.tar ${CONT} +@@ -55,6 +56,9 @@ function test_image_export() + + declare -i ans=0 + +-test_image_export || ((ans++)) ++for element in ${RUNTIME_LIST[@]}; ++do ++ test_image_export $element || ((ans++)) ++done + + show_result ${ans} "${curr_path}/${0}" +diff --git a/CI/test_cases/container_cases/hook_ignore_poststart_error.sh b/CI/test_cases/container_cases/hook_ignore_poststart_error.sh +index 5c86a4c1..8c636f7e 100755 +--- a/CI/test_cases/container_cases/hook_ignore_poststart_error.sh ++++ b/CI/test_cases/container_cases/hook_ignore_poststart_error.sh +@@ -28,7 +28,8 @@ function test_hook_ignore_poststart_error_spec() + { + local ret=0 + local image="busybox" +- local test="container hook test => (${FUNCNAME[@]})" ++ local runtime=$1 ++ local test="container hook test => (${FUNCNAME[@]}) => $runtime" + CONT=test_hook_spec + cp ${test_data_path}/poststart.sh /tmp/ + +@@ -40,7 +41,7 @@ function test_hook_ignore_poststart_error_spec() + isula images | grep busybox + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++)) + +- isula run -n $CONT -itd --hook-spec ${test_data_path}/oci_hook_poststart_check.json ${image} & ++ isula run -n $CONT -itd --runtime $runtime --hook-spec ${test_data_path}/oci_hook_poststart_check.json ${image} & + + for a in `seq 20` + do +@@ -74,6 +75,9 @@ function test_hook_ignore_poststart_error_spec() + + declare -i ans=0 + +-test_hook_ignore_poststart_error_spec || ((ans++)) ++for element in ${RUNTIME_LIST[@]}; ++do ++ test_hook_ignore_poststart_error_spec $1 || ((ans++)) ++done + + show_result ${ans} "${curr_path}/${0}" +diff --git a/CI/test_cases/container_cases/hook_spec_test.sh b/CI/test_cases/container_cases/hook_spec_test.sh +index c88ed340..33b7c2e5 100755 +--- a/CI/test_cases/container_cases/hook_spec_test.sh ++++ b/CI/test_cases/container_cases/hook_spec_test.sh +@@ -28,7 +28,8 @@ function test_hook_spec() + { + local ret=0 + local image="busybox" +- local test="container hook test => (${FUNCNAME[@]})" ++ local runtime=$1 ++ local test="container hook test => (${FUNCNAME[@]}) => $runtime" + msg_info "${test} starting..." + + isula pull ${image} +@@ -37,7 +38,7 @@ function test_hook_spec() + isula images | grep busybox + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++)) + +- CONT=`isula run -itd --hook-spec ${test_data_path}/test-hookspec.json ${image}` ++ CONT=`isula run -itd --runtime $runtime --hook-spec ${test_data_path}/test-hookspec.json ${image}` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + + isula stop -t 0 ${CONT} +@@ -51,7 +52,7 @@ function test_hook_spec() + isula run -n $no_permission_container -itd --hook-spec ${test_data_path}/no_permission.json ${image} > $runlog 2>&1 + [[ $? -ne 126 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check exit code container with image: ${image}" && ((ret++)) + +- cat $runlog | grep "Permission denied" ++ cat $runlog | grep -i "Permission denied" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to get no_permission output: ${image}" && ((ret++)) + + isula rm -f $no_permission_container +@@ -95,6 +96,9 @@ EOF + + declare -i ans=0 + +-test_hook_spec || ((ans++)) ++for element in ${RUNTIME_LIST[@]}; ++do ++ test_hook_spec $element || ((ans++)) ++done + + show_result ${ans} "${curr_path}/${0}" +diff --git a/CI/test_cases/container_cases/bigdata_stream.sh b/CI/test_cases/container_cases/lcr_bigdata_stream.sh +similarity index 93% +rename from CI/test_cases/container_cases/bigdata_stream.sh +rename to CI/test_cases/container_cases/lcr_bigdata_stream.sh +index 3bfc2d50..c8ecc48a 100755 +--- a/CI/test_cases/container_cases/bigdata_stream.sh ++++ b/CI/test_cases/container_cases/lcr_bigdata_stream.sh +@@ -40,7 +40,7 @@ function set_up() + isula images | grep busybox + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++)) + +- CID=$(isula run -itd ${image} sh) ++ CID=$(isula run --runtime lcr -itd ${image} sh) + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + + isula exec -it $CID dd if=/dev/zero of=test_500M bs=1M count=500 +@@ -389,33 +389,6 @@ function test_stream_with_kill_isulad() + return ${ret} + } + +-function test_stream_with_runc() +-{ +- local ret=0 +- local image="busybox" +- local test="test_stream_with_runc => (${FUNCNAME[@]})" +- msg_info "${test} starting..." +- +- RUNCID=$(isula run -itd --runtime runc ${image} sh) +- isula exec -it $RUNCID dd if=/dev/zero of=test_500M bs=1M count=500 +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to create bigdata" && ((ret++)) +- +- isula exec -it $RUNCID cat test_500M > /home/iocopy_stream_data_500M +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to cat bigdata" && ((ret++)) +- +- sync && sync +- total_size=$(stat -c"%s" /home/iocopy_stream_data_500M) +- [[ $total_size -ne 524288000 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stream iocopy loss data" && ((ret++)) +- +- isula rm -f $RUNCID +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container" && ((ret++)) +- +- rm -rf /home/iocopy_stream_data_500M +- +- msg_info "${test} finished with return ${ret}..." +- return ${ret} +-} +- + function tear_down() + { + local ret=0 +@@ -438,7 +411,7 @@ function test_memory_leak_with_bigdata_stream() + + start_isulad_with_valgrind + +- CID=$(isula run -itd ${image} sh) ++ CID=$(isula run --runtime lcr -itd ${image} sh) + + isula exec -it $CID dd if=/dev/zero of=test_100M bs=1M count=100 + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to create bigdata" && ((ret++)) +@@ -477,7 +450,6 @@ test_stream_with_stop_lxc_monitor || ((ans++)) + test_stream_with_kill_lxc_monitor || ((ans++)) + test_stream_with_stop_isulad || ((ans++)) + test_stream_with_kill_isulad || ((ans++)) +-test_stream_with_runc || ((ans++)) + tear_down || ((ans++)) + + test_memory_leak_with_bigdata_stream || ((ans++)) +diff --git a/CI/test_cases/container_cases/exec.sh b/CI/test_cases/container_cases/lcr_exec.sh +similarity index 97% +rename from CI/test_cases/container_cases/exec.sh +rename to CI/test_cases/container_cases/lcr_exec.sh +index 96ceb884..4f51773d 100755 +--- a/CI/test_cases/container_cases/exec.sh ++++ b/CI/test_cases/container_cases/lcr_exec.sh +@@ -30,7 +30,7 @@ function exec_workdir() + + isula rm -f `isula ps -a -q` + +- isula run -tid -n cont_workdir busybox sh ++ isula run -tid --runtime lcr -n cont_workdir busybox sh + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with --workdir" && ((ret++)) + + isula exec -ti --workdir /workdir cont_workdir pwd | grep "/workdir" +diff --git a/CI/test_cases/container_cases/nano_cpus.sh b/CI/test_cases/container_cases/nano_cpus.sh +index c679958d..85223038 100755 +--- a/CI/test_cases/container_cases/nano_cpus.sh ++++ b/CI/test_cases/container_cases/nano_cpus.sh +@@ -26,7 +26,8 @@ function test_cpu_nano_spec() + { + local ret=0 + local image="busybox" +- local test="container blkio nano test => (${FUNCNAME[@]})" ++ local runtime=$1 ++ local test="container blkio nano test => (${FUNCNAME[@]}) => $runtime" + + msg_info "${test} starting..." + +@@ -108,6 +109,9 @@ function test_cpu_nano_spec() + + declare -i ans=0 + +-test_cpu_nano_spec || ((ans++)) ++for element in ${RUNTIME_LIST[@]}; ++do ++ test_cpu_nano_spec $element || ((ans++)) ++done + + show_result ${ans} "${curr_path}/${0}" +diff --git a/CI/test_cases/container_cases/restart.sh b/CI/test_cases/container_cases/restart.sh +index 5902af06..fddee1f7 100755 +--- a/CI/test_cases/container_cases/restart.sh ++++ b/CI/test_cases/container_cases/restart.sh +@@ -26,7 +26,8 @@ source ../helpers.sh + function do_test_t() + { + containername=test_restart +- isula run --name $containername -td busybox ++ ++ isula run --runtime $1 --name $containername -td busybox + fn_check_eq "$?" "0" "run failed" + testcontainer $containername running + +@@ -46,9 +47,12 @@ function do_test_t() + + ret=0 + +-do_test_t +-if [ $? -ne 0 ];then +- let "ret=$ret + 1" +-fi ++for element in ${RUNTIME_LIST[@]}; ++do ++ do_test_t $element ++ if [ $? -ne 0 ];then ++ let "ret=$ret + 1" ++ fi ++done + + show_result $ret "basic restart" +diff --git a/CI/test_cases/container_cases/run.sh b/CI/test_cases/container_cases/run.sh +index ad449402..8ea3e514 100755 +--- a/CI/test_cases/container_cases/run.sh ++++ b/CI/test_cases/container_cases/run.sh +@@ -25,7 +25,7 @@ source ../helpers.sh + + function do_test_t() + { +- tid=`isula run -tid --name hostname busybox` ++ tid=`isula run --runtime $1 -tid --name hostname busybox` + chostname=`isula exec -it $tid hostname` + fn_check_eq "$chostname" "${tid:0:12}" "default hostname is id of container" + isula exec -it hostname env | grep HOSTNAME +@@ -37,7 +37,7 @@ function do_test_t() + containername=test_basic_run + containername2=container_to_join + +- isula run --name $containername -td busybox ++ isula run --runtime $1 --name $containername -td busybox + fn_check_eq "$?" "0" "run failed" + testcontainer $containername running + +@@ -48,7 +48,7 @@ function do_test_t() + isula rm $containername + fn_check_eq "$?" "0" "rm failed" + +- isula run --name $containername -td -v /dev/shm:/dev/shm busybox ++ isula run --runtime $1 --name $containername -td -v /dev/shm:/dev/shm busybox + fn_check_eq "$?" "0" "run failed" + testcontainer $containername running + +@@ -61,7 +61,7 @@ function do_test_t() + + echo AA > /tmp/test_run_env + +- isula run --name $containername -itd --user 100:100 -e AAA=BB -e BAA --env-file /tmp/test_run_env busybox ++ isula run --runtime $1 --name $containername -itd --user 100:100 -e AAA=BB -e BAA --env-file /tmp/test_run_env busybox + fn_check_eq "$?" "0" "run failed" + testcontainer $containername running + +@@ -72,18 +72,21 @@ function do_test_t() + isula rm $containername + fn_check_eq "$?" "0" "rm failed" + +- isula run --name $containername -itd --external-rootfs / --read-only none sh +- fn_check_eq "$?" "0" "run container with host rootfs failed" +- testcontainer $containername running ++ # runc directly uses the root directory as external rootfs and will report the error pivot_root .: device or resource busy ++ if [ $runtime == "lcr" ]; then ++ isula run --runtime $1 --name $containername -itd --external-rootfs / --read-only none sh ++ fn_check_eq "$?" "0" "run container with host rootfs failed" ++ testcontainer $containername running + +- isula stop -t 0 $containername +- fn_check_eq "$?" "0" "stop failed" +- testcontainer $containername exited ++ isula stop -t 0 $containername ++ fn_check_eq "$?" "0" "stop failed" ++ testcontainer $containername exited + +- isula rm $containername +- fn_check_eq "$?" "0" "rm failed" ++ isula rm $containername ++ fn_check_eq "$?" "0" "rm failed" ++ fi + +- isula run --name $containername -itd --net=host --pid=host --ipc=host --uts=host busybox ++ isula run --runtime $1 --name $containername -itd --net=host --pid=host --ipc=host --uts=host busybox + fn_check_eq "$?" "0" "run failed" + testcontainer $containername running + +@@ -94,7 +97,7 @@ function do_test_t() + isula rm $containername + fn_check_eq "$?" "0" "rm failed" + +- isula run --name $containername -itd --net=none --pid=none --ipc=none --uts=none busybox ++ isula run --runtime $1 --name $containername -itd --net=none --pid=none --ipc=none --uts=none busybox + fn_check_eq "$?" "0" "run failed" + testcontainer $containername running + +@@ -105,11 +108,11 @@ function do_test_t() + isula rm $containername + fn_check_eq "$?" "0" "rm failed" + +- isula run --name $containername2 -itd busybox ++ isula run --runtime $1 --name $containername2 -itd busybox + fn_check_eq "$?" "0" "run failed" + testcontainer $containername2 running + +- isula run --name $containername -itd --net=container:$containername2 --pid=container:$containername2 --ipc=container:$containername2 --uts=container:$containername2 busybox ++ isula run --runtime $1 --name $containername -itd --net=container:$containername2 --pid=container:$containername2 --ipc=container:$containername2 --uts=container:$containername2 busybox + fn_check_eq "$?" "0" "run failed" + testcontainer $containername running + +@@ -135,7 +138,7 @@ function do_run_remote_test_t() + local ret=0 + local image="busybox" + local config='tcp://127.0.0.1:2890' +- local test="container start with --attach remote test => (${FUNCNAME[@]})" ++ local test="container start with --attach remote test => (${FUNCNAME[@]}) => $1" + + check_valgrind_log + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop isulad failed" && ((ret++)) +@@ -144,13 +147,13 @@ function do_run_remote_test_t() + + containername=run_remote + +- isula run -ti -H "$config" --name $containername busybox xxx ++ isula run --runtime $1 -ti -H "$config" --name $containername busybox xxx + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed check invalid run ${containername} remote" && ((ret++)) + testcontainer $containername exited + isula rm -f -H "$config" $containername + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container remote" && ((ret++)) + +- isula run -ti -H "$config" --name $containername busybox /bin/sh -c 'echo "hello"' | grep hello ++ isula run --runtime $1 -ti -H "$config" --name $containername busybox /bin/sh -c 'echo "hello"' | grep hello + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run ${containername} remote" && ((ret++)) + testcontainer $containername exited + +@@ -169,8 +172,10 @@ function do_run_remote_test_t() + + declare -i ans=0 + +-do_test_t || ((ans++)) +- +-do_run_remote_test_t || ((ans++)) ++for element in ${RUNTIME_LIST[@]}; ++do ++ do_test_t $element || ((ans++)) ++ do_run_remote_test_t $element || ((ans++)) ++done + + show_result ${ans} "${curr_path}/${0}" +diff --git a/CI/test_cases/container_cases/bigdata_stream_runc.sh b/CI/test_cases/container_cases/runc_bigdata_stream.sh +similarity index 100% +rename from CI/test_cases/container_cases/bigdata_stream_runc.sh +rename to CI/test_cases/container_cases/runc_bigdata_stream.sh +diff --git a/CI/test_cases/container_cases/exec_runc.sh b/CI/test_cases/container_cases/runc_exec.sh +similarity index 100% +rename from CI/test_cases/container_cases/exec_runc.sh +rename to CI/test_cases/container_cases/runc_exec.sh +diff --git a/CI/test_cases/container_cases/seccomp.sh b/CI/test_cases/container_cases/seccomp.sh +index 9e886d10..3cb08d84 100755 +--- a/CI/test_cases/container_cases/seccomp.sh ++++ b/CI/test_cases/container_cases/seccomp.sh +@@ -39,8 +39,9 @@ function do_pre() { + + function do_test() { + local ret=0 +- +- msg_info "this is $0 do_test" ++ local runtime=$1 ++ local test="seccomp test => (${runtime})" ++ msg_info "${test} starting..." + + cid1=$(isula run -tid --security-opt seccomp=/etc/isulad/seccomp_default.json busybox sh) + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Failed to run container with the default seccomp profile" && ((ret++)) +@@ -52,7 +53,7 @@ function do_test() { + --security-opt seccomp=${test_data_path}/seccomp_profile_without_archmap.json busybox sh) + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Failed to run container with multiple seccomp profiles" && ((ret++)) + +- isula stop "${cid1}" "${cid2}" "${cid3}" ++ isula stop -t 0 "${cid1}" "${cid2}" "${cid3}" + + isula rm -f $(isula ps -qa) + +@@ -69,7 +70,10 @@ declare -i ans=0 + + do_pre || ((ans++)) + +-do_test || ((ans++)) ++for element in ${RUNTIME_LIST[@]}; ++do ++ do_test $element || ((ans++)) ++done + + do_post + +diff --git a/CI/test_cases/container_cases/stop.sh b/CI/test_cases/container_cases/stop.sh +index 962e72f3..13292710 100755 +--- a/CI/test_cases/container_cases/stop.sh ++++ b/CI/test_cases/container_cases/stop.sh +@@ -25,8 +25,12 @@ source ../helpers.sh + + function do_test_t() + { ++ local runtime=$1 ++ local test="start_test => (${runtime})" ++ msg_info "${test} starting..." ++ + containername=test_stop +- isula run --name $containername -td busybox ++ isula run --runtime $runtime --name $containername -td busybox + fn_check_eq "$?" "0" "run failed" + testcontainer $containername running + +@@ -61,14 +65,19 @@ function do_test_t() + isula rm $containername + fn_check_eq "$?" "0" "rm failed" + ++ msg_info "${test} finished with return ${ret}..." ++ + return $TC_RET_T + } + + ret=0 + +-do_test_t +-if [ $? -ne 0 ];then +- let "ret=$ret + 1" +-fi ++for element in ${RUNTIME_LIST[@]}; ++do ++ do_test_t $element ++ if [ $? -ne 0 ];then ++ let "ret=$ret + 1" ++ fi ++done + + show_result $ret "basic stop" +diff --git a/CI/test_cases/critest.sh b/CI/test_cases/critest.sh +index 044ce2ed..f8d4975e 100755 +--- a/CI/test_cases/critest.sh ++++ b/CI/test_cases/critest.sh +@@ -130,7 +130,7 @@ function test_critest() { + function do_test_t() { + local ret=0 + +- local runtime="lcr" ++ local runtime="runc" + local test="critest => $runtime" + msg_info "${test} starting..." + echo "${test}" >> ${testcase_data}/critest.log +@@ -143,11 +143,11 @@ function do_test_t() { + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop isulad failed" && ((ret++)) + + # replace default runtime +- sed -i 's/"default-runtime": "lcr"/"default-runtime": "runc"/g' /etc/isulad/daemon.json ++ sed -i 's/"default-runtime": "runc"/"default-runtime": "lcr"/g' /etc/isulad/daemon.json + start_isulad_without_valgrind --selinux-enabled --network-plugin cni + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start isulad with selinux and cni failed" && ((ret++)) + +- runtime=runc ++ runtime=lcr + test="critest => $runtime" + msg_info "${test} starting..." + echo "${test}" >> ${testcase_data}/critest.log +-- +2.42.0 + diff --git a/0032-add-ut-for-devicemapper.patch b/0032-add-ut-for-devicemapper.patch new file mode 100644 index 0000000..e28c5fa --- /dev/null +++ b/0032-add-ut-for-devicemapper.patch @@ -0,0 +1,737 @@ +From ca297d26dc1e7b47d6987c6bbbd92dd2e3d78670 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Wed, 22 Nov 2023 22:05:04 +0800 +Subject: [PATCH 32/64] add ut for devicemapper + +Signed-off-by: jikai <jikai11@huawei.com> +--- + test/image/oci/storage/layers/CMakeLists.txt | 2 + + .../storage/layers/devmapper/CMakeLists.txt | 71 +++++ + ...9702e4bd316dd50ae85467b0378a419b23b60ba73d | 6 + + ...a9fb83febf6dc0b1548dfe896161533668281c9f4f | 6 + + ...0a625721fdbea5c94ca6da897acdd814d710149770 | 6 + + .../devmapper/data/devicemapper/metadata/base | 7 + + .../devicemapper/metadata/deviceset-metadata | 5 + + .../metadata/transaction-metadata | 5 + + .../layers/devmapper/driver_devmapper_ut.cc | 283 ++++++++++++++++++ + test/mocks/libdevmapper_mock.cc | 191 ++++++++++++ + test/mocks/libdevmapper_mock.h | 52 ++++ + 11 files changed, 634 insertions(+) + create mode 100644 test/image/oci/storage/layers/devmapper/CMakeLists.txt + create mode 100644 test/image/oci/storage/layers/devmapper/data/devicemapper/metadata/068615102be4457b22d40c9702e4bd316dd50ae85467b0378a419b23b60ba73d + create mode 100644 test/image/oci/storage/layers/devmapper/data/devicemapper/metadata/3d24ee258efc3bfe4066a1a9fb83febf6dc0b1548dfe896161533668281c9f4f + create mode 100644 test/image/oci/storage/layers/devmapper/data/devicemapper/metadata/ba0dae6243cc9fa2890df40a625721fdbea5c94ca6da897acdd814d710149770 + create mode 100644 test/image/oci/storage/layers/devmapper/data/devicemapper/metadata/base + create mode 100644 test/image/oci/storage/layers/devmapper/data/devicemapper/metadata/deviceset-metadata + create mode 100644 test/image/oci/storage/layers/devmapper/data/devicemapper/metadata/transaction-metadata + create mode 100644 test/image/oci/storage/layers/devmapper/driver_devmapper_ut.cc + create mode 100644 test/mocks/libdevmapper_mock.cc + create mode 100644 test/mocks/libdevmapper_mock.h + +diff --git a/test/image/oci/storage/layers/CMakeLists.txt b/test/image/oci/storage/layers/CMakeLists.txt +index 413a8b38..e1c76453 100644 +--- a/test/image/oci/storage/layers/CMakeLists.txt ++++ b/test/image/oci/storage/layers/CMakeLists.txt +@@ -1,5 +1,7 @@ + project(iSulad_UT) + ++add_subdirectory(devmapper) ++ + # storage_driver_ut + SET(DRIVER_EXE storage_driver_ut) + +diff --git a/test/image/oci/storage/layers/devmapper/CMakeLists.txt b/test/image/oci/storage/layers/devmapper/CMakeLists.txt +new file mode 100644 +index 00000000..f98de1a8 +--- /dev/null ++++ b/test/image/oci/storage/layers/devmapper/CMakeLists.txt +@@ -0,0 +1,71 @@ ++project(iSulad_UT) ++ ++# driver_devmapper_ut ++SET(DRIVER_DEVMAPPER_EXE driver_devmapper_ut) ++ ++add_executable(${DRIVER_DEVMAPPER_EXE} ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/utils/cutils/utils.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/utils/cutils/utils_regex.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/utils/cutils/utils_verify.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/utils/cutils/utils_array.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/utils/cutils/utils_string.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/utils/cutils/utils_convert.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/utils/cutils/utils_file.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/utils/cutils/utils_fs.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/utils/cutils/util_atomic.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/utils/cutils/utils_base64.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/utils/cutils/utils_timestamp.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/utils/cutils/path.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/utils/cutils/map/map.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/utils/cutils/map/rb_tree.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/utils/buffer/buffer.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/utils/tar/util_archive.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/utils/tar/util_gzip.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/utils/sha256/sha256.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/config/daemon_arguments.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/config/isulad_config.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/common/err_msg.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/common/selinux_label.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/deviceset.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/driver_devmapper.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/metadata_store.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/wrapper_devmapper.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../mocks/libdevmapper_mock.cc ++ driver_devmapper_ut.cc) ++ ++target_include_directories(${DRIVER_DEVMAPPER_EXE} PUBLIC ++ ${GTEST_INCLUDE_DIR} ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../include ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/utils/tar ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/utils/cutils ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/utils/cutils/map ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/utils/sha256 ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/utils/console ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/utils/buffer ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/config ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/modules/api ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/modules/image ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/modules/image/oci/storage ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2 ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/modules/image/oci/storage/remote_layer_support ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/quota ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../mocks ++ ) ++ ++set_target_properties(${DRIVER_DEVMAPPER_EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,util_exec_cmd -Wl,--wrap,util_mount -Wl,--wrap,umount2") ++ ++target_link_libraries(${DRIVER_DEVMAPPER_EXE} ++ ${GTEST_BOTH_LIBRARIES} ++ ${GMOCK_LIBRARY} ++ ${GMOCK_MAIN_LIBRARY} ++ ${CMAKE_THREAD_LIBS_INIT} ++ ${ISULA_LIBUTILS_LIBRARY} ++ ${LIBTAR_LIBRARY} ++ -lcrypto -lyajl -larchive ${SELINUX_LIBRARY} -lz -lcap) ++ ++add_test(NAME ${DRIVER_DEVMAPPER_EXE} COMMAND ${DRIVER_DEVMAPPER_EXE} --gtest_output=xml:${DRIVER_DEVMAPPER_EXE}-Results.xml) ++set_tests_properties(${DRIVER_DEVMAPPER_EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/image/oci/storage/layers/devmapper/data/devicemapper/metadata/068615102be4457b22d40c9702e4bd316dd50ae85467b0378a419b23b60ba73d b/test/image/oci/storage/layers/devmapper/data/devicemapper/metadata/068615102be4457b22d40c9702e4bd316dd50ae85467b0378a419b23b60ba73d +new file mode 100644 +index 00000000..f51ae926 +--- /dev/null ++++ b/test/image/oci/storage/layers/devmapper/data/devicemapper/metadata/068615102be4457b22d40c9702e4bd316dd50ae85467b0378a419b23b60ba73d +@@ -0,0 +1,6 @@ ++{ ++ "hash": "068615102be4457b22d40c9702e4bd316dd50ae85467b0378a419b23b60ba73d", ++ "device_id": 6, ++ "size": 10737418240, ++ "transaction_id": 8 ++} +diff --git a/test/image/oci/storage/layers/devmapper/data/devicemapper/metadata/3d24ee258efc3bfe4066a1a9fb83febf6dc0b1548dfe896161533668281c9f4f b/test/image/oci/storage/layers/devmapper/data/devicemapper/metadata/3d24ee258efc3bfe4066a1a9fb83febf6dc0b1548dfe896161533668281c9f4f +new file mode 100644 +index 00000000..de727a79 +--- /dev/null ++++ b/test/image/oci/storage/layers/devmapper/data/devicemapper/metadata/3d24ee258efc3bfe4066a1a9fb83febf6dc0b1548dfe896161533668281c9f4f +@@ -0,0 +1,6 @@ ++{ ++ "hash": "3d24ee258efc3bfe4066a1a9fb83febf6dc0b1548dfe896161533668281c9f4f", ++ "device_id": 4, ++ "size": 10737418240, ++ "transaction_id": 4 ++} +diff --git a/test/image/oci/storage/layers/devmapper/data/devicemapper/metadata/ba0dae6243cc9fa2890df40a625721fdbea5c94ca6da897acdd814d710149770 b/test/image/oci/storage/layers/devmapper/data/devicemapper/metadata/ba0dae6243cc9fa2890df40a625721fdbea5c94ca6da897acdd814d710149770 +new file mode 100644 +index 00000000..e1e8988e +--- /dev/null ++++ b/test/image/oci/storage/layers/devmapper/data/devicemapper/metadata/ba0dae6243cc9fa2890df40a625721fdbea5c94ca6da897acdd814d710149770 +@@ -0,0 +1,6 @@ ++{ ++ "hash": "ba0dae6243cc9fa2890df40a625721fdbea5c94ca6da897acdd814d710149770", ++ "device_id": 2, ++ "size": 10737418240, ++ "transaction_id": 2 ++} +diff --git a/test/image/oci/storage/layers/devmapper/data/devicemapper/metadata/base b/test/image/oci/storage/layers/devmapper/data/devicemapper/metadata/base +new file mode 100644 +index 00000000..2412113d +--- /dev/null ++++ b/test/image/oci/storage/layers/devmapper/data/devicemapper/metadata/base +@@ -0,0 +1,7 @@ ++{ ++ "hash": "base", ++ "device_id": 1, ++ "size": 10737418240, ++ "transaction_id": 1, ++ "initialized": true ++} +diff --git a/test/image/oci/storage/layers/devmapper/data/devicemapper/metadata/deviceset-metadata b/test/image/oci/storage/layers/devmapper/data/devicemapper/metadata/deviceset-metadata +new file mode 100644 +index 00000000..94f7a6a3 +--- /dev/null ++++ b/test/image/oci/storage/layers/devmapper/data/devicemapper/metadata/deviceset-metadata +@@ -0,0 +1,5 @@ ++{ ++ "next_device_id": 7, ++ "BaseDeviceFilesystem": "ext4", ++ "BaseDeviceUUID": "4fa22307-0c88-4fa4-8f16-a9459e9cbc4a" ++} +diff --git a/test/image/oci/storage/layers/devmapper/data/devicemapper/metadata/transaction-metadata b/test/image/oci/storage/layers/devmapper/data/devicemapper/metadata/transaction-metadata +new file mode 100644 +index 00000000..a011249a +--- /dev/null ++++ b/test/image/oci/storage/layers/devmapper/data/devicemapper/metadata/transaction-metadata +@@ -0,0 +1,5 @@ ++{ ++ "open_transaction_id": 8, ++ "device_hash": "068615102be4457b22d40c9702e4bd316dd50ae85467b0378a419b23b60ba73d", ++ "device_id": 6 ++} +diff --git a/test/image/oci/storage/layers/devmapper/driver_devmapper_ut.cc b/test/image/oci/storage/layers/devmapper/driver_devmapper_ut.cc +new file mode 100644 +index 00000000..59e53f97 +--- /dev/null ++++ b/test/image/oci/storage/layers/devmapper/driver_devmapper_ut.cc +@@ -0,0 +1,283 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2023-11-22 ++ * Description: provide oci storage driver unit test for devmapper ++ ******************************************************************************/ ++ ++#include <gtest/gtest.h> ++#include <gmock/gmock.h> ++ ++#include "driver_devmapper.h" ++#include "mock.h" ++#include "path.h" ++#include "utils.h" ++#include "libdevmapper_mock.h" ++ ++using ::testing::Invoke; ++using ::testing::NiceMock; ++using ::testing::Return; ++using ::testing::_; ++ ++extern "C" { ++ DECLARE_WRAPPER_V(util_exec_cmd, bool, (exec_func_t cb_func, void *args, const char *stdin_msg, char **stdout_msg, char **stderr_msg)); ++ DEFINE_WRAPPER_V(util_exec_cmd, bool, (exec_func_t cb_func, void *args, const char *stdin_msg, char **stdout_msg, char **stderr_msg), (cb_func, args, stdin_msg, stdout_msg, stderr_msg)); ++ ++ DECLARE_WRAPPER(util_mount, int, (const char *src, const char *dst, const char *mtype, const char *mntopts)); ++ DEFINE_WRAPPER(util_mount, int, (const char *src, const char *dst, const char *mtype, const char *mntopts), (src, dst, mtype, mntopts)); ++ ++ DECLARE_WRAPPER(umount2, int, (const char *__special_file, int __flags)); ++ DEFINE_WRAPPER(umount2, int, (const char *__special_file, int __flags), (__special_file, __flags)); ++} ++ ++static std::string GetDirectory() ++{ ++ char abs_path[PATH_MAX] { 0x00 }; ++ int ret = readlink("/proc/self/exe", abs_path, sizeof(abs_path)); ++ if (ret < 0 || static_cast<size_t>(ret) >= sizeof(abs_path)) { ++ return ""; ++ } ++ ++ for (int i { ret }; i >= 0; --i) { ++ if (abs_path[i] == '/') { ++ abs_path[i + 1] = '\0'; ++ break; ++ } ++ } ++ ++ return static_cast<std::string>(abs_path) + "../../../../../../../test/image/oci/storage/layers/devmapper"; ++} ++ ++static bool invokeUtilExecCmd(exec_func_t cb_func, void *args, const char *stdin_msg, char **stdout_msg, char **stderr_msg) ++{ ++ if (cb_func == nullptr || args == nullptr || stdout_msg == nullptr || stderr_msg == nullptr) { ++ return false; ++ } ++ ++ char **tmp_args = static_cast<char **>(args); ++ ++ if (util_array_len((const char **)tmp_args) < 1) { ++ return false; ++ } ++ ++ if (strcmp(tmp_args[0], "blkid") == 0) { ++ *stdout_msg = util_strdup_s("4fa22307-0c88-4fa4-8f16-a9459e9cbc4a"); ++ } ++ return true; ++} ++ ++static struct dm_task *invokeDMTaskCreate(int type) { ++ return static_cast<struct dm_task *>(util_common_calloc_s(sizeof(0))); ++} ++ ++static void invokeDMTaskDestroy(struct dm_task *task) { ++ free(task); ++ return; ++} ++ ++static int invokeDMTaskGetDriverVersion(struct dm_task *task, char *version, size_t size) { ++ if (task == nullptr || version == nullptr || strncpy(version, "4.27.0", size) == NULL) { ++ return 0; ++ } ++ ++ return 1; ++} ++ ++static int invokeDMTaskGetInfo(struct dm_task *task, struct dm_info *dmi) { ++ if (task == nullptr || dmi == nullptr) { ++ return 0; ++ } ++ ++ dmi->exists = 1; ++ return 1; ++} ++ ++static void *invokeDMGetNextTarget(struct dm_task *task, void *next, uint64_t *start, uint64_t *length, ++ char **target_type, char **params) { ++ static char type[] = "thin-pool"; ++ static char par[] = "0 0/1024 0/1024"; ++ if (target_type) { ++ *target_type = type; ++ } ++ if (params) { ++ *params = par; ++ } ++ return nullptr; ++} ++ ++class DriverDevmapperUnitTest : public testing::Test { ++protected: ++ void SetUp() override ++ { ++ MockLibdevmapper_SetMock(&m_libdevmapper_mock); ++ std::string isulad_dir { "/tmp/isulad/" }; ++ mkdir(isulad_dir.c_str(), 0755); ++ std::string root_dir = isulad_dir + "data"; ++ std::string run_dir = isulad_dir + "data/run"; ++ std::string data_dir = GetDirectory() + "/data"; ++ std::string driver_home = root_dir + "/devicemapper"; ++ ++ ASSERT_STRNE(util_clean_path(data_dir.c_str(), data_path, sizeof(data_path)), nullptr); ++ std::string cp_command = "cp -r " + std::string(data_path) + " " + isulad_dir; ++ ASSERT_EQ(system(cp_command.c_str()), 0); ++ ++ char **driver_opts = static_cast<char **>(util_common_calloc_s(3 * sizeof(char *))); ++ driver_opts[0] = strdup("dm.thinpooldev=/dev/mapper/isulad0-thinpool"); ++ driver_opts[1] = strdup("dm.fs=ext4"); ++ driver_opts[2] = strdup("dm.min_free_space=10%"); ++ int driver_opts_len = 3; ++ ++ ASSERT_EQ(devmapper_init(&driver, nullptr, (const char **)driver_opts, driver_opts_len), -1); ++ ++ EXPECT_CALL(m_libdevmapper_mock, DMTaskCreate(_)).WillRepeatedly(Invoke(invokeDMTaskCreate)); ++ EXPECT_CALL(m_libdevmapper_mock, DMTaskSetMessage(_, _)).WillRepeatedly(Return(1)); ++ EXPECT_CALL(m_libdevmapper_mock, DMTaskSetSector(_, _)).WillRepeatedly(Return(1)); ++ EXPECT_CALL(m_libdevmapper_mock, DMTaskSetAddNode(_, _)).WillRepeatedly(Return(1)); ++ EXPECT_CALL(m_libdevmapper_mock, DMTaskAddTarget(_, _, _, _, _)).WillRepeatedly(Return(1)); ++ EXPECT_CALL(m_libdevmapper_mock, DMTaskSetName(_, _)).WillRepeatedly(Return(1)); ++ EXPECT_CALL(m_libdevmapper_mock, DMTaskRun(_)).WillRepeatedly(Return(1)); ++ EXPECT_CALL(m_libdevmapper_mock, DMTaskDestroy(_)).WillRepeatedly(Invoke(invokeDMTaskDestroy)); ++ EXPECT_CALL(m_libdevmapper_mock, DMTaskGetInfo(_, _)).WillRepeatedly(Invoke(invokeDMTaskGetInfo)); ++ EXPECT_CALL(m_libdevmapper_mock, DMGetNextTarget(_, _, _, _, _, _)).WillRepeatedly(Invoke(invokeDMGetNextTarget)); ++ EXPECT_CALL(m_libdevmapper_mock, DMTaskSetCookie(_, _, _)).WillRepeatedly(Return(1)); ++ EXPECT_CALL(m_libdevmapper_mock, DMUdevWait(_)).WillRepeatedly(Return(1)); ++ EXPECT_CALL(m_libdevmapper_mock, DMUdevComplete(_)).WillRepeatedly(Return(1)); ++ EXPECT_CALL(m_libdevmapper_mock, DMTaskDeferredRemove(_)).WillRepeatedly(Return(1)); ++ ++ ++ char *names = static_cast<char *>(util_common_calloc_s(sizeof(struct dm_names) + strlen("isulad0-pool") + 1)); ++ struct dm_names *dname = (struct dm_names *)names; ++ dname->dev = 1; ++ dname->next = 0; ++ strcpy(names + sizeof(struct dm_names), "isulad0-pool"); ++ EXPECT_CALL(m_libdevmapper_mock, DMTaskGetNames(_)).WillOnce(Return(dname)); ++ EXPECT_CALL(m_libdevmapper_mock, DMSetDevDir(_)).WillOnce(Return(1)); ++ EXPECT_CALL(m_libdevmapper_mock, DMTaskGetDriverVersion(_, _, _)).WillOnce(Invoke(invokeDMTaskGetDriverVersion)); ++ EXPECT_CALL(m_libdevmapper_mock, DMUdevGetSyncSupport()).WillOnce(Return(1)); ++ ++ MOCK_SET_V(util_exec_cmd, invokeUtilExecCmd); ++ ++ ASSERT_EQ(devmapper_init(&driver, driver_home.c_str(), (const char **)driver_opts, driver_opts_len), 0); ++ MOCK_CLEAR(util_exec_cmd); ++ ++ util_free_array_by_len(driver_opts, driver_opts_len); ++ free(names); ++ } ++ ++ void TearDown() override ++ { ++ MockLibdevmapper_SetMock(nullptr); ++ std::string rm_command = "rm -rf /tmp/isulad/"; ++ ASSERT_EQ(system(rm_command.c_str()), 0); ++ } ++ ++ NiceMock<MockLibdevmapper> m_libdevmapper_mock; ++ char data_path[PATH_MAX] = { 0x00 }; ++ graphdriver driver = {.ops = nullptr, .name = "devicemapper", }; ++}; ++ ++TEST_F(DriverDevmapperUnitTest, test_devmapper_layer_exists) ++{ ++ std::string id { "3d24ee258efc3bfe4066a1a9fb83febf6dc0b1548dfe896161533668281c9f4f" }; ++ std::string incorrectId { "eb29745b8228e1e97c01b1d5c2554a319c00a94d8dd5746a3904222ad65a13f8" }; ++ ASSERT_TRUE(devmapper_layer_exist(id.c_str(), &driver)); ++ ASSERT_FALSE(devmapper_layer_exist(incorrectId.c_str(), &driver)); ++} ++ ++TEST_F(DriverDevmapperUnitTest, test_devmapper_create_rw) ++{ ++ std::string id { "eb29745b8228e1e97c01b1d5c2554a319c00a94d8dd5746a3904222ad65a13f8" }; ++ struct driver_create_opts *create_opts; ++ ++ create_opts = (struct driver_create_opts *)util_common_calloc_s(sizeof(struct driver_create_opts)); ++ ASSERT_NE(create_opts, nullptr); ++ ++ create_opts->storage_opt = static_cast<json_map_string_string *>(util_common_calloc_s(sizeof(json_map_string_string))); ++ ASSERT_NE(create_opts->storage_opt, nullptr); ++ create_opts->storage_opt->keys = static_cast<char **>(util_common_calloc_s(sizeof(char *))); ++ create_opts->storage_opt->values = static_cast<char **>(util_common_calloc_s(sizeof(char *))); ++ create_opts->storage_opt->keys[0] = strdup("size"); ++ create_opts->storage_opt->values[0] = strdup("10G"); ++ create_opts->storage_opt->len = 1; ++ ++ ASSERT_EQ(devmapper_create_rw(id.c_str(), nullptr, &driver, create_opts), 0); ++ ASSERT_TRUE(devmapper_layer_exist(id.c_str(), &driver)); ++} ++ ++TEST_F(DriverDevmapperUnitTest, test_devmapper_mount_layer) ++{ ++ std::string id { "3d24ee258efc3bfe4066a1a9fb83febf6dc0b1548dfe896161533668281c9f4f" }; ++ std::string merged_dir = "/tmp/isulad/data/devicemapper/mnt/" + id + "/rootfs"; ++ struct driver_mount_opts *mount_opts = nullptr; ++ char* mount_dir = nullptr; ++ ++ MOCK_SET(util_mount, 0); ++ mount_dir = devmapper_mount_layer(id.c_str(), &driver, mount_opts); ++ ASSERT_STREQ(mount_dir, merged_dir.c_str()); ++ MOCK_CLEAR(util_mount); ++ ++ MOCK_SET(umount2, 0); ++ ASSERT_EQ(devmapper_umount_layer(id.c_str(), &driver), 0); ++ MOCK_CLEAR(umount2); ++ free(mount_dir); ++ mount_dir = nullptr; ++ ++ mount_opts = static_cast<struct driver_mount_opts *>(util_common_calloc_s(sizeof(struct driver_mount_opts))); ++ ASSERT_NE(mount_opts, nullptr); ++ mount_opts->options = static_cast<char **>(util_common_calloc_s(1 * sizeof(char *))); ++ mount_opts->options[0] = strdup("ro"); ++ mount_opts->options_len = 1; ++ ++ MOCK_SET(util_mount, 0); ++ mount_dir = devmapper_mount_layer(id.c_str(), &driver, mount_opts); ++ ASSERT_STREQ(mount_dir, merged_dir.c_str()); ++ MOCK_CLEAR(util_mount); ++ ++ MOCK_SET(umount2, 0); ++ ASSERT_EQ(devmapper_umount_layer(id.c_str(), &driver), 0); ++ MOCK_CLEAR(umount2); ++ free(mount_opts->mount_label); ++ util_free_array_by_len(mount_opts->options, mount_opts->options_len); ++ free(mount_opts); ++ free(mount_dir); ++} ++ ++TEST_F(DriverDevmapperUnitTest, test_devmapper_get_layer_metadata) ++{ ++ std::string id { "3d24ee258efc3bfe4066a1a9fb83febf6dc0b1548dfe896161533668281c9f4f" }; ++ json_map_string_string *map_info = static_cast<json_map_string_string *>(util_common_calloc_s(sizeof(json_map_string_string))); ++ ++ ASSERT_EQ(devmapper_get_layer_metadata(id.c_str(), &driver, map_info), 0); ++ ASSERT_EQ(map_info->len, 4); ++ ASSERT_STREQ(map_info->keys[0], "DeviceId"); ++ ASSERT_STREQ(map_info->values[0], "4"); ++ ASSERT_STREQ(map_info->keys[1], "DeviceSize"); ++ ASSERT_STREQ(map_info->values[1], "10737418240"); ++ ASSERT_STREQ(map_info->keys[2], "DeviceName"); ++ ASSERT_STREQ(map_info->keys[3], "MergedDir"); ++ ASSERT_STREQ(map_info->values[3], "/tmp/isulad/data/devicemapper/mnt/3d24ee258efc3bfe4066a1a9fb83febf6dc0b1548dfe896161533668281c9f4f/rootfs"); ++ ++ free_json_map_string_string(map_info); ++} ++ ++TEST_F(DriverDevmapperUnitTest, test_devmapper_get_driver_status) ++{ ++ struct graphdriver_status *status = static_cast<struct graphdriver_status *>(util_common_calloc_s(sizeof(struct graphdriver_status))); ++ ++ EXPECT_CALL(m_libdevmapper_mock, DMUdevGetSyncSupport()).WillOnce(Return(1)); ++ ++ ASSERT_EQ(devmapper_get_driver_status(&driver, status), 0); ++ ASSERT_STREQ(status->driver_name, "devicemapper"); ++ free(status->driver_name); ++ free(status->backing_fs); ++ free(status->status); ++ free(status); ++} +diff --git a/test/mocks/libdevmapper_mock.cc b/test/mocks/libdevmapper_mock.cc +new file mode 100644 +index 00000000..7d6c8024 +--- /dev/null ++++ b/test/mocks/libdevmapper_mock.cc +@@ -0,0 +1,191 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2023-11-22 ++ * Description: provide lib device mapper mock ++ ******************************************************************************/ ++ ++#include "libdevmapper_mock.h" ++ ++namespace { ++MockLibdevmapper *g_libdevmapper_mock = nullptr; ++} ++ ++void MockLibdevmapper_SetMock(MockLibdevmapper* mock) ++{ ++ g_libdevmapper_mock = mock; ++} ++ ++struct dm_task *dm_task_create(int type) ++{ ++ if (g_libdevmapper_mock != nullptr) { ++ return g_libdevmapper_mock->DMTaskCreate(type); ++ } ++ return nullptr; ++} ++ ++int dm_task_set_message(struct dm_task *dmt, const char *msg) ++{ ++ if (g_libdevmapper_mock != nullptr) { ++ return g_libdevmapper_mock->DMTaskSetMessage(dmt, msg); ++ } ++ return 0; ++} ++ ++int dm_task_set_sector(struct dm_task *dmt, uint64_t sector) ++{ ++ if (g_libdevmapper_mock != nullptr) { ++ return g_libdevmapper_mock->DMTaskSetSector(dmt, sector); ++ } ++ return 0; ++} ++ ++int dm_task_set_add_node(struct dm_task *dmt, dm_add_node_t add_node) ++{ ++ if (g_libdevmapper_mock != nullptr) { ++ return g_libdevmapper_mock->DMTaskSetAddNode(dmt, add_node); ++ } ++ return 0; ++} ++ ++int dm_task_add_target(struct dm_task *dmt, uint64_t start, uint64_t size, const char *ttype, const char *params) ++{ ++ if (g_libdevmapper_mock != nullptr) { ++ return g_libdevmapper_mock->DMTaskAddTarget(dmt, start, size, ttype, params); ++ } ++ return 0; ++} ++ ++int dm_set_dev_dir(const char *dir) ++{ ++ if (g_libdevmapper_mock != nullptr) { ++ return g_libdevmapper_mock->DMSetDevDir(dir); ++ } ++ return 0; ++} ++ ++int dm_task_set_name(struct dm_task *dmt, const char *name) ++{ ++ if (g_libdevmapper_mock != nullptr) { ++ return g_libdevmapper_mock->DMTaskSetName(dmt, name); ++ } ++ return 0; ++} ++ ++int dm_task_run(struct dm_task *dmt) ++{ ++ if (g_libdevmapper_mock != nullptr) { ++ return g_libdevmapper_mock->DMTaskRun(dmt); ++ } ++ return 0; ++} ++ ++int dm_task_get_driver_version(struct dm_task *dmt, char *version, size_t size) ++{ ++ if (g_libdevmapper_mock != nullptr) { ++ return g_libdevmapper_mock->DMTaskGetDriverVersion(dmt, version, size); ++ } ++ return 0; ++} ++ ++void dm_task_destroy(struct dm_task *dmt) ++{ ++ if (g_libdevmapper_mock != nullptr) { ++ g_libdevmapper_mock->DMTaskDestroy(dmt); ++ } ++} ++ ++int dm_get_library_version(char *version, size_t size) ++{ ++ if (g_libdevmapper_mock != nullptr) { ++ return g_libdevmapper_mock->DMGetLibraryVersion(version, size); ++ } ++ return 0; ++} ++ ++int dm_task_get_info(struct dm_task *dmt, struct dm_info *info) ++{ ++ if (g_libdevmapper_mock != nullptr) { ++ return g_libdevmapper_mock->DMTaskGetInfo(dmt, info); ++ } ++ return 0; ++} ++ ++void *dm_get_next_target(struct dm_task *dmt, void *next, uint64_t *start, uint64_t *length, ++ char **target_type, char **params) ++{ ++ if (g_libdevmapper_mock != nullptr) { ++ return g_libdevmapper_mock->DMGetNextTarget(dmt, next, start, length, target_type, params); ++ } ++ return nullptr; ++} ++ ++int dm_task_set_cookie(struct dm_task *dmt, uint32_t *cookie, uint16_t flags) ++{ ++ if (g_libdevmapper_mock != nullptr) { ++ return g_libdevmapper_mock->DMTaskSetCookie(dmt, cookie, flags); ++ } ++ return 0; ++} ++ ++int dm_udev_wait(uint32_t cookie) ++{ ++ if (g_libdevmapper_mock != nullptr) { ++ return g_libdevmapper_mock->DMUdevWait(cookie); ++ } ++ return 0; ++} ++ ++int dm_udev_complete(uint32_t cookie) ++{ ++ if (g_libdevmapper_mock != nullptr) { ++ return g_libdevmapper_mock->DMUdevComplete(cookie); ++ } ++ return 0; ++} ++ ++int dm_task_deferred_remove(struct dm_task *dmt) ++{ ++ if (g_libdevmapper_mock != nullptr) { ++ return g_libdevmapper_mock->DMTaskDeferredRemove(dmt); ++ } ++ return 0; ++} ++ ++struct dm_names *dm_task_get_names(struct dm_task *dmt) ++{ ++ if (g_libdevmapper_mock != nullptr) { ++ return g_libdevmapper_mock->DMTaskGetNames(dmt); ++ } ++ return nullptr; ++} ++ ++int dm_udev_get_sync_support(void) ++{ ++ if (g_libdevmapper_mock != nullptr) { ++ return g_libdevmapper_mock->DMUdevGetSyncSupport(); ++ } ++ return 0; ++} ++ ++void dm_udev_set_sync_support(int sync_with_udev) ++{ ++ if (g_libdevmapper_mock != nullptr) { ++ g_libdevmapper_mock->DMUdevSetSyncSupport(sync_with_udev); ++ } ++} ++ ++void dm_log_with_errno_init(void log_cb(int level, const char *file, int line, int dm_errno_or_class, const char *f, ...)) ++{ ++ if (g_libdevmapper_mock != nullptr) { ++ g_libdevmapper_mock->DMLogWithErrnoInit(log_cb); ++ } ++} +diff --git a/test/mocks/libdevmapper_mock.h b/test/mocks/libdevmapper_mock.h +new file mode 100644 +index 00000000..53c5ad4b +--- /dev/null ++++ b/test/mocks/libdevmapper_mock.h +@@ -0,0 +1,52 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2023-11-22 ++ * Description: provide lib device mapper mock ++ ******************************************************************************/ ++ ++#ifndef _ISULAD_TEST_MOCKS_DEVMAPPER_MOCK_H ++#define _ISULAD_TEST_MOCKS_DEVMAPPER_MOCK_H ++ ++#include <gmock/gmock.h> ++ ++#include <libdevmapper.h> ++ ++class MockLibdevmapper { ++public: ++ virtual ~MockLibdevmapper() = default; ++ MOCK_METHOD1(DMTaskCreate, struct dm_task*(int type)); ++ MOCK_METHOD2(DMTaskSetMessage, int(struct dm_task *dmt, const char *msg)); ++ MOCK_METHOD2(DMTaskSetSector, int(struct dm_task *dmt, uint64_t sector)); ++ MOCK_METHOD2(DMTaskSetAddNode, int(struct dm_task *dmt, dm_add_node_t add_node)); ++ MOCK_METHOD5(DMTaskAddTarget, int(struct dm_task *dmt, uint64_t start, uint64_t size, const char *ttype, const char *params)); ++ MOCK_METHOD1(DMSetDevDir, int(const char *dir)); ++ MOCK_METHOD2(DMTaskSetName, int(struct dm_task *dmt, const char *name)); ++ MOCK_METHOD1(DMTaskRun, int(struct dm_task *dmt)); ++ MOCK_METHOD3(DMTaskGetDriverVersion, int(struct dm_task *dmt, char *version, size_t size)); ++ MOCK_METHOD1(DMTaskDestroy, void(struct dm_task *dmt)); ++ MOCK_METHOD2(DMGetLibraryVersion, int(char *version, size_t size)); ++ MOCK_METHOD2(DMTaskGetInfo, int(struct dm_task *dmt, struct dm_info *info)); ++ MOCK_METHOD6(DMGetNextTarget, void*(struct dm_task *dmt, void *next, uint64_t *start, uint64_t *length, ++ char **target_type, char **params)); ++ MOCK_METHOD3(DMTaskSetCookie, int(struct dm_task *dmt, uint32_t *cookie, uint16_t flags)); ++ MOCK_METHOD1(DMUdevWait, int(uint32_t cookie)); ++ MOCK_METHOD1(DMUdevComplete, int(uint32_t cookie)); ++ MOCK_METHOD1(DMTaskDeferredRemove, int(struct dm_task *dmt)); ++ MOCK_METHOD1(DMTaskGetNames, struct dm_names *(struct dm_task *dmt)); ++ MOCK_METHOD0(DMUdevGetSyncSupport, int(void)); ++ MOCK_METHOD1(DMUdevSetSyncSupport, void(int sync_with_udev)); ++ MOCK_METHOD1(DMLogWithErrnoInit, void(void log_cb(int level, const char *file, int line, int dm_errno_or_class, const char *f, ...))); ++}; ++ ++void MockLibdevmapper_SetMock(MockLibdevmapper* mock); ++ ++#endif +-- +2.42.0 + diff --git a/0033-2275-bugfix-for-rt_lcr_rebuild_config.patch b/0033-2275-bugfix-for-rt_lcr_rebuild_config.patch new file mode 100644 index 0000000..ec42563 --- /dev/null +++ b/0033-2275-bugfix-for-rt_lcr_rebuild_config.patch @@ -0,0 +1,46 @@ +From fa7356538c7f747a81aa3d0a511a662ee4345afe Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Fri, 24 Nov 2023 08:33:45 +0000 +Subject: [PATCH 33/64] !2275 bugfix for rt_lcr_rebuild_config * bugfix for + rt_lcr_rebuild_config + +--- + src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c | 8 +++++--- + 1 file changed, 5 insertions(+), 3 deletions(-) + +diff --git a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c +index 8f7211d7..44ecab5a 100644 +--- a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c ++++ b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c +@@ -777,6 +777,7 @@ int rt_lcr_rebuild_config(const char *name, const char *runtime, const rt_rebuil + { + int ret = -1; + int nret = 0; ++ bool rebuild_success = false; + char config_file[PATH_MAX] = { 0 }; + char bak_config_file[PATH_MAX] = { 0 }; + char oci_config_file[PATH_MAX] = { 0 }; +@@ -825,8 +826,8 @@ int rt_lcr_rebuild_config(const char *name, const char *runtime, const rt_rebuil + goto out; + } + +- nret = engine_ops->engine_create_op(name, params->rootpath, (void *)oci_spec); +- if (nret != 0) { ++ rebuild_success = engine_ops->engine_create_op(name, params->rootpath, (void *)oci_spec); ++ if (!rebuild_success) { + // delete the invalid config file to prevent rename failed + if (util_fileself_exists(config_file) && util_path_remove(config_file) != 0) { + WARN("Failed to remove bak_config_file for container %s", name); +@@ -835,7 +836,8 @@ int rt_lcr_rebuild_config(const char *name, const char *runtime, const rt_rebuil + WARN("Failed to rename backup old config to config for container %s", name); + } + } +- ret = nret != 0 ? -1 : 0; ++ ret = rebuild_success ? 0 : -1; ++ + out: + if (engine_ops != NULL && engine_ops->engine_clear_errmsg_op != NULL) { + engine_ops->engine_clear_errmsg_op(); +-- +2.42.0 + diff --git a/0034-2277-remove-shim-v2-format-error-log.patch b/0034-2277-remove-shim-v2-format-error-log.patch new file mode 100644 index 0000000..1573b9e --- /dev/null +++ b/0034-2277-remove-shim-v2-format-error-log.patch @@ -0,0 +1,25 @@ +From fe03c12676b8a48a2aede2d177f2cbcbdd68f930 Mon Sep 17 00:00:00 2001 +From: jake <jikai11@huawei.com> +Date: Sat, 25 Nov 2023 03:34:01 +0000 +Subject: [PATCH 34/64] !2277 remove shim v2 format error log * remove shim v2 + format error log + +--- + src/daemon/modules/runtime/shim/shim_rt_ops.c | 1 - + 1 file changed, 1 deletion(-) + +diff --git a/src/daemon/modules/runtime/shim/shim_rt_ops.c b/src/daemon/modules/runtime/shim/shim_rt_ops.c +index 1bc9dc54..5066f804 100644 +--- a/src/daemon/modules/runtime/shim/shim_rt_ops.c ++++ b/src/daemon/modules/runtime/shim/shim_rt_ops.c +@@ -115,7 +115,6 @@ bool is_valid_v2_runtime(const char* name) + + parts_len = util_array_len((const char **)parts); + if (!(parts_len == 4 && strcmp(parts[0], "io") == 0 && strcmp(parts[1], "containerd") == 0)) { +- ERROR("ShimV2 runtime format is wrong"); + util_free_array(parts); + return false; + } +-- +2.42.0 + diff --git a/0035-2276-bugfix-for-integration_check.sh.patch b/0035-2276-bugfix-for-integration_check.sh.patch new file mode 100644 index 0000000..c870e52 --- /dev/null +++ b/0035-2276-bugfix-for-integration_check.sh.patch @@ -0,0 +1,26 @@ +From a2c565705f80f787e50ffc15db38ba367f517eb2 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Sat, 25 Nov 2023 03:34:50 +0000 +Subject: [PATCH 35/64] !2276 bugfix for integration_check.sh * bugfix for + integration_check.sh + +--- + CI/test_cases/image_cases/integration_check.sh | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/CI/test_cases/image_cases/integration_check.sh b/CI/test_cases/image_cases/integration_check.sh +index e43369e3..6ec3ab52 100755 +--- a/CI/test_cases/image_cases/integration_check.sh ++++ b/CI/test_cases/image_cases/integration_check.sh +@@ -65,7 +65,7 @@ function test_image_info() + echo "xxx:11" >> ${change_file} + + sed -i 's#image-layer-check": false#image-layer-check": true#g' /etc/isulad/daemon.json +- pkill -9 isulad ++ kill -9 $(pidof isulad) + start_isulad_with_valgrind + + isula ps -a | grep ${cid} +-- +2.42.0 + diff --git a/0036-modify-create_network.sh-for-default-runtime-changed.patch b/0036-modify-create_network.sh-for-default-runtime-changed.patch new file mode 100644 index 0000000..40e9feb --- /dev/null +++ b/0036-modify-create_network.sh-for-default-runtime-changed.patch @@ -0,0 +1,40 @@ +From e422c6cf725240dea80e1c51ba21cae8ee6641c6 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Sat, 25 Nov 2023 18:21:56 +0800 +Subject: [PATCH 36/64] modify create_network.sh for default runtime changed + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + CI/test_cases/container_cases/create_network.sh | 2 +- + CI/test_cases/helpers.sh | 2 ++ + 2 files changed, 3 insertions(+), 1 deletion(-) + +diff --git a/CI/test_cases/container_cases/create_network.sh b/CI/test_cases/container_cases/create_network.sh +index 470bda70..5bafbc60 100755 +--- a/CI/test_cases/container_cases/create_network.sh ++++ b/CI/test_cases/container_cases/create_network.sh +@@ -37,7 +37,7 @@ function test_network_param() + + msg_info "${test} starting..." + +- root="`isula info | grep 'iSulad Root Dir' | awk -F ':' '{print $2}'`/engines/lcr" ++ root="`isula info | grep 'iSulad Root Dir' | awk -F ':' '{print $2}'`/engines/$DEFAULT_RUNTIME" + + isula pull ${image} + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${image}" && return ${FAILURE} +diff --git a/CI/test_cases/helpers.sh b/CI/test_cases/helpers.sh +index f3eeb54d..c5eba8a2 100755 +--- a/CI/test_cases/helpers.sh ++++ b/CI/test_cases/helpers.sh +@@ -29,6 +29,8 @@ ISULAD_RUN_ROOT_PATH="/var/run/isulad" + + RUNTIME_LIST=(lcr runc) + ++DEFAULT_RUNTIME=runc ++ + testcase_data="/tmp/testcases_data" + + enable_native_network=0 +-- +2.42.0 + diff --git a/0037-modify-the-container-runtime-when-running-embedded.s.patch b/0037-modify-the-container-runtime-when-running-embedded.s.patch new file mode 100644 index 0000000..7ba3583 --- /dev/null +++ b/0037-modify-the-container-runtime-when-running-embedded.s.patch @@ -0,0 +1,141 @@ +From 8e4b6eceeb117fc90b5b638329f8888e43d3f442 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Mon, 27 Nov 2023 17:21:15 +0800 +Subject: [PATCH 37/64] modify the container runtime when running embedded.sh + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + CI/test_cases/image_cases/embedded.sh | 30 +++++++++++++-------------- + 1 file changed, 15 insertions(+), 15 deletions(-) + +diff --git a/CI/test_cases/image_cases/embedded.sh b/CI/test_cases/image_cases/embedded.sh +index cdc75e50..a1d4c37a 100755 +--- a/CI/test_cases/image_cases/embedded.sh ++++ b/CI/test_cases/image_cases/embedded.sh +@@ -81,14 +81,14 @@ function test_run_image() + { + local ret=0 + +- isula run -t -n embedded_test1 nonexistentname1:v1 /bin/sh ++ isula run --runtime lcr -t -n embedded_test1 nonexistentname1:v1 /bin/sh + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - run nonexistent image should failed" && ((ret++)) + + isula load -i "$embedded_manifest" -t embedded + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - load embedded image failed" && ((ret++)) + + # run container based on embedded image +- isula run --name embedded_test1 test:v1 ls /home/home/home ++ isula run --runtime lcr --name embedded_test1 test:v1 ls /home/home/home + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - run embedded image failed" && ((ret++)) + + # delete container based on embedded image +@@ -96,7 +96,7 @@ function test_run_image() + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - delete container based on embedded image failed" && ((ret++)) + + # test image's env +- isula run --name embedded_test1 test:v1 /bin/sh -c "echo \$c | grep \"d e\"" ++ isula run --runtime lcr --name embedded_test1 test:v1 /bin/sh -c "echo \$c | grep \"d e\"" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - test image's env failed" && ((ret++)) + + # delete container based on embedded image +@@ -119,7 +119,7 @@ function test_mount() + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - load embedded imagefailed" && ((ret++)) + + # run --mount +- isula run --mount type=bind,src="$embedded_basedir",dst=/usr,ro=true,bind-propagation=rprivate --name embedded_test2 test:v1 true ++ isula run --runtime lcr --mount type=bind,src="$embedded_basedir",dst=/usr,ro=true,bind-propagation=rprivate --name embedded_test2 test:v1 true + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - run --mount failed" && ((ret++)) + + testcontainer embedded_test2 exited +@@ -127,25 +127,25 @@ function test_mount() + isula rm embedded_test2 + + # test invalid mode +- isula run --mount type=bind,src="$embedded_basedir",dst=/usr,ro=invalid --name embedded_test2 test:v1 true ++ isula run --runtime lcr --mount type=bind,src="$embedded_basedir",dst=/usr,ro=invalid --name embedded_test2 test:v1 true + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - invalid mode should failed" && ((ret++)) + + isula rm embedded_test2 + + # test invalid bind propagation mode +- isula run --mount type=bind,src="$embedded_basedir",dst=/usr,bind-propagation=invalid --name embedded_test2 test:v1 true ++ isula run --runtime lcr --mount type=bind,src="$embedded_basedir",dst=/usr,bind-propagation=invalid --name embedded_test2 test:v1 true + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - invalid bind propagation mode should failed" && ((ret++)) + + isula rm embedded_test2 + + # test source not exist +- isula run --mount type=bind,src=abcdefg/notexist,dst=/usr --name embedded_test2 test:v1 true ++ isula run --runtime lcr --mount type=bind,src=abcdefg/notexist,dst=/usr --name embedded_test2 test:v1 true + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - invalid source not exist should failed" && ((ret++)) + + isula rm embedded_test2 + + # test source not a regular file +- isula run --mount type=squashfs,src=/tmp,dst=/usr --name embedded_test2 test:v1 true ++ isula run --runtime lcr --mount type=squashfs,src=/tmp,dst=/usr --name embedded_test2 test:v1 true + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - source not a regular file should failed" && ((ret++)) + + isula rm embedded_test2 +@@ -153,7 +153,7 @@ function test_mount() + # test path //tmp/test + mkdir -p /tmp/test_mount + mkdir -p /tmp/test_mount1/test +- isula run -v /tmp/test_mount:/tmp --mount type=bind,src=/tmp/test_mount1,dst=//tmp/test_mount1,ro=true,bind-propagation=rprivate --name embedded_test2 test:v1 ls /tmp/test_mount1/test ++ isula run --runtime lcr -v /tmp/test_mount:/tmp --mount type=bind,src=/tmp/test_mount1,dst=//tmp/test_mount1,ro=true,bind-propagation=rprivate --name embedded_test2 test:v1 ls /tmp/test_mount1/test + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - test path //tmp/test failed" && ((ret++)) + + isula rm embedded_test2 +@@ -186,7 +186,7 @@ function test_query_image() + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - inspect nonexist item should failed" && ((ret++)) + + # test inspect container, it should conatainer image info +- isula run --name embedded_inspect test:v1 ls /home/home/home ++ isula run --runtime lcr --name embedded_inspect test:v1 ls /home/home/home + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - run container for inspect failed" && ((ret++)) + + isula inspect -f '{{json .Image}}' embedded_inspect +@@ -437,19 +437,19 @@ function test_entrypoint() + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - load embedded image failed" && ((ret++)) + + # test image's entrypoint +- isula run --name embedded_entrypoint1 test:v1 ++ isula run --runtime lcr --name embedded_entrypoint1 test:v1 + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - test image's entrypoint failed" && ((ret++)) + + isula rm embedded_entrypoint1 + + # test image's entrypoint with cmds +- isula run --name embedded_entrypoint1 test:v1 /bin ++ isula run --runtime lcr --name embedded_entrypoint1 test:v1 /bin + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - test image's entrypoint with cmds failed" && ((ret++)) + + isula rm embedded_entrypoint1 + + # test image's entrypoint override image's entrypoint +- isula run --entrypoint=/bin/ls --name embedded_entrypoint1 test:v1 /bin ++ isula run --runtime lcr --entrypoint=/bin/ls --name embedded_entrypoint1 test:v1 /bin + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - test image's entrypoint override image's entrypoint failed" && ((ret++)) + + isula rm embedded_entrypoint1 +@@ -464,7 +464,7 @@ function test_entrypoint() + isula load -i "$embedded_manifest_invalid" -t embedded + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - test entrypoint with variable failed" && ((ret++)) + +- isula run -e env_id=me --name embedded_entrypoint1 test:v1 ++ isula run --runtime lcr -e env_id=me --name embedded_entrypoint1 test:v1 + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - test run embedded image with env failed" && ((ret++)) + + isula rm embedded_entrypoint1 +@@ -519,7 +519,7 @@ function test_symbolic() + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - load embedded image failed" && ((ret++)) + + # run container based on embedded image +- isula run --name embedded_test_symbolic test:v1 ls /home/home/home ++ isula run --runtime lcr --name embedded_test_symbolic test:v1 ls /home/home/home + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - run container based on embedded image failed" && ((ret++)) + + isula rm embedded_test_symbolic +-- +2.42.0 + diff --git a/0038-save-sandbox-to-disk-after-network-ready.patch b/0038-save-sandbox-to-disk-after-network-ready.patch new file mode 100644 index 0000000..b77ff29 --- /dev/null +++ b/0038-save-sandbox-to-disk-after-network-ready.patch @@ -0,0 +1,68 @@ +From e33b7915d9ef5092252bc3ce5d93fbde74d73990 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Mon, 27 Nov 2023 15:09:39 +0800 +Subject: [PATCH 38/64] save sandbox to disk after network ready + +Signed-off-by: jikai <jikai11@huawei.com> +--- + .../cri/v1/v1_cri_pod_sandbox_manager_service.cc | 13 ++++++++++--- + src/daemon/sandbox/sandbox.cc | 6 ------ + 2 files changed, 10 insertions(+), 9 deletions(-) + +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +index 0f6b8508..a0c45111 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +@@ -358,14 +358,21 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1::PodSandboxConfig + goto cleanup_sandbox; + } + +- // Step 8: Call sandbox create. ++ // Step 8: Save sandbox to disk ++ sandbox->Save(error); ++ if (error.NotEmpty()) { ++ ERROR("Failed to save sandbox, %s", sandboxName.c_str()); ++ goto cleanup_network; ++ } ++ ++ // Step 9: Call sandbox create. + sandbox->Create(error); + if (error.NotEmpty()) { + ERROR("Failed to create sandbox: %s", sandboxName.c_str()); + goto cleanup_network; + } + +- // Step 9: Save network settings json to disk ++ // Step 10: Save network settings json to disk + // Update network settings before start sandbox since sandbox container will use the sandbox key + if (namespace_is_cni(networkMode.c_str())) { + Errors tmpErr; +@@ -376,7 +383,7 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1::PodSandboxConfig + } + } + +- // Step 10: Call sandbox start. ++ // Step 11: Call sandbox start. + sandbox->Start(error); + if (error.NotEmpty()) { + ERROR("Failed to start sandbox: %s", sandboxName.c_str()); +diff --git a/src/daemon/sandbox/sandbox.cc b/src/daemon/sandbox/sandbox.cc +index b1832265..9fe9fa48 100644 +--- a/src/daemon/sandbox/sandbox.cc ++++ b/src/daemon/sandbox/sandbox.cc +@@ -599,12 +599,6 @@ void Sandbox::PrepareSandboxDirs(Errors &error) + goto out; + } + +- if (!Save(error)) { +- error.Errorf("Failed to save sandbox, %s", m_id.c_str()); +- ERROR("Failed to save sandbox, %s", m_id.c_str()); +- goto out; +- } +- + umask(mask); + return; + +-- +2.42.0 + diff --git a/0039-fix-the-problem-of-abnormal-branches-not-waiting-for.patch b/0039-fix-the-problem-of-abnormal-branches-not-waiting-for.patch new file mode 100644 index 0000000..0f878d4 --- /dev/null +++ b/0039-fix-the-problem-of-abnormal-branches-not-waiting-for.patch @@ -0,0 +1,153 @@ +From b26654a73694c20fcd895b3b93ad5d42a1d5b3fb Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Mon, 27 Nov 2023 14:52:43 +0800 +Subject: [PATCH 39/64] fix the problem of abnormal branches not waiting for + child processes + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/cmd/isulad-shim/common.c | 6 +++--- + src/cmd/isulad-shim/process.c | 14 ++++++++------ + src/daemon/modules/runtime/isula/isula_rt_ops.c | 16 ++++++++++------ + src/daemon/modules/runtime/shim/shim_rt_ops.c | 15 +++++++++------ + 4 files changed, 30 insertions(+), 21 deletions(-) + +diff --git a/src/cmd/isulad-shim/common.c b/src/cmd/isulad-shim/common.c +index 48d266dc..3cc7d2a7 100644 +--- a/src/cmd/isulad-shim/common.c ++++ b/src/cmd/isulad-shim/common.c +@@ -126,12 +126,12 @@ int cmd_combined_output(const char *binary, const char *params[], void *output, + } + *output_len = isula_file_read_nointr(stdio[0], output, BUFSIZ - 1); + +- close(stdio[0]); +- close(exec_fd[0]); +- wait(&status); + ret = SHIM_OK; + + out: ++ close(stdio[0]); ++ close(exec_fd[0]); ++ wait(&status); + if (ret != SHIM_OK) { + kill(pid, 9); + } +diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c +index 187067d2..e8cb9b32 100644 +--- a/src/cmd/isulad-shim/process.c ++++ b/src/cmd/isulad-shim/process.c +@@ -1472,7 +1472,7 @@ static void exec_runtime_process(process_t *p, int exec_fd) + const char *params[MAX_RUNTIME_ARGS] = { 0 }; + get_runtime_cmd(p, log_path, pid_path, process_desc, params); + execvp(p->runtime, (char * const *)params); +- (void)dprintf(exec_fd, "fork/exec error: %s", strerror(errno)); ++ (void)dprintf(exec_fd, "run process: %s error: %s", p->runtime, strerror(errno)); + _exit(EXIT_FAILURE); + } + +@@ -1510,11 +1510,6 @@ int create_process(process_t *p) + close_fd(&p->stdio->resize); + } + nread = isula_file_read_nointr(exec_fd[0], exec_buff, sizeof(exec_buff) - 1); +- if (nread > 0) { +- write_message(ERR_MSG, "runtime error"); +- ret = SHIM_ERR; +- goto out; +- } + + /* block to wait runtime pid exit */ + ret = waitpid(pid, NULL, 0); +@@ -1524,6 +1519,13 @@ int create_process(process_t *p) + goto out; + } + ++ // if an error occurs in exec_runtime_process, jump directly to the out branch after waitpid. ++ if (nread > 0) { ++ write_message(ERR_MSG, "%s", exec_buff); ++ ret = SHIM_ERR; ++ goto out; ++ } ++ + /* save runtime pid */ + data = read_text_file("pid"); + if (data == NULL) { +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 859356e5..5d7ae500 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -906,17 +906,13 @@ realexec: + } + + execvp(SHIM_BINARY, (char * const *)params); +- (void)dprintf(shim_stderr_pipe[1], "exec failed: %s", strerror(errno)); ++ (void)dprintf(shim_stderr_pipe[1], "run process: %s failed: %s", SHIM_BINARY, strerror(errno)); ++ exit(EXIT_FAILURE); + } + + close(shim_stderr_pipe[1]); + close(shim_stdout_pipe[1]); + num = util_read_nointr(shim_stderr_pipe[0], exec_buff, sizeof(exec_buff) - 1); +- if (num > 0) { +- ERROR("Exec failed: %s", exec_buff); +- ret = -1; +- goto out; +- } + + status = util_wait_for_pid_status(pid); + if (status < 0) { +@@ -925,6 +921,14 @@ realexec: + goto out; + } + ++ // if failed to exec, jump directly to the out branch after waitpid. ++ if (num > 0) { ++ ERROR("%s", exec_buff); ++ isulad_set_error_message("%s", exec_buff); ++ ret = -1; ++ goto out; ++ } ++ + *shim_exit_code = status_to_exit_code(status); + if (*shim_exit_code != 0) { + ERROR("Isulad-shim exit error"); +diff --git a/src/daemon/modules/runtime/shim/shim_rt_ops.c b/src/daemon/modules/runtime/shim/shim_rt_ops.c +index 5066f804..81daf224 100644 +--- a/src/daemon/modules/runtime/shim/shim_rt_ops.c ++++ b/src/daemon/modules/runtime/shim/shim_rt_ops.c +@@ -251,17 +251,13 @@ static int shim_bin_v2_create(const char *runtime, const char *id, const char *w + } + + execvp(binary, (char * const *)params); +- (void)dprintf(exec_fd[1], "exec failed: %s", strerror(errno)); ++ (void)dprintf(exec_fd[1], "run process: %s failed: %s", binary, strerror(errno)); + exit(EXIT_FAILURE); + } + + close(exec_fd[1]); + exec_fd[1] = -1; +- if (util_read_nointr(exec_fd[0], exec_buff, sizeof(exec_buff) - 1) > 0) { +- ERROR("exec failed: %s", exec_buff); +- ret = -1; +- goto out; +- } ++ nret = util_read_nointr(exec_fd[0], exec_buff, sizeof(exec_buff) - 1); + close(exec_fd[0]); + exec_fd[0] = -1; + +@@ -272,6 +268,13 @@ static int shim_bin_v2_create(const char *runtime, const char *id, const char *w + goto out; + } + ++ // if failed to exec, jump directly to the out branch after waitpid. ++ if (nret > 0) { ++ ERROR("%s", exec_buff); ++ ret = -1; ++ goto out; ++ } ++ + status = status_to_exit_code(status); + + close(out_fd[1]); +-- +2.42.0 + diff --git a/0040-remove-embedded-image-support-in-readme.patch b/0040-remove-embedded-image-support-in-readme.patch new file mode 100644 index 0000000..9ace537 --- /dev/null +++ b/0040-remove-embedded-image-support-in-readme.patch @@ -0,0 +1,62 @@ +From 6f9661d7e12e22ff4eeb76647cbe862c5fe7e18d Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Mon, 27 Nov 2023 20:50:33 +0800 +Subject: [PATCH 40/64] remove embedded image support in readme + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + README.md | 6 +----- + README_zh.md | 6 +----- + 2 files changed, 2 insertions(+), 10 deletions(-) + +diff --git a/README.md b/README.md +index e7949dee..970b6e72 100644 +--- a/README.md ++++ b/README.md +@@ -32,7 +32,7 @@ kata-runtime start secure containers with lightweight virtual machines. + + ### Image + +-`iSulad` supports multiple image formats, including OCI, external rootfs and embedded image. ++`iSulad` supports multiple image formats, including OCI and external rootfs. + + #### OCI + +@@ -42,10 +42,6 @@ OCI is a docker-compatible image format that supports pulling images and running + + External rootfs allows users to prepare a bootable `root fs` directory, which is mainly used in system container scenarios. + +-#### embedded image +- +-Embedded image is a unique embedded image format of `iSulad`, which occupies low resources and is mainly used in embedded application scenarios. +- + ### Operation Interface + + `iSulad` provides two different interfaces for image and container management operations: CLI and CRI. +diff --git a/README_zh.md b/README_zh.md +index 1c4dff4f..5db28f3a 100755 +--- a/README_zh.md ++++ b/README_zh.md +@@ -32,7 +32,7 @@ kata-runtime是一个安全容器runtime，用于启动安全容器时使用。 + + ### Image + +-`iSulad`支持多种镜像格式，包括OCI标准镜像格式、external rootfs镜像格式和embedded image镜像格式。 ++`iSulad`支持多种镜像格式，包括OCI标准镜像格式和external rootfs镜像格式。 + + #### OCI + +@@ -42,10 +42,6 @@ OCI标准镜像格式是与docker兼容的镜像格式，支持从远程镜像 + + external rootfs镜像格式允许用户自行准备可启动的`root fs`目录，主要用于系统容器场景。 + +-#### embedded image +- +-embedded image镜像格式是`iSulad`特有的嵌入式镜像格式，占用资源低，主要用于嵌入式应用场景。 +- + ### Operation Interface + + `iSulad`提供两种不同的镜像和容器管理操作接口，分别为CLI和CRI。 +-- +2.42.0 + diff --git a/0041-Acquire-system-info-in-on-demand.patch b/0041-Acquire-system-info-in-on-demand.patch new file mode 100644 index 0000000..4e178e8 --- /dev/null +++ b/0041-Acquire-system-info-in-on-demand.patch @@ -0,0 +1,392 @@ +From 13bc364cb5d8c03b701dde2b2811be84ee608e92 Mon Sep 17 00:00:00 2001 +From: xuxuepeng <xuxuepeng1@huawei.com> +Date: Fri, 24 Nov 2023 14:18:32 +0800 +Subject: [PATCH 41/64] Acquire system info in on demand + +Signed-off-by: xuxuepeng <xuxuepeng1@huawei.com> +--- + src/cmd/isulad/main.c | 2 -- + src/daemon/common/sysinfo.c | 10 +----- + src/daemon/common/sysinfo.h | 6 ++++ + .../executor/container_cb/execution_create.c | 32 +++++++++---------- + .../executor/container_cb/execution_extend.c | 9 +++++- + src/daemon/modules/spec/verify.c | 32 ++++++++----------- + src/daemon/modules/spec/verify.h | 5 +-- + test/mocks/sysinfo_mock.cc | 8 +++++ + test/mocks/sysinfo_mock.h | 1 + + test/mocks/verify_mock.cc | 4 +-- + test/mocks/verify_mock.h | 2 +- + 11 files changed, 60 insertions(+), 51 deletions(-) + +diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c +index 5b971a72..95454e2a 100644 +--- a/src/cmd/isulad/main.c ++++ b/src/cmd/isulad/main.c +@@ -1765,8 +1765,6 @@ int main(int argc, char **argv) + + update_isulad_rlimits(); + +- (void)get_sys_info(true); +- + clock_gettime(CLOCK_MONOTONIC, &t_start); + + if (pre_init_daemon(argc, argv) != 0) { +diff --git a/src/daemon/common/sysinfo.c b/src/daemon/common/sysinfo.c +index 39338925..28665834 100644 +--- a/src/daemon/common/sysinfo.c ++++ b/src/daemon/common/sysinfo.c +@@ -19,7 +19,6 @@ + #include <errno.h> + #include <sys/sysinfo.h> + +-#include <isula_libutils/auto_cleanup.h> + #include <isula_libutils/log.h> + + #include "err_msg.h" +@@ -30,8 +29,6 @@ + #define etcOsRelease "/etc/os-release" + #define altOsRelease "/usr/lib/os-release" + +-static sysinfo_t *g_sysinfo = NULL; +- + static char *get_pagesize(const char *pline) + { + size_t headlen; +@@ -382,10 +379,6 @@ sysinfo_t *get_sys_info(bool quiet) + sysinfo_t *sysinfo = NULL; + int ret = 0; + +- if (g_sysinfo != NULL) { +- return g_sysinfo; +- } +- + sysinfo = util_common_calloc_s(sizeof(sysinfo_t)); + if (sysinfo == NULL) { + ERROR("Out of memory"); +@@ -413,7 +406,6 @@ sysinfo_t *get_sys_info(bool quiet) + if (ret != 0) { + goto out; + } +- g_sysinfo = sysinfo; + out: + if (ret != 0) { + free_sysinfo(sysinfo); +@@ -577,7 +569,7 @@ char *sysinfo_cgroup_controller_cpurt_mnt_path(void) + __isula_auto_free char *mnt = NULL; + __isula_auto_free char *root = NULL; + char fpath[PATH_MAX] = { 0 }; +- sysinfo_t *sysinfo = NULL; ++ __isula_auto_sysinfo_t sysinfo_t *sysinfo = NULL; + + sysinfo = get_sys_info(true); + if (sysinfo == NULL) { +diff --git a/src/daemon/common/sysinfo.h b/src/daemon/common/sysinfo.h +index 363576a9..cb44d1c5 100644 +--- a/src/daemon/common/sysinfo.h ++++ b/src/daemon/common/sysinfo.h +@@ -21,6 +21,7 @@ extern "C" { + + #include <stdbool.h> + #include <stdint.h> ++#include <isula_libutils/auto_cleanup.h> + + #include "cgroup.h" + +@@ -96,6 +97,11 @@ void free_mounts_info(mountinfo_t **minfos); + + char *sysinfo_cgroup_controller_cpurt_mnt_path(void); + ++// define auto free function callback for sysinfo_t ++define_auto_cleanup_callback(free_sysinfo, sysinfo_t) ++// define auto free macro for sysinfo_t ++#define __isula_auto_sysinfo_t auto_cleanup_tag(free_sysinfo) ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c +index 6b6c3b75..ca2a9163 100644 +--- a/src/daemon/executor/container_cb/execution_create.c ++++ b/src/daemon/executor/container_cb/execution_create.c +@@ -145,7 +145,7 @@ static int merge_external_rootfs_to_host_config(host_config *host_spec, const ch + return 0; + } + +-static host_config *get_host_spec(const container_create_request *request) ++static host_config *get_host_spec(const container_create_request *request, const sysinfo_t *sysinfo) + { + host_config *host_spec = NULL; + +@@ -158,7 +158,7 @@ static host_config *get_host_spec(const container_create_request *request) + goto error_out; + } + +- if (verify_host_config_settings(host_spec, false)) { ++ if (verify_host_config_settings(host_spec, sysinfo, false)) { + ERROR("Failed to verify host config settings"); + goto error_out; + } +@@ -1109,17 +1109,9 @@ static int preparate_runtime_environment(const container_create_request *request + return 0; + } + +-static int adapt_host_spec(host_config *host_spec) ++static int adapt_host_spec(host_config *host_spec, const sysinfo_t *sysinfo) + { + int ret = 0; +- sysinfo_t *sysinfo = NULL; +- +- sysinfo = get_sys_info(true); +- if (sysinfo == NULL) { +- ERROR("Can not get system info"); +- ret = -1; +- goto out; +- } + + if (host_spec->memory > 0 && host_spec->memory_swap == 0 && sysinfo->cgmeminfo.swap) { + if (host_spec->memory > (INT64_MAX / 2)) { +@@ -1136,14 +1128,14 @@ out: + } + + static int get_basic_spec(const container_create_request *request, host_config **host_spec, +- container_config **container_spec) ++ container_config **container_spec, const sysinfo_t *sysinfo) + { +- *host_spec = get_host_spec(request); ++ *host_spec = get_host_spec(request, sysinfo); + if (*host_spec == NULL) { + return -1; + } + +- if (adapt_host_spec(*host_spec) != 0) { ++ if (adapt_host_spec(*host_spec, sysinfo) != 0) { + return -1; + } + +@@ -1393,6 +1385,7 @@ int container_create_cb(const container_create_request *request, container_creat + int ret = 0; + bool skip_id_name_manage = false; + bool skip_sandbox_key_manage = false; ++ __isula_auto_sysinfo_t sysinfo_t *sysinfo = NULL; + + DAEMON_CLEAR_ERRMSG(); + +@@ -1413,7 +1406,14 @@ int container_create_cb(const container_create_request *request, container_creat + goto clean_nameindex; + } + +- if (get_basic_spec(request, &host_spec, &container_spec) != 0) { ++ sysinfo = get_sys_info(true); ++ if (sysinfo == NULL) { ++ ERROR("Failed to get system info"); ++ cc = ISULAD_ERR_EXEC; ++ goto clean_nameindex; ++ } ++ ++ if (get_basic_spec(request, &host_spec, &container_spec, sysinfo) != 0) { + cc = ISULAD_ERR_INPUT; + goto clean_container_root_dir; + } +@@ -1540,7 +1540,7 @@ int container_create_cb(const container_create_request *request, container_creat + goto clean_netns; + } + +- if (verify_container_settings(oci_spec) != 0) { ++ if (verify_container_settings(oci_spec, sysinfo) != 0) { + ERROR("Failed to verify container settings"); + cc = ISULAD_ERR_EXEC; + goto umount_channel; +diff --git a/src/daemon/executor/container_cb/execution_extend.c b/src/daemon/executor/container_cb/execution_extend.c +index de017b4e..25ec5d3b 100644 +--- a/src/daemon/executor/container_cb/execution_extend.c ++++ b/src/daemon/executor/container_cb/execution_extend.c +@@ -1110,8 +1110,15 @@ static int update_host_config_check(container_t *cont, host_config *hostconfig) + { + int ret = 0; + const char *id = cont->common_config->id; ++ __isula_auto_sysinfo_t sysinfo_t *sysinfo = NULL; + +- ret = verify_host_config_settings(hostconfig, true); ++ sysinfo = get_sys_info(true); ++ if (sysinfo == NULL) { ++ ERROR("Failed to get system info for updating container %s", id); ++ return -1; ++ } ++ ++ ret = verify_host_config_settings(hostconfig, sysinfo, true); + if (ret != 0) { + return -1; + } +diff --git a/src/daemon/modules/spec/verify.c b/src/daemon/modules/spec/verify.c +index 2a8b3259..b9e3c606 100644 +--- a/src/daemon/modules/spec/verify.c ++++ b/src/daemon/modules/spec/verify.c +@@ -41,7 +41,6 @@ + #include "constants.h" + #include "err_msg.h" + #include "isula_libutils/log.h" +-#include "sysinfo.h" + #include "selinux_label.h" + #include "image_api.h" + #include "utils.h" +@@ -1614,16 +1613,13 @@ out: + } + + /* verify container settings */ +-int verify_container_settings(const oci_runtime_spec *container) ++int verify_container_settings(const oci_runtime_spec *container, const sysinfo_t *sysinfo) + { + int ret = 0; +- sysinfo_t *sysinfo = NULL; + +- sysinfo = get_sys_info(true); +- if (sysinfo == NULL) { +- ERROR("Can not get system info"); +- ret = -1; +- goto out; ++ if (container == NULL || sysinfo == NULL) { ++ ERROR("Invalid input arguments for verifying container settings"); ++ return -1; + } + + if (!util_valid_host_name(container->hostname)) { +@@ -1987,16 +1983,9 @@ static int host_config_settings_restart_policy(const host_config *hostconfig) + return verify_restart_policy_name(rp, hostconfig); + } + +-static int host_config_settings_with_sysinfo(host_config *hostconfig, bool update) ++static int host_config_settings_with_sysinfo(host_config *hostconfig, const sysinfo_t *sysinfo, bool update) + { + int ret = 0; +- sysinfo_t *sysinfo = NULL; +- +- sysinfo = get_sys_info(true); +- if (sysinfo == NULL) { +- ERROR("Can not get system info"); +- return -1; +- } + + ret = verify_host_config_hugetlbs(sysinfo, &(hostconfig->hugetlbs), &(hostconfig->hugetlbs_len)); + if (ret != 0) { +@@ -2055,7 +2044,7 @@ out: + } + + /* verify host config settings */ +-int verify_host_config_settings(host_config *hostconfig, bool update) ++int verify_host_config_settings(host_config *hostconfig, const sysinfo_t *sysinfo, bool update) + { + int ret = 0; + #ifdef ENABLE_USERNS_REMAP +@@ -2066,6 +2055,13 @@ int verify_host_config_settings(host_config *hostconfig, bool update) + goto out; + } + ++ if (sysinfo == NULL) { ++ ERROR("Invalid sysinfo for verifying host config settings"); ++ isulad_set_error_message("Invalid sysinfo for verifying host config settings"); ++ ret = -1; ++ goto out; ++ } ++ + #ifdef ENABLE_USERNS_REMAP + if (userns_remap != NULL && hostconfig->user_remap != NULL) { + ERROR("invalid --user-remap command option, daemon already configed --userns-remap"); +@@ -2081,7 +2077,7 @@ int verify_host_config_settings(host_config *hostconfig, bool update) + goto out; + } + +- ret = host_config_settings_with_sysinfo(hostconfig, update); ++ ret = host_config_settings_with_sysinfo(hostconfig, sysinfo, update); + if (ret != 0) { + goto out; + } +diff --git a/src/daemon/modules/spec/verify.h b/src/daemon/modules/spec/verify.h +index 21e8fba8..0224f9fb 100644 +--- a/src/daemon/modules/spec/verify.h ++++ b/src/daemon/modules/spec/verify.h +@@ -20,18 +20,19 @@ + #include "isula_libutils/oci_runtime_spec.h" + #include "isula_libutils/host_config.h" + #include "isula_libutils/container_config.h" ++#include "sysinfo.h" + + #ifdef __cplusplus + extern "C" { + #endif + +-int verify_container_settings(const oci_runtime_spec *container); ++int verify_container_settings(const oci_runtime_spec *container, const sysinfo_t *sysinfo); + + int verify_oci_hook(const oci_runtime_spec_hooks *h); + + int verify_container_settings_start(const oci_runtime_spec *oci_spec); + +-int verify_host_config_settings(host_config *hostconfig, bool update); ++int verify_host_config_settings(host_config *hostconfig, const sysinfo_t *sysinfo, bool update); + + int verify_container_config(const container_config *container_spec, const char *runtime); + +diff --git a/test/mocks/sysinfo_mock.cc b/test/mocks/sysinfo_mock.cc +index f9abc786..d8f33f84 100644 +--- a/test/mocks/sysinfo_mock.cc ++++ b/test/mocks/sysinfo_mock.cc +@@ -63,6 +63,14 @@ char *validate_hugetlb(const char *pagesize, uint64_t limit) + return nullptr; + } + ++sysinfo_t *get_sys_info(bool quiet) ++{ ++ if (g_sysinfo_mock != nullptr) { ++ return g_sysinfo_mock->GetSysInfo(quiet); ++ } ++ return nullptr; ++} ++ + void free_sysinfo(sysinfo_t *sysinfo) + { + if (g_sysinfo_mock != nullptr) { +diff --git a/test/mocks/sysinfo_mock.h b/test/mocks/sysinfo_mock.h +index 45208b0f..2b8e926d 100644 +--- a/test/mocks/sysinfo_mock.h ++++ b/test/mocks/sysinfo_mock.h +@@ -26,6 +26,7 @@ public: + MOCK_METHOD1(FreeMountsInfo, void(mountinfo_t **minfos)); + MOCK_METHOD0(GetDefaultHugePageSize, char *(void)); + MOCK_METHOD2(ValidateHugetlb, char*(const char *pagesize, uint64_t limit)); ++ MOCK_METHOD1(GetSysInfo, sysinfo_t *(bool quiet)); + MOCK_METHOD1(FreeSysinfo, void(sysinfo_t *sysinfo)); + }; + +diff --git a/test/mocks/verify_mock.cc b/test/mocks/verify_mock.cc +index 0e7e7461..4c481676 100644 +--- a/test/mocks/verify_mock.cc ++++ b/test/mocks/verify_mock.cc +@@ -24,10 +24,10 @@ void MockVerify_SetMock(MockVerify *mock) + g_verify_mock = mock; + } + +-int verify_host_config_settings(host_config *hostconfig, bool update) ++int verify_host_config_settings(host_config *hostconfig, const sysinfo_t *sysinfo, bool update) + { + if (g_verify_mock != nullptr) { +- return g_verify_mock->VerifyHostConfigSettings(hostconfig, update); ++ return g_verify_mock->VerifyHostConfigSettings(hostconfig, sysinfo, update); + } + return 0; + } +diff --git a/test/mocks/verify_mock.h b/test/mocks/verify_mock.h +index 7890159f..b9ad8627 100644 +--- a/test/mocks/verify_mock.h ++++ b/test/mocks/verify_mock.h +@@ -21,7 +21,7 @@ + + class MockVerify { + public: +- MOCK_METHOD2(VerifyHostConfigSettings, int(host_config *hostconfig, bool update)); ++ MOCK_METHOD3(VerifyHostConfigSettings, int(host_config *hostconfig, const sysinfo_t *sysinfo, bool update)); + }; + + void MockVerify_SetMock(MockVerify* mock); +-- +2.42.0 + diff --git a/0042-2268-bugfix-for-the-bliko-zero-value-exception-when-.patch b/0042-2268-bugfix-for-the-bliko-zero-value-exception-when-.patch new file mode 100644 index 0000000..2c9f725 --- /dev/null +++ b/0042-2268-bugfix-for-the-bliko-zero-value-exception-when-.patch @@ -0,0 +1,92 @@ +From dddba4ec73b56bc2fcf3a95171fad104e962dfda Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Wed, 29 Nov 2023 09:33:53 +0000 +Subject: [PATCH 42/64] =?UTF-8?q?!2268=20bugfix=20for=20the=20bliko=20zero?= + =?UTF-8?q?=20value=20exception=20when=20executing=20the=20stats=20command?= + =?UTF-8?q?=20on=20the=20oci=20container=20*=20bugfix=20for=20the=20bliko?= + =?UTF-8?q?=20zero=20value=20exception=20when=20executing=20the=20stats=20?= + =?UTF-8?q?com=E2=80=A6?= +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +--- + .../modules/runtime/isula/isula_rt_ops.c | 55 +++++++++++++------ + 1 file changed, 38 insertions(+), 17 deletions(-) + +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 5d7ae500..1e2ecdb2 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -610,6 +610,43 @@ out: + return ret; + } + ++static void transform_stats_info_from_runtime(shim_client_runtime_stats *stats, struct runtime_container_resources_stats_info *info) ++{ ++ if (stats == NULL || stats->data == NULL) { ++ return; ++ } ++ if (stats->data->pids != NULL) { ++ info->pids_current = stats->data->pids->current; ++ } ++ if (stats->data->cpu != NULL && stats->data->cpu->usage != NULL) { ++ info->cpu_use_nanos = stats->data->cpu->usage->total; ++ info->cpu_system_use = stats->data->cpu->usage->kernel; ++ } ++ shim_client_runtime_stats_data_memory *memory = stats->data->memory; ++ if (memory != NULL && memory->usage != NULL) { ++ info->mem_used = memory->usage->usage; ++ info->mem_limit = memory->usage->limit; ++ } ++ if (memory != NULL && memory->raw != NULL) { ++ info->inactive_file_total = memory->raw->total_inactive_file; ++ info->rss_bytes = memory->raw->rss; ++ info->page_faults = memory->raw->pgfault; ++ info->major_page_faults = memory->raw->pgmajfault; ++ } ++ shim_client_runtime_stats_data_blkio *blkio = stats->data->blkio; ++ if (blkio == NULL) { ++ return; ++ } ++ for (size_t i = 0; i < blkio->io_service_bytes_recursive_len; i++) { ++ if (strcasecmp(blkio->io_service_bytes_recursive[i]->op, "read") == 0) { ++ info->blkio_read += blkio->io_service_bytes_recursive[i]->value; ++ } ++ if (strcasecmp(blkio->io_service_bytes_recursive[i]->op, "write") == 0) { ++ info->blkio_write += blkio->io_service_bytes_recursive[i]->value; ++ } ++ } ++} ++ + static int runtime_call_stats(const char *workdir, const char *runtime, const char *id, + struct runtime_container_resources_stats_info *info) + { +@@ -658,23 +695,7 @@ static int runtime_call_stats(const char *workdir, const char *runtime, const ch + goto out; + } + +- if (stats != NULL && stats->data != NULL && stats->data->pids != NULL) { +- info->pids_current = stats->data->pids->current; +- } +- if (stats != NULL && stats->data != NULL && stats->data->cpu != NULL && stats->data->cpu->usage) { +- info->cpu_use_nanos = stats->data->cpu->usage->total; +- info->cpu_system_use = stats->data->cpu->usage->kernel; +- } +- if (stats != NULL && stats->data != NULL && stats->data->memory != NULL && stats->data->memory->usage) { +- info->mem_used = stats->data->memory->usage->usage; +- info->mem_limit = stats->data->memory->usage->limit; +- } +- if (stats != NULL && stats->data != NULL && stats->data->memory != NULL && stats->data->memory->raw) { +- info->inactive_file_total = stats->data->memory->raw->total_inactive_file; +- info->rss_bytes = stats->data->memory->raw->rss; +- info->page_faults = stats->data->memory->raw->pgfault; +- info->major_page_faults = stats->data->memory->raw->pgmajfault; +- } ++ transform_stats_info_from_runtime(stats, info); + + out: + free_shim_client_runtime_stats(stats); +-- +2.42.0 + diff --git a/0043-move-variable-declaration-out-of-loop.patch b/0043-move-variable-declaration-out-of-loop.patch new file mode 100644 index 0000000..26748eb --- /dev/null +++ b/0043-move-variable-declaration-out-of-loop.patch @@ -0,0 +1,34 @@ +From 261a924b656eea9eff2ca6cbdd611eb1f9555af7 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Thu, 30 Nov 2023 16:02:44 +1400 +Subject: [PATCH 43/64] move variable declaration out of loop + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/modules/runtime/isula/isula_rt_ops.c | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 1e2ecdb2..3950ff4a 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -612,6 +612,7 @@ out: + + static void transform_stats_info_from_runtime(shim_client_runtime_stats *stats, struct runtime_container_resources_stats_info *info) + { ++ size_t i; + if (stats == NULL || stats->data == NULL) { + return; + } +@@ -637,7 +638,7 @@ static void transform_stats_info_from_runtime(shim_client_runtime_stats *stats, + if (blkio == NULL) { + return; + } +- for (size_t i = 0; i < blkio->io_service_bytes_recursive_len; i++) { ++ for (i = 0; i < blkio->io_service_bytes_recursive_len; i++) { + if (strcasecmp(blkio->io_service_bytes_recursive[i]->op, "read") == 0) { + info->blkio_read += blkio->io_service_bytes_recursive[i]->value; + } +-- +2.42.0 + diff --git a/0044-2289-check-protobuf-and-grpc-version-in-cmake-for-cr.patch b/0044-2289-check-protobuf-and-grpc-version-in-cmake-for-cr.patch new file mode 100644 index 0000000..5fa3a05 --- /dev/null +++ b/0044-2289-check-protobuf-and-grpc-version-in-cmake-for-cr.patch @@ -0,0 +1,40 @@ +From 8045fcfb3765698d8cc3f07186fcc29d6702ee71 Mon Sep 17 00:00:00 2001 +From: jake <jikai11@huawei.com> +Date: Thu, 30 Nov 2023 11:58:47 +0000 +Subject: [PATCH 44/64] !2289 check protobuf and grpc version in cmake for cri + v1 * check protobuf and grpc version in cmake for cri v1 + +--- + cmake/checker.cmake | 9 ++++++++- + 1 file changed, 8 insertions(+), 1 deletion(-) + +diff --git a/cmake/checker.cmake b/cmake/checker.cmake +index cc4a1fc3..e19618e4 100644 +--- a/cmake/checker.cmake ++++ b/cmake/checker.cmake +@@ -125,7 +125,11 @@ endif() + + if (GRPC_CONNECTOR) + # check protobuf +- pkg_check_modules(PC_PROTOBUF "protobuf>=3.1.0") ++ if (ENABLE_CRI_API_V1) ++ pkg_check_modules(PC_PROTOBUF "protobuf>=3.14.0") ++ else() ++ pkg_check_modules(PC_PROTOBUF "protobuf>=3.1.0") ++ endif() + find_library(PROTOBUF_LIBRARY protobuf + HINTS ${PC_PROTOBUF_LIBDIR} ${PC_PROTOBUF_LIBRARY_DIRS}) + _CHECK(PROTOBUF_LIBRARY "PROTOBUF_LIBRARY-NOTFOUND" "libprotobuf.so") +@@ -136,6 +140,9 @@ if (GRPC_CONNECTOR) + _CHECK(CMD_GRPC_CPP_PLUGIN "CMD_GRPC_CPP_PLUGIN-NOTFOUND" "grpc_cpp_plugin") + + # check grpc ++ if (ENABLE_CRI_API_V1) ++ pkg_check_modules(PC_GRPC++ "grpc++>=1.41.0") ++ endif() + find_path(GRPC_INCLUDE_DIR grpc/grpc.h) + _CHECK(GRPC_INCLUDE_DIR "GRPC_INCLUDE_DIR-NOTFOUND" "grpc/grpc.h") + find_library(GRPC_PP_REFLECTION_LIBRARY grpc++_reflection) +-- +2.42.0 + diff --git a/0045-improve-ut-for-devicemapper.patch b/0045-improve-ut-for-devicemapper.patch new file mode 100644 index 0000000..ad87cba --- /dev/null +++ b/0045-improve-ut-for-devicemapper.patch @@ -0,0 +1,381 @@ +From 2ad7ecf5adbd75f1ba4678e69d768d4b807ae08d Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Wed, 29 Nov 2023 17:08:31 +0800 +Subject: [PATCH 45/64] improve ut for devicemapper + +Signed-off-by: jikai <jikai11@huawei.com> +--- + .../storage/layers/devmapper/CMakeLists.txt | 8 +- + .../id | 1 + + .../layers/devmapper/driver_devmapper_ut.cc | 209 +++++++++++++----- + 3 files changed, 165 insertions(+), 53 deletions(-) + create mode 100644 test/image/oci/storage/layers/devmapper/data/devicemapper/mnt/068615102be4457b22d40c9702e4bd316dd50ae85467b0378a419b23b60ba73d/id + +diff --git a/test/image/oci/storage/layers/devmapper/CMakeLists.txt b/test/image/oci/storage/layers/devmapper/CMakeLists.txt +index f98de1a8..e6ba0307 100644 +--- a/test/image/oci/storage/layers/devmapper/CMakeLists.txt ++++ b/test/image/oci/storage/layers/devmapper/CMakeLists.txt +@@ -23,14 +23,18 @@ add_executable(${DRIVER_DEVMAPPER_EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/utils/tar/util_gzip.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/utils/sha256/sha256.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/config/daemon_arguments.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/config/isulad_config.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/common/err_msg.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/common/selinux_label.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/deviceset.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/driver_devmapper.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/metadata_store.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/wrapper_devmapper.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/quota/project_quota.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../mocks/libdevmapper_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../mocks/isulad_config_mock.cc + driver_devmapper_ut.cc) + + target_include_directories(${DRIVER_DEVMAPPER_EXE} PUBLIC +@@ -56,7 +60,7 @@ target_include_directories(${DRIVER_DEVMAPPER_EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../mocks + ) + +-set_target_properties(${DRIVER_DEVMAPPER_EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,util_exec_cmd -Wl,--wrap,util_mount -Wl,--wrap,umount2") ++set_target_properties(${DRIVER_DEVMAPPER_EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,util_exec_cmd -Wl,--wrap,util_mount -Wl,--wrap,umount2 -Wl,--wrap,archive_unpack") + + target_link_libraries(${DRIVER_DEVMAPPER_EXE} + ${GTEST_BOTH_LIBRARIES} +diff --git a/test/image/oci/storage/layers/devmapper/data/devicemapper/mnt/068615102be4457b22d40c9702e4bd316dd50ae85467b0378a419b23b60ba73d/id b/test/image/oci/storage/layers/devmapper/data/devicemapper/mnt/068615102be4457b22d40c9702e4bd316dd50ae85467b0378a419b23b60ba73d/id +new file mode 100644 +index 00000000..5e6b1b2a +--- /dev/null ++++ b/test/image/oci/storage/layers/devmapper/data/devicemapper/mnt/068615102be4457b22d40c9702e4bd316dd50ae85467b0378a419b23b60ba73d/id +@@ -0,0 +1 @@ ++3d24ee258efc3bfe4066a1a9fb83febf6dc0b1548dfe896161533668281c9f4f +\ No newline at end of file +diff --git a/test/image/oci/storage/layers/devmapper/driver_devmapper_ut.cc b/test/image/oci/storage/layers/devmapper/driver_devmapper_ut.cc +index 59e53f97..088aa4d4 100644 +--- a/test/image/oci/storage/layers/devmapper/driver_devmapper_ut.cc ++++ b/test/image/oci/storage/layers/devmapper/driver_devmapper_ut.cc +@@ -16,11 +16,16 @@ + #include <gtest/gtest.h> + #include <gmock/gmock.h> + ++#include "driver.h" + #include "driver_devmapper.h" ++#include "driver_overlay2.h" + #include "mock.h" + #include "path.h" + #include "utils.h" ++#include "util_archive.h" + #include "libdevmapper_mock.h" ++#include "isulad_config_mock.h" ++#include "wrapper_devmapper.h" + + using ::testing::Invoke; + using ::testing::NiceMock; +@@ -29,13 +34,20 @@ using ::testing::_; + + extern "C" { + DECLARE_WRAPPER_V(util_exec_cmd, bool, (exec_func_t cb_func, void *args, const char *stdin_msg, char **stdout_msg, char **stderr_msg)); +- DEFINE_WRAPPER_V(util_exec_cmd, bool, (exec_func_t cb_func, void *args, const char *stdin_msg, char **stdout_msg, char **stderr_msg), (cb_func, args, stdin_msg, stdout_msg, stderr_msg)); ++ DEFINE_WRAPPER_V(util_exec_cmd, bool, (exec_func_t cb_func, void *args, const char *stdin_msg, char **stdout_msg, char **stderr_msg), ++ (cb_func, args, stdin_msg, stdout_msg, stderr_msg)); + + DECLARE_WRAPPER(util_mount, int, (const char *src, const char *dst, const char *mtype, const char *mntopts)); + DEFINE_WRAPPER(util_mount, int, (const char *src, const char *dst, const char *mtype, const char *mntopts), (src, dst, mtype, mntopts)); + + DECLARE_WRAPPER(umount2, int, (const char *__special_file, int __flags)); + DEFINE_WRAPPER(umount2, int, (const char *__special_file, int __flags), (__special_file, __flags)); ++ ++ DECLARE_WRAPPER(archive_unpack, int, (const io_read_wrapper *content, const char *dstdir, const archive_options *options, ++ const char *root_dir, char **errmsg)); ++ DEFINE_WRAPPER(archive_unpack, int, (const io_read_wrapper *content, const char *dstdir, const archive_options *options, ++ const char *root_dir, char **errmsg), ++ (content, dstdir, options, root_dir, errmsg)); + } + + static std::string GetDirectory() +@@ -118,6 +130,7 @@ protected: + void SetUp() override + { + MockLibdevmapper_SetMock(&m_libdevmapper_mock); ++ MockIsuladConf_SetMock(&m_isulad_conf_mock); + std::string isulad_dir { "/tmp/isulad/" }; + mkdir(isulad_dir.c_str(), 0755); + std::string root_dir = isulad_dir + "data"; +@@ -129,13 +142,18 @@ protected: + std::string cp_command = "cp -r " + std::string(data_path) + " " + isulad_dir; + ASSERT_EQ(system(cp_command.c_str()), 0); + +- char **driver_opts = static_cast<char **>(util_common_calloc_s(3 * sizeof(char *))); +- driver_opts[0] = strdup("dm.thinpooldev=/dev/mapper/isulad0-thinpool"); +- driver_opts[1] = strdup("dm.fs=ext4"); +- driver_opts[2] = strdup("dm.min_free_space=10%"); +- int driver_opts_len = 3; +- +- ASSERT_EQ(devmapper_init(&driver, nullptr, (const char **)driver_opts, driver_opts_len), -1); ++ opts = (struct storage_module_init_options *)util_common_calloc_s(sizeof(struct storage_module_init_options)); ++ opts->storage_root = strdup(root_dir.c_str()); ++ opts->storage_run_root = strdup(run_dir.c_str()); ++ opts->driver_name = strdup("devicemapper"); ++ opts->driver_opts = (char **)util_common_calloc_s(6 * sizeof(char *)); ++ opts->driver_opts[0] = strdup("dm.thinpooldev=/dev/mapper/isulad0-thinpool"); ++ opts->driver_opts[1] = strdup("dm.fs=ext4"); ++ opts->driver_opts[2] = strdup("dm.min_free_space=10%"); ++ opts->driver_opts[3] = strdup("dm.basesize=12G"); ++ opts->driver_opts[4] = strdup("dm.mkfsarg=-q"); ++ opts->driver_opts[5] = strdup("dm.mountopt=rw"); ++ opts->driver_opts_len = 6; + + EXPECT_CALL(m_libdevmapper_mock, DMTaskCreate(_)).WillRepeatedly(Invoke(invokeDMTaskCreate)); + EXPECT_CALL(m_libdevmapper_mock, DMTaskSetMessage(_, _)).WillRepeatedly(Return(1)); +@@ -152,7 +170,6 @@ protected: + EXPECT_CALL(m_libdevmapper_mock, DMUdevComplete(_)).WillRepeatedly(Return(1)); + EXPECT_CALL(m_libdevmapper_mock, DMTaskDeferredRemove(_)).WillRepeatedly(Return(1)); + +- + char *names = static_cast<char *>(util_common_calloc_s(sizeof(struct dm_names) + strlen("isulad0-pool") + 1)); + struct dm_names *dname = (struct dm_names *)names; + dname->dev = 1; +@@ -164,32 +181,76 @@ protected: + EXPECT_CALL(m_libdevmapper_mock, DMUdevGetSyncSupport()).WillOnce(Return(1)); + + MOCK_SET_V(util_exec_cmd, invokeUtilExecCmd); +- +- ASSERT_EQ(devmapper_init(&driver, driver_home.c_str(), (const char **)driver_opts, driver_opts_len), 0); ++ MOCK_SET(util_mount, 0); ++ MOCK_SET(umount2, 0); ++ ASSERT_EQ(graphdriver_init(opts), 0); + MOCK_CLEAR(util_exec_cmd); +- +- util_free_array_by_len(driver_opts, driver_opts_len); +- free(names); ++ MOCK_CLEAR(util_mount); ++ MOCK_CLEAR(umount2); + } + + void TearDown() override + { ++ MOCK_SET(umount2, 0); ++ ASSERT_EQ(graphdriver_cleanup(), 0); ++ MOCK_CLEAR(umount2); ++ + MockLibdevmapper_SetMock(nullptr); ++ MockIsuladConf_SetMock(nullptr); + std::string rm_command = "rm -rf /tmp/isulad/"; + ASSERT_EQ(system(rm_command.c_str()), 0); ++ ++ if (opts != NULL) { ++ free(opts->storage_root); ++ free(opts->storage_run_root); ++ free(opts->driver_name); ++ util_free_array_by_len(opts->driver_opts, opts->driver_opts_len); ++ free(opts); ++ } + } + + NiceMock<MockLibdevmapper> m_libdevmapper_mock; ++ NiceMock<MockIsuladConf> m_isulad_conf_mock; + char data_path[PATH_MAX] = { 0x00 }; +- graphdriver driver = {.ops = nullptr, .name = "devicemapper", }; ++ struct storage_module_init_options *opts = NULL; + }; + ++TEST_F(DriverDevmapperUnitTest, test_devmapper_init) ++{ ++ // cleanup before ++ MOCK_SET(umount2, 0); ++ ASSERT_EQ(graphdriver_cleanup(), 0); ++ MOCK_CLEAR(umount2); ++ ++ std::string rm_command = "rm -rf /tmp/isulad/"; ++ ASSERT_EQ(system(rm_command.c_str()), 0); ++ std::string mk_command = "mkdir -p /tmp/isulad/data/devicemapper/mnt"; ++ ASSERT_EQ(system(mk_command.c_str()), 0); ++ char *names = static_cast<char *>(util_common_calloc_s(sizeof(struct dm_names) + strlen("isulad0-pool") + 1)); ++ struct dm_names *dname = (struct dm_names *)names; ++ dname->dev = 1; ++ dname->next = 0; ++ strcpy(names + sizeof(struct dm_names), "isulad0-pool"); ++ EXPECT_CALL(m_libdevmapper_mock, DMTaskGetNames(_)).WillOnce(Return(dname)); ++ EXPECT_CALL(m_libdevmapper_mock, DMSetDevDir(_)).WillOnce(Return(1)); ++ EXPECT_CALL(m_libdevmapper_mock, DMTaskGetDriverVersion(_, _, _)).WillOnce(Invoke(invokeDMTaskGetDriverVersion)); ++ EXPECT_CALL(m_libdevmapper_mock, DMUdevGetSyncSupport()).WillOnce(Return(1)); ++ ++ MOCK_SET_V(util_exec_cmd, invokeUtilExecCmd); ++ MOCK_SET(util_mount, 0); ++ MOCK_SET(umount2, 0); ++ ASSERT_EQ(graphdriver_init(opts), 0); ++ MOCK_CLEAR(util_exec_cmd); ++ MOCK_CLEAR(util_mount); ++ MOCK_CLEAR(umount2); ++} ++ + TEST_F(DriverDevmapperUnitTest, test_devmapper_layer_exists) + { + std::string id { "3d24ee258efc3bfe4066a1a9fb83febf6dc0b1548dfe896161533668281c9f4f" }; + std::string incorrectId { "eb29745b8228e1e97c01b1d5c2554a319c00a94d8dd5746a3904222ad65a13f8" }; +- ASSERT_TRUE(devmapper_layer_exist(id.c_str(), &driver)); +- ASSERT_FALSE(devmapper_layer_exist(incorrectId.c_str(), &driver)); ++ ASSERT_TRUE(graphdriver_layer_exists(id.c_str())); ++ ASSERT_FALSE(graphdriver_layer_exists(incorrectId.c_str())); + } + + TEST_F(DriverDevmapperUnitTest, test_devmapper_create_rw) +@@ -205,11 +266,42 @@ TEST_F(DriverDevmapperUnitTest, test_devmapper_create_rw) + create_opts->storage_opt->keys = static_cast<char **>(util_common_calloc_s(sizeof(char *))); + create_opts->storage_opt->values = static_cast<char **>(util_common_calloc_s(sizeof(char *))); + create_opts->storage_opt->keys[0] = strdup("size"); +- create_opts->storage_opt->values[0] = strdup("10G"); ++ create_opts->storage_opt->values[0] = strdup("12G"); ++ create_opts->storage_opt->len = 1; ++ ++ ASSERT_EQ(graphdriver_create_rw(id.c_str(), nullptr, create_opts), 0); ++ ASSERT_TRUE(graphdriver_layer_exists(id.c_str())); ++ free_driver_create_opts(create_opts); ++} ++ ++TEST_F(DriverDevmapperUnitTest, test_devmapper_create_ro) ++{ ++ std::string id { "eb29745b8228e1e97c01b1d5c2554a319c00a94d8dd5746a3904222ad65a13f8" }; ++ struct driver_create_opts *create_opts; ++ ++ create_opts = (struct driver_create_opts *)util_common_calloc_s(sizeof(struct driver_create_opts)); ++ ASSERT_NE(create_opts, nullptr); ++ ++ create_opts->storage_opt = static_cast<json_map_string_string *>(util_common_calloc_s(sizeof(json_map_string_string))); ++ ASSERT_NE(create_opts->storage_opt, nullptr); ++ create_opts->storage_opt->keys = static_cast<char **>(util_common_calloc_s(sizeof(char *))); ++ create_opts->storage_opt->values = static_cast<char **>(util_common_calloc_s(sizeof(char *))); ++ create_opts->storage_opt->keys[0] = strdup("size"); ++ create_opts->storage_opt->values[0] = strdup("12G"); + create_opts->storage_opt->len = 1; + +- ASSERT_EQ(devmapper_create_rw(id.c_str(), nullptr, &driver, create_opts), 0); +- ASSERT_TRUE(devmapper_layer_exist(id.c_str(), &driver)); ++ ASSERT_EQ(graphdriver_create_ro(id.c_str(), nullptr, create_opts), 0); ++ ASSERT_TRUE(graphdriver_layer_exists(id.c_str())); ++ free_driver_create_opts(create_opts); ++} ++ ++TEST_F(DriverDevmapperUnitTest, test_devmapper_rm_layer) ++{ ++ std::string existed_id { "3d24ee258efc3bfe4066a1a9fb83febf6dc0b1548dfe896161533668281c9f4f" }; ++ std::string not_existed_id { "eb29745b8228e1e97c01b1d5c2554a319c00a94d8dd5746a3904222ad65a13f8" }; ++ ++ ASSERT_EQ(graphdriver_rm_layer(existed_id.c_str()), 0); ++ ASSERT_EQ(graphdriver_rm_layer(not_existed_id.c_str()), 0); + } + + TEST_F(DriverDevmapperUnitTest, test_devmapper_mount_layer) +@@ -220,12 +312,12 @@ TEST_F(DriverDevmapperUnitTest, test_devmapper_mount_layer) + char* mount_dir = nullptr; + + MOCK_SET(util_mount, 0); +- mount_dir = devmapper_mount_layer(id.c_str(), &driver, mount_opts); ++ mount_dir = graphdriver_mount_layer(id.c_str(), mount_opts); + ASSERT_STREQ(mount_dir, merged_dir.c_str()); + MOCK_CLEAR(util_mount); + + MOCK_SET(umount2, 0); +- ASSERT_EQ(devmapper_umount_layer(id.c_str(), &driver), 0); ++ ASSERT_EQ(graphdriver_umount_layer(id.c_str()), 0); + MOCK_CLEAR(umount2); + free(mount_dir); + mount_dir = nullptr; +@@ -237,47 +329,62 @@ TEST_F(DriverDevmapperUnitTest, test_devmapper_mount_layer) + mount_opts->options_len = 1; + + MOCK_SET(util_mount, 0); +- mount_dir = devmapper_mount_layer(id.c_str(), &driver, mount_opts); ++ mount_dir = graphdriver_mount_layer(id.c_str(), mount_opts); + ASSERT_STREQ(mount_dir, merged_dir.c_str()); + MOCK_CLEAR(util_mount); + + MOCK_SET(umount2, 0); +- ASSERT_EQ(devmapper_umount_layer(id.c_str(), &driver), 0); ++ ASSERT_EQ(graphdriver_umount_layer(id.c_str()), 0); + MOCK_CLEAR(umount2); +- free(mount_opts->mount_label); +- util_free_array_by_len(mount_opts->options, mount_opts->options_len); +- free(mount_opts); +- free(mount_dir); ++ free_driver_mount_opts(mount_opts); + } + +-TEST_F(DriverDevmapperUnitTest, test_devmapper_get_layer_metadata) ++TEST_F(DriverDevmapperUnitTest, test_devmapper_get_data) + { + std::string id { "3d24ee258efc3bfe4066a1a9fb83febf6dc0b1548dfe896161533668281c9f4f" }; +- json_map_string_string *map_info = static_cast<json_map_string_string *>(util_common_calloc_s(sizeof(json_map_string_string))); +- +- ASSERT_EQ(devmapper_get_layer_metadata(id.c_str(), &driver, map_info), 0); +- ASSERT_EQ(map_info->len, 4); +- ASSERT_STREQ(map_info->keys[0], "DeviceId"); +- ASSERT_STREQ(map_info->values[0], "4"); +- ASSERT_STREQ(map_info->keys[1], "DeviceSize"); +- ASSERT_STREQ(map_info->values[1], "10737418240"); +- ASSERT_STREQ(map_info->keys[2], "DeviceName"); +- ASSERT_STREQ(map_info->keys[3], "MergedDir"); +- ASSERT_STREQ(map_info->values[3], "/tmp/isulad/data/devicemapper/mnt/3d24ee258efc3bfe4066a1a9fb83febf6dc0b1548dfe896161533668281c9f4f/rootfs"); +- +- free_json_map_string_string(map_info); +-} + +-TEST_F(DriverDevmapperUnitTest, test_devmapper_get_driver_status) +-{ +- struct graphdriver_status *status = static_cast<struct graphdriver_status *>(util_common_calloc_s(sizeof(struct graphdriver_status))); ++ container_inspect_graph_driver *inspect = graphdriver_get_metadata(id.c_str()); ++ ASSERT_NE(inspect, nullptr); ++ ASSERT_STREQ(inspect->data->device_id, "4"); ++ ASSERT_STREQ(inspect->data->device_size, "10737418240"); ++ ASSERT_STREQ(inspect->data->merged_dir, "/tmp/isulad/data/devicemapper/mnt/3d24ee258efc3bfe4066a1a9fb83febf6dc0b1548dfe896161533668281c9f4f/rootfs"); ++ free_container_inspect_graph_driver(inspect); + + EXPECT_CALL(m_libdevmapper_mock, DMUdevGetSyncSupport()).WillOnce(Return(1)); + +- ASSERT_EQ(devmapper_get_driver_status(&driver, status), 0); ++ struct graphdriver_status *status = graphdriver_get_status(); ++ ASSERT_NE(status, nullptr); + ASSERT_STREQ(status->driver_name, "devicemapper"); +- free(status->driver_name); +- free(status->backing_fs); +- free(status->status); +- free(status); ++ free_graphdriver_status(status); ++ ++ ASSERT_EQ(devmapper_repair_lowers(nullptr, nullptr, nullptr), 0); ++ ASSERT_EQ(devmapper_get_layer_fs_info(nullptr, nullptr, nullptr), 0); + } ++ ++TEST_F(DriverDevmapperUnitTest, test_devmapper_apply_diff) ++{ ++ struct io_read_wrapper reader = {0}; ++ std::string id { "3d24ee258efc3bfe4066a1a9fb83febf6dc0b1548dfe896161533668281c9f4f" }; ++ MOCK_SET(util_mount, 0); ++ MOCK_SET(archive_unpack, 0); ++ MOCK_SET(umount2, 0); ++ EXPECT_CALL(m_isulad_conf_mock, ConfGetISuladRootDir()).WillOnce(Return(util_strdup_s("/tmp/isulad"))); ++ ASSERT_EQ(graphdriver_apply_diff(id.c_str(), &reader), 0); ++ MOCK_CLEAR(archive_unpack); ++ MOCK_CLEAR(util_mount); ++ MOCK_CLEAR(umount2); ++} ++ ++TEST_F(DriverDevmapperUnitTest, test_wrapper_devmapper) ++{ ++ ASSERT_STREQ(dev_strerror(ERR_TASK_RUN), "Task run error"); ++ ASSERT_STREQ(dev_strerror(ERR_TASK_SET_COOKIE), "Task set cookie error"); ++ ASSERT_STREQ(dev_strerror(ERR_NIL_COOKIE), "cookie ptr can't be nil"); ++ ASSERT_STREQ(dev_strerror(ERR_TASK_SET_ADD_NODE), "Task add dm node failed"); ++ ASSERT_STREQ(dev_strerror(ERR_BUSY), "Device busy"); ++ ASSERT_STREQ(dev_strerror(ERR_DEVICE_ID_EXISTS), "Device exists already"); ++ ASSERT_STREQ(dev_strerror(ERR_ENXIO), "No such device of address"); ++ ASSERT_STREQ(dev_strerror(ERR_TASK_ADD_TARGET), "Task add target device error"); ++ ASSERT_STREQ(dev_strerror(ERR_TASK_DEFERRED_REMOVE), "dm_task_deferred_remove failed"); ++ ASSERT_STREQ(dev_strerror(100), "Unknown error"); ++} +\ No newline at end of file +-- +2.42.0 + diff --git a/0046-2292-bugfix-for-run.sh-and-add-build-notify-msg-for-.patch b/0046-2292-bugfix-for-run.sh-and-add-build-notify-msg-for-.patch new file mode 100644 index 0000000..3831382 --- /dev/null +++ b/0046-2292-bugfix-for-run.sh-and-add-build-notify-msg-for-.patch @@ -0,0 +1,56 @@ +From d813e654b5b964f79857df3c9130f174443a76be Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Mon, 4 Dec 2023 09:44:42 +0000 +Subject: [PATCH 46/64] !2292 bugfix for run.sh and add build notify msg for + ENABLE_GRPC_REMOTE_CONNECT * bugfix for run.sh and add build notify msg for + ENABLE_GRPC_REMOTE_CONNECT + +--- + CI/test_cases/container_cases/run.sh | 6 +++--- + cmake/options.cmake | 1 + + 2 files changed, 4 insertions(+), 3 deletions(-) + +diff --git a/CI/test_cases/container_cases/run.sh b/CI/test_cases/container_cases/run.sh +index 8ea3e514..1bfd388b 100755 +--- a/CI/test_cases/container_cases/run.sh ++++ b/CI/test_cases/container_cases/run.sh +@@ -27,7 +27,8 @@ function do_test_t() + { + tid=`isula run --runtime $1 -tid --name hostname busybox` + chostname=`isula exec -it $tid hostname` +- fn_check_eq "$chostname" "${tid:0:12}" "default hostname is id of container" ++ clean_hostname=$(echo "$hostname" | sed 's/[\x01-\x1F\x7F]//g') ++ fn_check_eq "${clean_hostname}" "${tid:0:12}" "default hostname is not id of container" + isula exec -it hostname env | grep HOSTNAME + fn_check_eq "$?" "0" "check HOSTNAME env failed" + isula stop -t 0 $tid +@@ -149,13 +150,12 @@ function do_run_remote_test_t() + + isula run --runtime $1 -ti -H "$config" --name $containername busybox xxx + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed check invalid run ${containername} remote" && ((ret++)) +- testcontainer $containername exited ++ + isula rm -f -H "$config" $containername + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container remote" && ((ret++)) + + isula run --runtime $1 -ti -H "$config" --name $containername busybox /bin/sh -c 'echo "hello"' | grep hello + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run ${containername} remote" && ((ret++)) +- testcontainer $containername exited + + isula rm -f -H "$config" $containername + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container remote" && ((ret++)) +diff --git a/cmake/options.cmake b/cmake/options.cmake +index aeb24662..bf7db93a 100644 +--- a/cmake/options.cmake ++++ b/cmake/options.cmake +@@ -110,6 +110,7 @@ option(ENABLE_GRPC_REMOTE_CONNECT "enable gRPC remote connect" OFF) + if (ENABLE_GRPC_REMOTE_CONNECT STREQUAL "ON") + add_definitions(-DENABLE_GRPC_REMOTE_CONNECT=1) + set(ENABLE_GRPC_REMOTE_CONNECT 1) ++ message("${Green}-- enable gRPC remote connect${ColourReset}") + endif() + + option(ENABLE_SHIM_V2 "enable shim v2 runtime" OFF) +-- +2.42.0 + diff --git a/0047-2295-keep-the-service-status-unchanged-after-iSulad-.patch b/0047-2295-keep-the-service-status-unchanged-after-iSulad-.patch new file mode 100644 index 0000000..661dc88 --- /dev/null +++ b/0047-2295-keep-the-service-status-unchanged-after-iSulad-.patch @@ -0,0 +1,251 @@ +From 98825c56135aeeb02f50a5eec5896d39d3ea649f Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Wed, 6 Dec 2023 01:56:52 +0000 +Subject: [PATCH 47/64] !2295 keep the service status unchanged after iSulad + service upgrade * keep the service status unchanged after iSulad service + upgrade and + +--- + iSulad.spec | 121 +++++++++++++++++++++++++++------------------------- + 1 file changed, 62 insertions(+), 59 deletions(-) + +diff --git a/iSulad.spec b/iSulad.spec +index 6be2067d..0efbf043 100644 +--- a/iSulad.spec ++++ b/iSulad.spec +@@ -1,24 +1,23 @@ + %global _version 2.1.4 +-%global _release 1 ++%global _release 2 + %global is_systemd 1 + %global enable_criv1 1 + %global enable_shimv2 1 +-%global enable_embedded 1 ++%global is_embedded 1 ++%global cpp_std 17 + + Name: iSulad + Version: %{_version} + Release: %{_release} + Summary: Lightweight Container Runtime Daemon + License: Mulan PSL v2 +-URL: isulad +-Source: iSulad-2.1.tar.gz ++URL: https://gitee.com/openeuler/iSulad ++Source: https://gitee.com/openeuler/iSulad/repository/archive/v%{version}.tar.gz + BuildRoot: {_tmppath}/iSulad-%{version} +-ExclusiveArch: x86_64 aarch64 + + %ifarch x86_64 aarch64 + Provides: libhttpclient.so()(64bit) + Provides: libisula_client.so()(64bit) +-Provides: libisulad_img.so()(64bit) + Provides: libisulad_tools.so()(64bit) + %endif + +@@ -33,40 +32,56 @@ Requires(preun): chkconfig + Requires(preun): initscripts + %endif + +-%if 0%{?enable_embedded} ++%if 0%{?is_embedded} + BuildRequires: sqlite-devel +-Requires: sqlite ++Requires: sqlite + %endif + +-%if 0%{?enable_shimv2} +-BuildRequires: lib-shim-v2-devel +-Requires: lib-shim-v2 ++%if %{defined openeuler} ++BuildRequires: gtest-devel gmock-devel + %endif + +-BuildRequires: cmake gcc-c++ lxc-devel lcr-devel yajl-devel libisula-devel +-BuildRequires: grpc-plugins grpc-devel protobuf-devel +-BuildRequires: libcurl-devel libarchive-devel device-mapper-devel ++%define lcrver_lower 2.1.3-0 ++%define lcrver_upper 2.1.4-0 ++ ++BuildRequires: libisula-devel > %{lcrver_lower} libisula-devel < %{lcrver_upper} ++BuildRequires: cmake gcc-c++ yajl-devel ++BuildRequires: grpc grpc-plugins grpc-devel protobuf-devel ++BuildRequires: libcurl libcurl-devel libarchive-devel device-mapper-devel + BuildRequires: http-parser-devel +-BuildRequires: libselinux-devel libwebsockets-devel ++BuildRequires: libseccomp-devel libcap-devel libselinux-devel libwebsockets libwebsockets-devel + BuildRequires: systemd-devel git ++BuildRequires: libevhtp-devel libevent-devel ++%if 0%{?enable_shimv2} ++BuildRequires: lib-shim-v2 lib-shim-v2-devel ++%endif ++ + +-Requires: libisula lxc +-Requires: grpc libcurl http-parser +-Requires: libselinux libwebsockets libarchive device-mapper ++Requires: libisula > %{lcrver_lower} libisula < %{lcrver_upper} ++Requires: grpc protobuf ++Requires: libcurl ++Requires: http-parser libseccomp ++Requires: libcap libselinux libwebsockets libarchive device-mapper + Requires: systemd ++Requires: (docker-runc or runc) ++BuildRequires: libevhtp libevent ++%if 0%{?enable_shimv2} ++Requires: lib-shim-v2 ++%endif + + %description + This is a umbrella project for gRPC-services based Lightweight Container + Runtime Daemon, written by C. + + %prep +-%autosetup -c -n iSulad-%{version} ++%autosetup -n iSulad-v%{_version} -Sgit -p1 + + %build + mkdir -p build + cd build + %cmake \ + -DDEBUG=ON \ ++ -DCMAKE_SKIP_RPATH=TRUE \ + -DLIB_INSTALL_DIR=%{_libdir} \ + -DCMAKE_INSTALL_PREFIX=/usr \ + %if 0%{?enable_criv1} +@@ -76,36 +91,47 @@ cd build + %if 0%{?enable_shimv2} + -DENABLE_SHIM_V2=ON \ + %endif ++%if %{defined openeuler} ++ -DENABLE_UT=OFF \ ++%endif ++ -DENABLE_GRPC_REMOTE_CONNECT=OFF \ ++ -DENABLE_GRPC=ON \ ++ -DCMAKE_CXX_STANDARD=%{cpp_std} \ + ../ ++ ++sed -i "10 a\# undef linux" grpc/src/api/services/cri/v1alpha/api.pb.h ++%if 0%{?enable_criv1} ++sed -i "10 a\# undef linux" grpc/src/api/services/cri/v1/api_v1.pb.h ++%endif ++ + %make_build + ++%check ++%if %{defined openeuler} ++cd build ++# registry_images_ut and volume_ut must run with root user ++ctest -E "registry_images_ut|volume_ut" ++%endif ++ + %install + rm -rf %{buildroot} + cd build + install -d $RPM_BUILD_ROOT/%{_libdir} +-install -m 0644 ./src/libisula_client.so %{buildroot}/%{_libdir}/libisula_client.so +-install -m 0644 ./src/utils/http/libhttpclient.so %{buildroot}/%{_libdir}/libhttpclient.so +-chrpath -d ./src/libisulad_tools.so +-install -m 0644 ./src/libisulad_tools.so %{buildroot}/%{_libdir}/libisulad_tools.so +-chrpath -d ./src/daemon/modules/image/libisulad_img.so +-install -m 0644 ./src/daemon/modules/image/libisulad_img.so %{buildroot}/%{_libdir}/libisulad_img.so +-chmod +x %{buildroot}/%{_libdir}/libisula_client.so +-chmod +x %{buildroot}/%{_libdir}/libhttpclient.so +-chmod +x %{buildroot}/%{_libdir}/libisulad_img.so ++install -m 0755 ./src/libisula_client.so %{buildroot}/%{_libdir}/libisula_client.so ++install -m 0755 ./src/utils/http/libhttpclient.so %{buildroot}/%{_libdir}/libhttpclient.so ++install -m 0755 ./src/libisulad_tools.so %{buildroot}/%{_libdir}/libisulad_tools.so + + install -d $RPM_BUILD_ROOT/%{_libdir}/pkgconfig + install -m 0640 ./conf/isulad.pc %{buildroot}/%{_libdir}/pkgconfig/isulad.pc + + install -d $RPM_BUILD_ROOT/%{_bindir} ++ + install -m 0755 ./src/isula %{buildroot}/%{_bindir}/isula + install -m 0755 ./src/isulad-shim %{buildroot}/%{_bindir}/isulad-shim +-install -m 0755 ./src/isulad %{buildroot}/%{_bindir}/isulad +-chrpath -d ./src/isula +-chrpath -d ./src/isulad-shim +-chrpath -d ./src/isulad ++ ++install -m 0755 ./src/isulad %{buildroot}/%{_bindir}/isulad + + install -d $RPM_BUILD_ROOT/%{_includedir}/isulad +-install -m 0644 ../src/daemon/modules/api/image_api.h %{buildroot}/%{_includedir}/isulad/image_api.h + + install -d $RPM_BUILD_ROOT/%{_sysconfdir}/isulad + install -m 0640 ../src/contrib/config/daemon.json %{buildroot}/%{_sysconfdir}/isulad/daemon.json +@@ -134,8 +160,6 @@ install -d $RPM_BUILD_ROOT/%{_initddir} + install -p -m 0640 ../src/contrib/init/isulad.init $RPM_BUILD_ROOT/%{_initddir}/isulad.init + %endif + +-install -d $RPM_BUILD_ROOT/usr/share/bash-completion/completions +-install -p -m 0644 ../src/contrib/completion/isula $RPM_BUILD_ROOT/usr/share/bash-completion/completions/isula + %clean + rm -rf %{buildroot} + +@@ -143,19 +167,17 @@ rm -rf %{buildroot} + # support update from lcrd to isulad, will remove in next version + if [ "$1" = "2" ]; then + %if 0%{?is_systemd} +-systemctl stop lcrd +-systemctl disable lcrd ++systemctl stop lcrd &>/dev/null ++systemctl disable lcrd &>/dev/null + if [ -e %{_sysconfdir}/isulad/daemon.json ];then + sed -i 's#/etc/default/lcrd/hooks#/etc/default/isulad/hooks#g' %{_sysconfdir}/isulad/daemon.json + fi + %else +-/sbin/chkconfig --del lcrd ++/sbin/chkconfig --del lcrd &>/dev/null + %endif + fi + + %post +-source /usr/share/bash-completion/completions/isula +- + if ! getent group isula > /dev/null; then + groupadd --system isula + fi +@@ -174,12 +196,6 @@ if [ -e %{_unitdir}/lcrd.service.rpmsave ]; then + mv %{_unitdir}/lcrd.service.rpmsave %{_unitdir}/isulad.service + sed -i 's/lcrd/isulad/g' %{_unitdir}/isulad.service + fi +-systemctl status isulad | grep 'Active:' | grep 'running' +-if [ $? -eq 0 ]; then +- systemctl restart isulad +-else +- systemctl start isulad +-fi + %else + /sbin/service isulad status | grep 'Active:' | grep 'running' + if [ $? -eq 0 ]; then +@@ -226,7 +242,6 @@ fi + %{_initddir}/isulad.init + %attr(0640,root,root) %{_initddir}/isulad.init + %endif +-%{_includedir}/isulad/* + %attr(0755,root,root) %{_libdir}/pkgconfig + %attr(0640,root,root) %{_libdir}/pkgconfig/isulad.pc + %defattr(0755,root,root,0755) +@@ -242,17 +257,5 @@ fi + %else + %config(noreplace,missingok) %{_initddir}/isulad.init + %endif +-/usr/share/bash-completion/completions/isula + + %changelog +-* Tue Sep 10 2020 openEuler Buildteam <buildteam@openeuler.org> - 2.0.5-20200910.140350.git72990229 +-- Type:enhancement +-- ID:NA +-- SUG:NA +-- DESC: add chrpath +- +-* Mon Aug 03 2020 openEuler Buildteam <buildteam@openeuler.org> - 2.0.3-20200803.130854.git0c7dc28a +-- Type:enhancement +-- ID:NA +-- SUG:NA +-- DESC: add debug packages +-- +2.42.0 + diff --git a/0048-modify-attach-socket-name.patch b/0048-modify-attach-socket-name.patch new file mode 100644 index 0000000..e1eb159 --- /dev/null +++ b/0048-modify-attach-socket-name.patch @@ -0,0 +1,40 @@ +From c01b761e14e6b4ea6745688e47b255f17ba26055 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Wed, 6 Dec 2023 15:15:32 +0800 +Subject: [PATCH 48/64] modify attach socket name + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/cmd/isulad-shim/common.h | 2 +- + src/daemon/modules/runtime/isula/isula_rt_ops.c | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/src/cmd/isulad-shim/common.h b/src/cmd/isulad-shim/common.h +index 2020a799..c4f86d24 100644 +--- a/src/cmd/isulad-shim/common.h ++++ b/src/cmd/isulad-shim/common.h +@@ -60,7 +60,7 @@ extern "C" { + #define CONTAINER_ACTION_REBOOT 129 + #define CONTAINER_ACTION_SHUTDOWN 130 + +-#define ATTACH_SOCKET "attach_socket.sock" ++#define ATTACH_SOCKET "attach.sock" + #define ATTACH_LOG_NAME "attach-log.json" + #define ATTACH_DETACH_MSG "read escape sequence" + #define MAX_ATTACH_NUM 16 +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 3950ff4a..fbb779f7 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -57,7 +57,7 @@ + + #define SHIM_BINARY "isulad-shim" + #define RESIZE_FIFO_NAME "resize_fifo" +-#define ATTACH_SOCKET "attach_socket.sock" ++#define ATTACH_SOCKET "attach.sock" + #define SHIM_LOG_SIZE ((BUFSIZ - 100) / 2) + #define RESIZE_DATA_SIZE 100 + #define PID_WAIT_TIME 120 +-- +2.42.0 + diff --git a/0049-2298-bugfix-for-hook_ignore_poststart_error-run-in-o.patch b/0049-2298-bugfix-for-hook_ignore_poststart_error-run-in-o.patch new file mode 100644 index 0000000..4c90ed4 --- /dev/null +++ b/0049-2298-bugfix-for-hook_ignore_poststart_error-run-in-o.patch @@ -0,0 +1,110 @@ +From 94122c5752936b4f5db14521cdd0f39a3dec6851 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Thu, 7 Dec 2023 03:32:15 +0000 +Subject: [PATCH 49/64] !2298 bugfix for hook_ignore_poststart_error run in oci + runtime * bugfix for hook_ignore_poststart_error run in oci runtime + +--- + .../hook_ignore_poststart_error.sh | 70 +++++++++++++------ + 1 file changed, 50 insertions(+), 20 deletions(-) + +diff --git a/CI/test_cases/container_cases/hook_ignore_poststart_error.sh b/CI/test_cases/container_cases/hook_ignore_poststart_error.sh +index 8c636f7e..38b6f021 100755 +--- a/CI/test_cases/container_cases/hook_ignore_poststart_error.sh ++++ b/CI/test_cases/container_cases/hook_ignore_poststart_error.sh +@@ -24,6 +24,48 @@ source ../helpers.sh + + test_data_path=$(realpath $curr_path/test_data) + ++# $1 hook process ++# $2 container id ++# $3 expect container status ++# $4 process statement ++function test_kill_hook() ++{ ++ for a in `seq 20` ++ do ++ bpid=`ps aux | grep "$1" | grep -v grep | awk '{print $2}'` ++ if [ "x" != "x$bpid" ];then ++ kill -9 $bpid ++ break ++ else ++ sleep .5 ++ continue ++ fi ++ done ++ ++ if [ "x" != "x$4" ];then ++ for a in `seq 20` ++ do ++ bpid=`ps aux | grep "$4" | grep -v grep | awk '{print $2}'` ++ if [ "x" != "x$bpid" ];then ++ kill -9 $bpid ++ break ++ else ++ sleep .5 ++ continue ++ fi ++ done ++ fi ++ ++ status=`isula inspect -f '{{json .State.Status}}' $2` ++ if [ "$status" == "$3" ];then ++ echo "get right status" ++ return 0 ++ else ++ echo "expect $2 $3, but get $status" ++ return 1 ++ fi ++} ++ + function test_hook_ignore_poststart_error_spec() + { + local ret=0 +@@ -42,27 +84,15 @@ function test_hook_ignore_poststart_error_spec() + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++)) + + isula run -n $CONT -itd --runtime $runtime --hook-spec ${test_data_path}/oci_hook_poststart_check.json ${image} & +- +- for a in `seq 20` +- do +- bpid=`ps aux | grep "poststart.sh" | grep -v grep | awk '{print $2}'` +- if [ "x" != "x$bpid" ];then +- kill -9 $bpid +- break +- else +- sleep .5 +- continue +- fi +- done +- +- status=`isula inspect -f '{{json .State.Status}}' $CONT` +- if [ "$status" == "\"running\"" ];then +- echo "get right status" ++ ++ # when runc container run poststart hook, the process structure is different from lxc ++ if [ $runtime == "lcr" ]; then ++ test_kill_hook "poststart.sh" $CONT \"running\" + else +- echo "expect $CONT running, but get $status" +- ret++ ++ test_kill_hook "poststart.sh" $CONT \"exited\" "sleep 300" + fi +- ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to test kill hook: ${image}" && ((ret++)) ++ + isula stop -t 0 ${CONT} + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to stop ${CONT}" && ((ret++)) + +@@ -77,7 +107,7 @@ declare -i ans=0 + + for element in ${RUNTIME_LIST[@]}; + do +- test_hook_ignore_poststart_error_spec $1 || ((ans++)) ++ test_hook_ignore_poststart_error_spec $element || ((ans++)) + done + + show_result ${ans} "${curr_path}/${0}" +-- +2.42.0 + diff --git a/0050-2304-remove-build-and-test-in-coverage.patch b/0050-2304-remove-build-and-test-in-coverage.patch new file mode 100644 index 0000000..8fccf01 --- /dev/null +++ b/0050-2304-remove-build-and-test-in-coverage.patch @@ -0,0 +1,28 @@ +From c2e9919ec8612d6e811644ec8aacf53cec0c4f20 Mon Sep 17 00:00:00 2001 +From: jake <jikai11@huawei.com> +Date: Tue, 12 Dec 2023 08:55:30 +0000 +Subject: [PATCH 50/64] !2304 remove build and test in coverage * remove build + and test in coverage + +--- + CI/generate_gcov.sh | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/CI/generate_gcov.sh b/CI/generate_gcov.sh +index 153c9f5a..76bf382b 100755 +--- a/CI/generate_gcov.sh ++++ b/CI/generate_gcov.sh +@@ -31,8 +31,8 @@ ctest + lcov --directory . --capture --output-file coverage.info --rc lcov_branch_coverage=1 + # Remove std/build files + lcov --remove coverage.info '/usr/*' -o coverage.info --rc lcov_branch_coverage=1 +-lcov --remove coverage.info 'build/*' -o coverage.info --rc lcov_branch_coverage=1 +-lcov --remove coverage.info 'test/*' -o coverage.info --rc lcov_branch_coverage=1 ++lcov --remove coverage.info "$ISULAD_SRC_PATH/build/*" -o coverage.info --rc lcov_branch_coverage=1 ++lcov --remove coverage.info "$ISULAD_SRC_PATH/test/*" -o coverage.info --rc lcov_branch_coverage=1 + + # Generate html + genhtml --ignore-errors source -o $GCOV_RESULT_PATH/coverage coverage.info --branch-coverage --rc lcov_branch_coverage=1 +-- +2.42.0 + diff --git a/0051-2303-use-a-timeout-epoll-loop-to-ensure-complete-dat.patch b/0051-2303-use-a-timeout-epoll-loop-to-ensure-complete-dat.patch new file mode 100644 index 0000000..87e528a --- /dev/null +++ b/0051-2303-use-a-timeout-epoll-loop-to-ensure-complete-dat.patch @@ -0,0 +1,197 @@ +From 7d1b8d25468528a59318430d50d839032f2c1a07 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Tue, 12 Dec 2023 12:26:32 +0000 +Subject: [PATCH 51/64] !2303 use a timeout epoll loop to ensure complete data + reception * use a timeout epoll loop to ensure complete data reception + +--- + src/cmd/isulad-shim/process.c | 105 +++++++++------------------------- + src/cmd/isulad-shim/process.h | 1 - + 2 files changed, 26 insertions(+), 80 deletions(-) + +diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c +index e8cb9b32..97524f1a 100644 +--- a/src/cmd/isulad-shim/process.c ++++ b/src/cmd/isulad-shim/process.c +@@ -169,6 +169,7 @@ static int get_exec_winsize(const char *buf, struct winsize *wsize) + + static int sync_exit_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t *descr) + { ++ isula_epoll_remove_handler(descr, fd); + return EPOLL_LOOP_HANDLE_CLOSE; + } + +@@ -364,23 +365,14 @@ static int stdout_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t + int r_count = 0; + int w_count = 0; + +- if (events & EPOLLHUP) { +- return EPOLL_LOOP_HANDLE_CLOSE; +- } +- +- if (!(events & EPOLLIN)) { +- return EPOLL_LOOP_HANDLE_CONTINUE; +- } +- + (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF); + +- if (p->block_read) { +- r_count = isula_file_read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF); +- } else { +- r_count = read(fd, p->buf, DEFAULT_IO_COPY_BUF); +- } +- if (r_count <= 0) { +- return EPOLL_LOOP_HANDLE_CLOSE; ++ r_count = isula_file_read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF); ++ if (r_count <= 0 ) { ++ isula_epoll_remove_handler(descr, fd); ++ // fd cannot be closed here, which will cause the container process to exit abnormally ++ // due to terminal fd receiving the sighup signal. ++ return EPOLL_LOOP_HANDLE_CONTINUE; + } + + shim_write_container_log_file(p->terminal, STDID_OUT, p->buf, r_count); +@@ -419,23 +411,14 @@ static int stderr_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t + int r_count = 0; + int w_count = 0; + +- if (events & EPOLLHUP) { +- return EPOLL_LOOP_HANDLE_CLOSE; +- } +- +- if (!(events & EPOLLIN)) { +- return EPOLL_LOOP_HANDLE_CONTINUE; +- } +- + (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF); + +- if (p->block_read) { +- r_count = isula_file_read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF); +- } else { +- r_count = read(fd, p->buf, DEFAULT_IO_COPY_BUF); +- } +- if (r_count <= 0) { +- return EPOLL_LOOP_HANDLE_CLOSE; ++ r_count = isula_file_read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF); ++ if (r_count <= 0 ) { ++ isula_epoll_remove_handler(descr, fd); ++ // fd cannot be closed here, which will cause the container process to exit abnormally ++ // due to terminal fd receiving the sighup signal. ++ return EPOLL_LOOP_HANDLE_CONTINUE; + } + + shim_write_container_log_file(p->terminal, STDID_ERR, p->buf, r_count); +@@ -474,18 +457,11 @@ static int resize_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t + int r_count = 0; + int resize_fd = -1; + +- if (events & EPOLLHUP) { +- return EPOLL_LOOP_HANDLE_CLOSE; +- } +- +- if (!(events & EPOLLIN)) { +- return EPOLL_LOOP_HANDLE_CONTINUE; +- } +- + (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF); + r_count = isula_file_read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF); + if (r_count <= 0) { +- return EPOLL_LOOP_HANDLE_CLOSE; ++ close(fd); ++ return EPOLL_LOOP_HANDLE_CONTINUE; + } + + resize_fd = p->recv_fd; +@@ -915,8 +891,6 @@ static int open_generic_io(process_t *p, isula_epoll_descr_t *descr) + static void *io_epoll_loop(void *data) + { + int ret = 0; +- int fd_out = -1; +- int fd_err = -1; + process_t *p = (process_t *)data; + isula_epoll_descr_t descr; + +@@ -953,49 +927,23 @@ static void *io_epoll_loop(void *data) + + (void)sem_post(&p->sem_mainloop); + ++ // th frist epoll_loop will exit in the following scenarios: ++ // 1. Receive sync fd event ++ // 2. stdin fd receive EPOLLHUP event ++ // 3. stdin fd read failed + ret = isula_epoll_loop(&descr, -1); + if (ret != 0) { + write_message(ERR_MSG, "epoll loop failed"); + exit(EXIT_FAILURE); + } + +- // in order to avoid data loss, set fd non-block and read it +- p->block_read = false; +- if (p->state->terminal) { +- fd_out = p->recv_fd; +- } else { +- fd_out = p->shim_io->out; +- fd_err = p->shim_io->err; +- } +- +- if (fd_out > 0) { +- ret = isula_set_non_block(fd_out); +- if (ret != SHIM_OK) { +- write_message(ERR_MSG, "set fd %d non_block failed:%d", fd_out, SHIM_SYS_ERR(errno)); +- exit(EXIT_FAILURE); +- } +- +- for (;;) { +- ret = stdout_cb(fd_out, EPOLLIN, p, &descr); +- if (ret == EPOLL_LOOP_HANDLE_CLOSE) { +- break; +- } +- } +- } +- +- if (fd_err > 0) { +- ret = isula_set_non_block(fd_err); +- if (ret != SHIM_OK) { +- write_message(ERR_MSG, "set fd %d non_block failed:%d", fd_err, SHIM_SYS_ERR(errno)); +- exit(EXIT_FAILURE); +- } +- +- for (;;) { +- ret = stderr_cb(fd_err, EPOLLIN, p, &descr); +- if (ret == EPOLL_LOOP_HANDLE_CLOSE) { +- break; +- } +- } ++ // use a timeout epoll loop to ensure complete data reception ++ // th second epoll_loop will exit in the following scenarios: ++ // 1. both stdout fd and stderr fd failed to read ++ // 2. no event received within 100 milliseconds ++ ret = isula_epoll_loop(&descr, 100); ++ if (ret != 0) { ++ write_message(ERR_MSG, "Repeat the epoll loop to ensure that all data is transferred"); + } + + return NULL; +@@ -1220,7 +1168,6 @@ process_t *new_process(char *id, char *bundle, char *runtime) + p->bundle = bundle; + p->runtime = runtime; + p->state = p_state; +- p->block_read = true; + p->console_sock_path = NULL; + p->exit_fd = -1; + p->io_loop_fd = -1; +diff --git a/src/cmd/isulad-shim/process.h b/src/cmd/isulad-shim/process.h +index 5607316c..32ba7366 100644 +--- a/src/cmd/isulad-shim/process.h ++++ b/src/cmd/isulad-shim/process.h +@@ -55,7 +55,6 @@ typedef struct process { + int sync_fd; + int listen_fd; + int recv_fd; +- bool block_read; + log_terminal *terminal; + stdio_t *stdio; // shim to on runtime side, in:r out/err: w + stdio_t *shim_io; // shim io on isulad side, in: w out/err: r +-- +2.42.0 + diff --git a/0052-modify-the-default-value-of-ISULAD_TMPDIR-to-var-lib.patch b/0052-modify-the-default-value-of-ISULAD_TMPDIR-to-var-lib.patch new file mode 100644 index 0000000..cf0c729 --- /dev/null +++ b/0052-modify-the-default-value-of-ISULAD_TMPDIR-to-var-lib.patch @@ -0,0 +1,170 @@ +From 06d42781cbfc3d9baa7155b480e22b9f4164ab91 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Tue, 12 Dec 2023 20:24:57 +0800 +Subject: [PATCH 52/64] modify the default value of ISULAD_TMPDIR to + /var/lib/isulad + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/cmd/isulad/main.c | 13 +++++++------ + src/common/constants.h | 2 ++ + src/contrib/config/iSulad.sysconfig | 4 ++-- + .../modules/container/leftover_cleanup/cleanup.c | 6 +++--- + src/daemon/modules/image/oci/utils_images.c | 2 +- + src/utils/cutils/utils_verify.c | 5 +++++ + src/utils/cutils/utils_verify.h | 2 ++ + src/utils/tar/util_archive.c | 9 +++++---- + 8 files changed, 27 insertions(+), 16 deletions(-) + +diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c +index 95454e2a..d33e4004 100644 +--- a/src/cmd/isulad/main.c ++++ b/src/cmd/isulad/main.c +@@ -1295,8 +1295,8 @@ static int ensure_isulad_tmpdir_security() + char *isulad_tmp_dir = NULL; + + isulad_tmp_dir = getenv("ISULAD_TMPDIR"); +- if (!util_valid_str(isulad_tmp_dir)) { +- isulad_tmp_dir = "/tmp"; ++ if (!util_valid_isulad_tmpdir(isulad_tmp_dir)) { ++ isulad_tmp_dir = DEFAULT_ISULAD_TMPDIR; + } + + if (do_ensure_isulad_tmpdir_security(isulad_tmp_dir) != 0) { +@@ -1304,14 +1304,15 @@ static int ensure_isulad_tmpdir_security() + return -1; + } + +- if (strcmp(isulad_tmp_dir, "/tmp") == 0) { ++ if (strcmp(isulad_tmp_dir, DEFAULT_ISULAD_TMPDIR) == 0) { + return 0; + } + + // No matter whether ISULAD_TMPDIR is set or not, +- // ensure the "/tmp" directory is a safe directory +- if (do_ensure_isulad_tmpdir_security("/tmp") != 0) { +- WARN("Failed to ensure the /tmp directory is a safe directory"); ++ // ensure the DEFAULT_ISULAD_TMPDIR directory is a safe directory ++ // TODO: if isula is no longer tarred in the future, we can delete it. ++ if (do_ensure_isulad_tmpdir_security(DEFAULT_ISULAD_TMPDIR) != 0) { ++ WARN("Failed to ensure the default ISULAD_TMPDIR : %s directory is a safe directory", DEFAULT_ISULAD_TMPDIR); + } + + return 0; +diff --git a/src/common/constants.h b/src/common/constants.h +index 5f12ae25..27d4956e 100644 +--- a/src/common/constants.h ++++ b/src/common/constants.h +@@ -129,6 +129,8 @@ extern "C" { + + #define OCI_IMAGE_GRAPH_ROOTPATH_NAME "storage" + ++#define DEFAULT_ISULAD_TMPDIR "/var/lib/isulad" ++ + #ifdef ENABLE_GRPC_REMOTE_CONNECT + #define DEFAULT_TCP_HOST "tcp://localhost:2375" + #define DEFAULT_TLS_HOST "tcp://localhost:2376" +diff --git a/src/contrib/config/iSulad.sysconfig b/src/contrib/config/iSulad.sysconfig +index 43ba7cbd..25099480 100644 +--- a/src/contrib/config/iSulad.sysconfig ++++ b/src/contrib/config/iSulad.sysconfig +@@ -22,5 +22,5 @@ + #SYSMONITOR_OPTIONS='-H tcp://127.0.0.1:2375 --tlsverify --tlscacert=/root/.iSulad/ca.pem --tlscert=/root/.iSulad/cert.pem --tlskey=/root/.iSulad/key.pem' + + # Location used for temporary files, such as those created by isula load and pull operations. +-# Default is /var/tmp. Can be overridden by setting the following env variable. +-# ISULAD_TMPDIR=/var/tmp ++# Default is /var/lib/isulad. Can be overridden by setting the following env variable. ++# ISULAD_TMPDIR=/var/lib/isulad +diff --git a/src/daemon/modules/container/leftover_cleanup/cleanup.c b/src/daemon/modules/container/leftover_cleanup/cleanup.c +index 9a38ffc2..af5f0eee 100644 +--- a/src/daemon/modules/container/leftover_cleanup/cleanup.c ++++ b/src/daemon/modules/container/leftover_cleanup/cleanup.c +@@ -203,12 +203,12 @@ void do_isulad_tmpdir_cleaner(void) + char *isula_tmp_dir = NULL; + + isula_tmp_dir = getenv("ISULAD_TMPDIR"); +- if (util_valid_str(isula_tmp_dir)) { ++ if (util_valid_isulad_tmpdir(isula_tmp_dir)) { + cleanup_path(isula_tmp_dir); + } + // No matter whether ISULAD_TMPDIR is set or not, +- // clean up the "/tmp" directory to prevent the mount point from remaining +- cleanup_path("/tmp"); ++ // clean up the DEFAULT_ISULAD_TMPDIR directory to prevent the mount point from remaining ++ cleanup_path(DEFAULT_ISULAD_TMPDIR); + + return; + } +diff --git a/src/daemon/modules/image/oci/utils_images.c b/src/daemon/modules/image/oci/utils_images.c +index f92ee59a..d94388bd 100644 +--- a/src/daemon/modules/image/oci/utils_images.c ++++ b/src/daemon/modules/image/oci/utils_images.c +@@ -595,7 +595,7 @@ char *oci_get_isulad_tmpdir(const char *root_dir) + } + + env_dir = getenv("ISULAD_TMPDIR"); +- if (util_valid_str(env_dir)) { ++ if (util_valid_isulad_tmpdir(env_dir)) { + isulad_tmpdir = util_path_join(env_dir, "isulad_tmpdir"); + } else { + isulad_tmpdir = util_path_join(root_dir, "isulad_tmpdir"); +diff --git a/src/utils/cutils/utils_verify.c b/src/utils/cutils/utils_verify.c +index f4ce3199..7f2db48b 100644 +--- a/src/utils/cutils/utils_verify.c ++++ b/src/utils/cutils/utils_verify.c +@@ -744,6 +744,11 @@ bool util_valid_volume_name(const char *name) + return util_reg_match(patten, name) == 0; + } + ++bool util_valid_isulad_tmpdir(const char *dir) ++{ ++ return util_valid_str(dir) && strcmp(dir, "/tmp") != 0; ++} ++ + #ifdef ENABLE_IMAGE_SEARCH + bool util_valid_search_name(const char *name) + { +diff --git a/src/utils/cutils/utils_verify.h b/src/utils/cutils/utils_verify.h +index 54d1ce71..bafd2a82 100644 +--- a/src/utils/cutils/utils_verify.h ++++ b/src/utils/cutils/utils_verify.h +@@ -124,6 +124,8 @@ bool util_valid_sysctl(const char *sysctl_key); + + bool util_valid_volume_name(const char *name); + ++bool util_valid_isulad_tmpdir(const char *dir); ++ + #ifdef ENABLE_IMAGE_SEARCH + bool util_valid_search_name(const char *name); + #endif +diff --git a/src/utils/tar/util_archive.c b/src/utils/tar/util_archive.c +index 82e940a5..e8fad391 100644 +--- a/src/utils/tar/util_archive.c ++++ b/src/utils/tar/util_archive.c +@@ -134,7 +134,7 @@ static void do_disable_unneccessary_caps() + // Add flock when bind mount and make it private. + // Because bind mount usually makes safedir shared mount point, + // and sometimes it will cause "mount point explosion". +-// E.g. concurrently execute isula cp /tmp/<XXX-File> <CONTAINER-ID>:<CONTAINER-PAT> ++// E.g. concurrently execute isula cp DEFAULT_ISULAD_TMPDIR/<XXX-File> <CONTAINER-ID>:<CONTAINER-PAT> + static int bind_mount_with_flock(const char *flock_path, const char *dstdir, const char *tmp_dir) + { + __isula_auto_close int fd = -1; +@@ -192,9 +192,10 @@ static int make_safedir_is_noexec(const char *flock_path, const char *dstdir, ch + int nret; + + isulad_tmpdir_env = getenv("ISULAD_TMPDIR"); +- if (!util_valid_str(isulad_tmpdir_env)) { +- // if not setted isulad tmpdir, just use /tmp +- isulad_tmpdir_env = "/tmp"; ++ if (!util_valid_isulad_tmpdir(isulad_tmpdir_env)) { ++ INFO("if not setted isulad tmpdir or setted unvalid dir, use DEFAULT_ISULAD_TMPDIR"); ++ // if not setted isulad tmpdir, just use DEFAULT_ISULAD_TMPDIR ++ isulad_tmpdir_env = DEFAULT_ISULAD_TMPDIR; + } + + nret = snprintf(isula_tmpdir, PATH_MAX, "%s/isulad_tmpdir", isulad_tmpdir_env); +-- +2.42.0 + diff --git a/0053-prevent-the-parent-dir-from-being-bind-mounted-to-th.patch b/0053-prevent-the-parent-dir-from-being-bind-mounted-to-th.patch new file mode 100644 index 0000000..9876957 --- /dev/null +++ b/0053-prevent-the-parent-dir-from-being-bind-mounted-to-th.patch @@ -0,0 +1,58 @@ +From 05117ed2887ee1535978170cd06596ee015951f4 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Tue, 12 Dec 2023 20:26:30 +0800 +Subject: [PATCH 53/64] prevent the parent dir from being bind mounted to the + subdir + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/utils/tar/util_archive.c | 26 ++++++++++++++++++++++++++ + 1 file changed, 26 insertions(+) + +diff --git a/src/utils/tar/util_archive.c b/src/utils/tar/util_archive.c +index e8fad391..29c2bc03 100644 +--- a/src/utils/tar/util_archive.c ++++ b/src/utils/tar/util_archive.c +@@ -182,6 +182,26 @@ unlock_out: + return ret; + } + ++static int is_parent_directory(const char *parent_path, const char *child_path) ++{ ++ size_t parent_len = strlen(parent_path); ++ size_t child_len = strlen(child_path); ++ ++ if (parent_len == 0 || child_len == 0 || parent_len >= child_len) { ++ return -1; ++ } ++ ++ if (strncmp(parent_path, child_path, parent_len) != 0) { ++ return -1; ++ } ++ ++ if (child_path[parent_len] != '/') { ++ return -1; ++ } ++ ++ return 0; ++} ++ + static int make_safedir_is_noexec(const char *flock_path, const char *dstdir, char **safe_dir) + { + struct stat buf; +@@ -235,6 +255,12 @@ static int make_safedir_is_noexec(const char *flock_path, const char *dstdir, ch + return -1; + } + ++ // prevent the parent directory from being bind mounted to the subdirectory ++ if (is_parent_directory(dstdir, tmp_dir) == 0) { ++ ERROR("Cannot bind mount the parent directory: %s to its subdirectory: %s", dstdir, tmp_dir); ++ return -1; ++ } ++ + if (bind_mount_with_flock(flock_path, dstdir, tmp_dir) != 0) { + ERROR("Failed to bind mount from %s to %s with flock", dstdir, tmp_dir); + if (util_path_remove(tmp_dir) != 0) { +-- +2.42.0 + diff --git a/0054-2308-Remove-unused-header-file.patch b/0054-2308-Remove-unused-header-file.patch new file mode 100644 index 0000000..c783229 --- /dev/null +++ b/0054-2308-Remove-unused-header-file.patch @@ -0,0 +1,25 @@ +From 93071602df77cc3b5508266b181f1ace947bd3be Mon Sep 17 00:00:00 2001 +From: xuxuepeng <xuxuepeng1@huawei.com> +Date: Wed, 13 Dec 2023 02:34:20 +0000 +Subject: [PATCH 54/64] !2308 Remove unused header file * Fix compiling failure + in image oci UT + +--- + src/daemon/config/isulad_config.c | 1 - + 1 file changed, 1 deletion(-) + +diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c +index 1af47127..51758adb 100644 +--- a/src/daemon/config/isulad_config.c ++++ b/src/daemon/config/isulad_config.c +@@ -32,7 +32,6 @@ + + #include "constants.h" + #include "utils.h" +-#include "sysinfo.h" + #include "err_msg.h" + #include "daemon_arguments.h" + #include "utils_array.h" +-- +2.42.0 + diff --git a/0055-verify-the-mount-dir-first-and-then-create-tmpdir.patch b/0055-verify-the-mount-dir-first-and-then-create-tmpdir.patch new file mode 100644 index 0000000..90caa48 --- /dev/null +++ b/0055-verify-the-mount-dir-first-and-then-create-tmpdir.patch @@ -0,0 +1,43 @@ +From 3d38013418d0c5304dfbafcb0b2a5b4062964c53 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Wed, 13 Dec 2023 15:13:12 +0800 +Subject: [PATCH 55/64] verify the mount dir first and then create tmpdir + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/utils/tar/util_archive.c | 12 ++++++------ + 1 file changed, 6 insertions(+), 6 deletions(-) + +diff --git a/src/utils/tar/util_archive.c b/src/utils/tar/util_archive.c +index 29c2bc03..655b3516 100644 +--- a/src/utils/tar/util_archive.c ++++ b/src/utils/tar/util_archive.c +@@ -235,6 +235,12 @@ static int make_safedir_is_noexec(const char *flock_path, const char *dstdir, ch + return -1; + } + ++ // prevent the parent directory from being bind mounted to the subdirectory ++ if (is_parent_directory(dstdir, tmp_dir) == 0) { ++ ERROR("Cannot bind mount the parent directory: %s to its subdirectory: %s", dstdir, tmp_dir); ++ return -1; ++ } ++ + if (stat(dstdir, &buf) < 0) { + SYSERROR("Check chroot dir failed"); + return -1; +@@ -255,12 +261,6 @@ static int make_safedir_is_noexec(const char *flock_path, const char *dstdir, ch + return -1; + } + +- // prevent the parent directory from being bind mounted to the subdirectory +- if (is_parent_directory(dstdir, tmp_dir) == 0) { +- ERROR("Cannot bind mount the parent directory: %s to its subdirectory: %s", dstdir, tmp_dir); +- return -1; +- } +- + if (bind_mount_with_flock(flock_path, dstdir, tmp_dir) != 0) { + ERROR("Failed to bind mount from %s to %s with flock", dstdir, tmp_dir); + if (util_path_remove(tmp_dir) != 0) { +-- +2.42.0 + diff --git a/0056-2300-Maintaining-a-uniform-code-style.patch b/0056-2300-Maintaining-a-uniform-code-style.patch new file mode 100644 index 0000000..f6e434c --- /dev/null +++ b/0056-2300-Maintaining-a-uniform-code-style.patch @@ -0,0 +1,26 @@ +From 2f36e5cae2414804040b6168b79011550281d8d7 Mon Sep 17 00:00:00 2001 +From: chen524 <chenkui_yewu@cmss.chinamobile.com> +Date: Wed, 13 Dec 2023 08:02:20 +0000 +Subject: [PATCH 56/64] !2300 Maintaining a uniform code style * update + src/cmd/command_parser.c. + +--- + src/cmd/command_parser.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/cmd/command_parser.c b/src/cmd/command_parser.c +index 1ad1d92b..93b19dae 100644 +--- a/src/cmd/command_parser.c ++++ b/src/cmd/command_parser.c +@@ -438,7 +438,7 @@ int command_valid_socket_append_array(command_option_t *option, const char *arg) + } + + if (util_array_append(option->data, arg) != 0) { +- ERROR("merge hosts config failed"); ++ COMMAND_ERROR("Merge hosts config failed"); + return -1; + } + len++; +-- +2.42.0 + diff --git a/0057-2312-Add-Huawei-Cloud-CodeArts-compilation-script.patch b/0057-2312-Add-Huawei-Cloud-CodeArts-compilation-script.patch new file mode 100644 index 0000000..3845146 --- /dev/null +++ b/0057-2312-Add-Huawei-Cloud-CodeArts-compilation-script.patch @@ -0,0 +1,36 @@ +From 5efff4c61ed885ce45d62e33e2e97a78519fefe8 Mon Sep 17 00:00:00 2001 +From: dreamloy <3038807110@qq.com> +Date: Wed, 13 Dec 2023 08:39:04 +0000 +Subject: [PATCH 57/64] =?UTF-8?q?!2312=20Add=20Huawei=20Cloud=20CodeArts?= + =?UTF-8?q?=20compilation=20script=20*=20=E5=B0=86codecheck=5Fcompile.sh?= + =?UTF-8?q?=20=E7=A7=BB=E5=8A=A8=E5=88=B0tools=E4=B8=8B=20*=20=E6=96=B0?= + =?UTF-8?q?=E5=A2=9E=E5=8D=8E=E4=B8=BA=E4=BA=91codeArs=E7=BC=96=E8=AF=91?= + =?UTF-8?q?=E8=84=9A=E6=9C=AC?= +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +--- + tools/codecheck_compile.sh | 9 +++++++++ + 1 file changed, 9 insertions(+) + create mode 100644 tools/codecheck_compile.sh + +diff --git a/tools/codecheck_compile.sh b/tools/codecheck_compile.sh +new file mode 100644 +index 00000000..99cadfe7 +--- /dev/null ++++ b/tools/codecheck_compile.sh +@@ -0,0 +1,9 @@ ++## 华为云codeArts执行版本检查时，规则集涉及到代码安全增强包需要编译脚本才能执行 ++BASEPATH=$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}" )" &> /dev/null && pwd ) ++ROOTDIR="$BASEPATH" ++PROGRAM=$(basename "${BASH_SOURCE[0]:-$0}") ++whoami ++ls ++cd docs/build_docs/guide/script ++chmod +x ./install_iSulad_on_Ubuntu_20_04_LTS.sh ++./install_iSulad_on_Ubuntu_20_04_LTS.sh +\ No newline at end of file +-- +2.42.0 + diff --git a/0058-bugfix-del-redundant-code.patch b/0058-bugfix-del-redundant-code.patch new file mode 100644 index 0000000..94072ce --- /dev/null +++ b/0058-bugfix-del-redundant-code.patch @@ -0,0 +1,26 @@ +From a593232e7f34de03142388fddecbea8f3b617245 Mon Sep 17 00:00:00 2001 +From: liuxu <liuxu156@huawei.com> +Date: Wed, 13 Dec 2023 17:06:37 +0800 +Subject: [PATCH 58/64] bugfix:del redundant code + +--- + src/daemon/modules/image/image.c | 3 --- + 1 file changed, 3 deletions(-) + +diff --git a/src/daemon/modules/image/image.c b/src/daemon/modules/image/image.c +index 4a1950fe..f01265bf 100644 +--- a/src/daemon/modules/image/image.c ++++ b/src/daemon/modules/image/image.c +@@ -602,9 +602,6 @@ void free_im_prepare_request(im_prepare_request *request) + free(request->mount_label); + request->mount_label = NULL; + +- free(request->mount_label); +- request->mount_label = NULL; +- + free_json_map_string_string(request->storage_opt); + request->storage_opt = NULL; + +-- +2.42.0 + diff --git a/0059-improve-code-of-pull.patch b/0059-improve-code-of-pull.patch new file mode 100644 index 0000000..e2c236e --- /dev/null +++ b/0059-improve-code-of-pull.patch @@ -0,0 +1,71 @@ +From e47abc01c8778cc07c11a331ae31ce46b6fd06a0 Mon Sep 17 00:00:00 2001 +From: haozi007 <liuhao27@huawei.com> +Date: Thu, 14 Dec 2023 10:59:34 +0800 +Subject: [PATCH 59/64] improve code of pull + +1. ignore unneccessary error log; +2. do not show progress, if stdout is not tty; + +Signed-off-by: haozi007 <liuhao27@huawei.com> +--- + src/cmd/isula/images/pull.c | 6 ++++++ + .../modules/image/oci/storage/image_store/image_type.c | 8 ++++++++ + .../layer_store/graphdriver/overlay2/driver_overlay2.c | 5 +++++ + 3 files changed, 19 insertions(+) + +diff --git a/src/cmd/isula/images/pull.c b/src/cmd/isula/images/pull.c +index 9d420778..b30cc0bd 100644 +--- a/src/cmd/isula/images/pull.c ++++ b/src/cmd/isula/images/pull.c +@@ -36,6 +36,12 @@ struct client_arguments g_cmd_pull_args = {}; + static bool is_terminal_show_supported() + { + #ifdef GRPC_CONNECTOR ++ // if stdout is not tty, just ingore progress ++ if (!isatty(STDOUT_FILENO)) { ++ WARN("Stdout is not tty device, just ignore progress."); ++ return false; ++ } ++ + // Initialize the terminfo database + setupterm(NULL, STDOUT_FILENO, (int *)0); + +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_type.c b/src/daemon/modules/image/oci/storage/image_store/image_type.c +index 50af0a69..50a81db2 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_type.c ++++ b/src/daemon/modules/image/oci/storage/image_store/image_type.c +@@ -77,6 +77,14 @@ int try_fill_image_spec(image_t *img, const char *id, const char *image_store_di + goto out; + } + ++ // for new_image(), first try will failed because config file not exist ++ // and image_store_set_big_data() will retry this function ++ if (!util_file_exists(config_file)) { ++ WARN("Oci image spec: %s not found.", config_file); ++ ret = -1; ++ goto out; ++ } ++ + img->spec = oci_image_spec_parse_file(config_file, NULL, &err); + if (img->spec == NULL) { + ERROR("Failed to parse oci image spec: %s", err); +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c +index 3bc433ae..3d814954 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c +@@ -1133,7 +1133,12 @@ static char *read_layer_lower_file(const char *layer_dir) + goto out; + } + ++ // lowest layer do not have lower file ++ if (!util_file_exists(lower_file)) { ++ goto out; ++ } + lower = util_read_text_file(lower_file); ++ + out: + free(lower_file); + return lower; +-- +2.42.0 + diff --git a/0060-remove-var-in-coverage-and-fix-build-test-remove.patch b/0060-remove-var-in-coverage-and-fix-build-test-remove.patch new file mode 100644 index 0000000..3c1de53 --- /dev/null +++ b/0060-remove-var-in-coverage-and-fix-build-test-remove.patch @@ -0,0 +1,30 @@ +From f1fa4c7bdc2c67a4ef9c476ba9e0e2de6b589bc5 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Thu, 14 Dec 2023 10:49:26 +0800 +Subject: [PATCH 60/64] remove /var/* in coverage and fix build/test remove + +Signed-off-by: jikai <jikai11@huawei.com> +--- + CI/generate_gcov.sh | 6 ++---- + 1 file changed, 2 insertions(+), 4 deletions(-) + +diff --git a/CI/generate_gcov.sh b/CI/generate_gcov.sh +index 76bf382b..7518c5c1 100755 +--- a/CI/generate_gcov.sh ++++ b/CI/generate_gcov.sh +@@ -29,10 +29,8 @@ cp -r ~/build $ISULAD_COPY_PATH + cd $ISULAD_COPY_PATH/build + ctest + lcov --directory . --capture --output-file coverage.info --rc lcov_branch_coverage=1 +-# Remove std/build files +-lcov --remove coverage.info '/usr/*' -o coverage.info --rc lcov_branch_coverage=1 +-lcov --remove coverage.info "$ISULAD_SRC_PATH/build/*" -o coverage.info --rc lcov_branch_coverage=1 +-lcov --remove coverage.info "$ISULAD_SRC_PATH/test/*" -o coverage.info --rc lcov_branch_coverage=1 ++# extract src only files ++lcov --extract coverage.info '*/iSulad/src/*' -o coverage.info --rc lcov_branch_coverage=1 + + # Generate html + genhtml --ignore-errors source -o $GCOV_RESULT_PATH/coverage coverage.info --branch-coverage --rc lcov_branch_coverage=1 +-- +2.42.0 + diff --git a/0061-2320-improve-CI-test.patch b/0061-2320-improve-CI-test.patch new file mode 100644 index 0000000..1c7326c --- /dev/null +++ b/0061-2320-improve-CI-test.patch @@ -0,0 +1,165 @@ +From 712d82656ac9bafda7d29be70e7dbcd761a01f98 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Sun, 17 Dec 2023 05:58:56 +0000 +Subject: [PATCH 61/64] !2320 improve CI test * improve CI test + +--- + .../container_cases/restartpolicy.sh | 35 ++++++++++++++----- + CI/test_cases/image_cases/image_digest.sh | 10 +++--- + CI/test_cases/image_cases/image_search.sh | 26 ++------------ + 3 files changed, 35 insertions(+), 36 deletions(-) + +diff --git a/CI/test_cases/container_cases/restartpolicy.sh b/CI/test_cases/container_cases/restartpolicy.sh +index 11c3608f..0ab09636 100755 +--- a/CI/test_cases/container_cases/restartpolicy.sh ++++ b/CI/test_cases/container_cases/restartpolicy.sh +@@ -23,18 +23,37 @@ curr_path=$(dirname $(readlink -f "$0")) + data_path=$(realpath $curr_path/../data) + source ../helpers.sh + ++# $1 : retry limit ++# $2 : retry_interval ++# $3 : container name ++# $4 : expect restart count ++function do_retry() ++{ ++ for i in $(seq 1 "$1"); do ++ count=`isula inspect --format='{{json .RestartCount}}' ${3}` ++ if [ $count -eq $4 ]; then ++ return 0 ++ fi ++ sleep $2 ++ done ++ echo "expect $4, get $count" ++ return 1 ++} ++ + function do_test_on_failure() + { ++ local retry_limit=15 ++ local retry_interval=1 + containername=test_rp_on_failure + isula run --name $containername -td --restart on-failure:3 busybox /bin/sh -c "exit 2" + fn_check_eq "$?" "0" "run failed" + +- sleep 8 +- count=`isula inspect --format='{{json .RestartCount}}' $containername` +- if [[ $count != "3" ]];then +- echo "expect 3 but get $count" ++ do_retry ${retry_limit} ${retry_interval} ${containername} 3 ++ if [[ $? -ne 0 ]];then + TC_RET_T=$(($TC_RET_T+1)) + fi ++ ++ isula stop -t 0 $containername + testcontainer $containername exited + + isula rm $containername +@@ -43,14 +62,14 @@ function do_test_on_failure() + + function do_test_unless_stopped() + { ++ local retry_limit=15 ++ local retry_interval=1 + containername=test_rp_unless_stopped + isula run --name $containername -td --restart unless-stopped busybox /bin/sh -c "exit 2" + fn_check_eq "$?" "0" "run failed" + +- sleep 8 +- count=`isula inspect --format='{{json .RestartCount}}' $containername` +- if [[ $count == "0" ]];then +- echo "expect not 0 but get $count" ++ do_retry ${retry_limit} ${retry_interval} ${containername} 0 ++ if [[ $? -ne 0 ]];then + TC_RET_T=$(($TC_RET_T+1)) + fi + +diff --git a/CI/test_cases/image_cases/image_digest.sh b/CI/test_cases/image_cases/image_digest.sh +index e30f29f0..cc8b0e48 100755 +--- a/CI/test_cases/image_cases/image_digest.sh ++++ b/CI/test_cases/image_cases/image_digest.sh +@@ -25,14 +25,14 @@ source ../helpers.sh + function test_image_with_digest() + { + local ret=0 +- local image="busybox" +- local image2="ubuntu" +- local image_digest="busybox@sha256:5cd3db04b8be5773388576a83177aff4f40a03457a63855f4b9cbe30542b9a43" ++ local image="3laho3y3.mirror.aliyuncs.com/library/busybox" ++ local image2="3laho3y3.mirror.aliyuncs.com/library/ubuntu" ++ local image_digest="3laho3y3.mirror.aliyuncs.com/library/busybox@sha256:62ffc2ed7554e4c6d360bce40bbcf196573dd27c4ce080641a2c59867e732dee" + local test="pull && inspect && tag image with digest test => (${FUNCNAME[@]})" + + msg_info "${test} starting..." + +- isula pull docker.io/library/${image_digest} ++ isula pull ${image_digest} + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${image}" && return ${FAILURE} + + isula tag ${image_digest} ${image}:digest_test +@@ -71,7 +71,7 @@ function test_image_with_digest() + isula inspect -f '{{.image.repo_tags}}' ${image_digest} | grep "${image}:digest_test" + [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - image digest delete error: ${image_digest}" && ((ret++)) + +- isula pull docker.io/library/${image2}:latest ++ isula pull ${image2}:latest + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${image2}" && return ${FAILURE} + + digest=$(isula inspect "${image2}:latest" | grep "@sha256" | awk -F"\"" '{print $2}') +diff --git a/CI/test_cases/image_cases/image_search.sh b/CI/test_cases/image_cases/image_search.sh +index 1d281cb2..11af02f1 100755 +--- a/CI/test_cases/image_cases/image_search.sh ++++ b/CI/test_cases/image_cases/image_search.sh +@@ -33,6 +33,7 @@ function test_image_search() + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop isulad failed" && return ${FAILURE} + + msg_info "${test} starting..." ++ rm -rf /etc/isulad/daemon.bak + cp /etc/isulad/daemon.json /etc/isulad/daemon.bak + sed -i "/registry-mirrors/a\ \"docker.io\"," /etc/isulad/daemon.json + +@@ -49,39 +50,18 @@ function test_image_search() + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - search ${invalid_image} should fail as it's search name is invalid" && return ${FAILURE} + + # test search options +- isula search --no-trunc ${image} +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to search images with no-trunc: ${image}" && ((ret++)) +- +- isula search --limit 5 ${image} +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to search images with limit: ${image}" && ((ret++)) ++ isula search --no-trunc --limit 5 --filter stars=3 --filter is-official=true --filter is-automated=false --format "table {{.Name}}\t{{.IsOfficial}}" ${image} ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to search images with search options: ${image}" && ((ret++)) + + isula search --limit -1 ${image} 2>&1 | grep "Invalid value" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to search images with limit: ${image} and and catch error msg" && ((ret++)) + +- isula search --filter stars=3 ${image} +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to search images with filter stars: ${image}" && ((ret++)) +- +- isula search --filter is-official=true ${image} +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to search images with filter is-official: ${image}" && ((ret++)) +- +- isula search --filter is-automated=true ${image} 2>&1 | grep "AUTOMATED" +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to search images with filter is-automated: ${image}" && ((ret++)) +- + isula search --filter aa=true ${image} 2>&1 | grep "Invalid filter" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to set filter for search ${image} and catch error msg" && ((ret++)) + +- isula search ${image} 2>&1 | grep "NAME" +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to search images with default table format: ${image}" && ((ret++)) +- +- isula search --format "table {{.IsAutomated}}\t{{.IsOfficial}}" ${image} 2>&1 | grep "AUTOMATED" +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to search images with table format: ${image}" && ((ret++)) +- + isula search --format "{{Name}}" ${image} 2>&1 | grep "invalid format field" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to set format for search ${image} and catch error msg" && ((ret++)) + +- isula search --format "{{.Name}}" ${image} 2>&1 +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to search images with none-table format: ${image}" && ((ret++)) +- + cp -f /etc/isulad/daemon.bak /etc/isulad/daemon.json + + check_valgrind_log +-- +2.42.0 + diff --git a/0062-verify-name-and-digest-consistency.patch b/0062-verify-name-and-digest-consistency.patch new file mode 100644 index 0000000..e56bbd7 --- /dev/null +++ b/0062-verify-name-and-digest-consistency.patch @@ -0,0 +1,319 @@ +From 950dc3c56f192061383de4d19229ace243eae503 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Mon, 18 Dec 2023 15:54:37 +0800 +Subject: [PATCH 62/64] verify name and digest consistency + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + .../oci/storage/image_store/image_store.c | 265 +++++++++++------- + 1 file changed, 162 insertions(+), 103 deletions(-) + +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.c b/src/daemon/modules/image/oci/storage/image_store/image_store.c +index 58baa47a..1b482504 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_store.c ++++ b/src/daemon/modules/image/oci/storage/image_store/image_store.c +@@ -22,6 +22,7 @@ + #include <isula_libutils/storage_image.h> + #include <isula_libutils/imagetool_images_list.h> + #include <isula_libutils/json_common.h> ++#include <isula_libutils/auto_cleanup.h> + #include <limits.h> + #include <pthread.h> + #include <stdlib.h> +@@ -444,11 +445,161 @@ out: + return value; + } + ++static int resort_image_names(const char **names, size_t names_len, char **first_name, char ***image_tags, ++ char ***image_digests) ++{ ++ int ret = 0; ++ size_t i; ++ char *prefix = NULL; ++ ++ for (i = 0; i < names_len; i++) { ++ size_t len = strlen(names[i]); ++ if (strlen(names[i]) > MAX_IMAGE_NAME_LENGTH) { ++ prefix = util_sub_string(names[i], len - MAX_IMAGE_NAME_LENGTH, ++ MAX_IMAGE_NAME_LENGTH - MAX_IMAGE_DIGEST_LENGTH); ++ } ++ ++ // TODO: maybe should support other digest ++ if (prefix != NULL && strcmp(prefix, DIGEST_PREFIX) == 0) { ++ if (util_array_append(image_digests, names[i]) != 0) { ++ ERROR("Failed to append image to digest: %s", names[i]); ++ ret = -1; ++ goto out; ++ } ++ } else { ++ if (util_array_append(image_tags, names[i]) != 0) { ++ ERROR("Failed to append image to tags: %s", names[i]); ++ ret = -1; ++ goto out; ++ } ++ } ++ } ++ ++ if (first_name == NULL) { ++ goto out; ++ } ++ ++ if (util_array_len((const char **)(*image_digests)) > 0) { ++ free(*first_name); ++ *first_name = util_strdup_s((*image_digests)[0]); ++ } ++ ++ if (util_array_len((const char **)(*image_tags)) > 0) { ++ free(*first_name); ++ *first_name = util_strdup_s((*image_tags)[0]); ++ } ++ ++out: ++ if (ret != 0) { ++ util_free_array(*image_digests); ++ util_free_array(*image_tags); ++ free(*first_name); ++ } ++ free(prefix); ++ return ret; ++} ++ ++// Validate checks that the contents is a valid digest ++static bool validate_digest(const char *digest) ++{ ++ bool ret = true; ++ const char *sha256_encode_patten = "^[a-f0-9]{64}$"; ++ char *value = util_strdup_s(digest); ++ char *index = strchr(value, ':'); ++ char *alg = NULL; ++ char *encode = NULL; ++ ++ // contains ':' and is not the last character ++ if (index == NULL || index - value + 1 == strlen(value)) { ++ INFO("Invalid checksum digest format"); ++ ret = false; ++ goto out; ++ } ++ ++ *index++ = '\0'; ++ ++ alg = value; ++ encode = index; ++ // Currently only support SHA256 algorithm ++ if (strcmp(alg, "sha256") != 0) { ++ DEBUG("Unsupported digest algorithm: %s", alg); ++ ret = false; ++ goto out; ++ } ++ ++ ret = util_reg_match(sha256_encode_patten, encode) == 0; ++ ++out: ++ free(value); ++ return ret; ++} ++ ++// Parsing a reference string as a possible identifier, full digest, or familiar name. ++static char *parse_digest_reference(const char *ref) ++{ ++ char *indentfier_patten = "^[a-f0-9]{64}$"; ++ ++ if (util_reg_match(indentfier_patten, ref) == 0) { ++ return util_string_append(ref, "sha256:"); ++ } ++ ++ if (validate_digest(ref)) { ++ return util_strdup_s(ref); ++ } ++ ++ return oci_normalize_image_name(ref); ++} ++ ++static int is_name_digest_consistent(const char *name, char **names, size_t names_len, const char *digest) ++{ ++ size_t i; ++ int ret = -1; ++ int nret = 0; ++ char *tag_pos = NULL; ++ char **tags = NULL; ++ char **digests = NULL; ++ ++ if (resort_image_names((const char **)names, names_len, NULL, &tags, &digests) != 0) { ++ ERROR("Failed to resort image names"); ++ goto out; ++ } ++ ++ for (i = 0; i < util_array_len((const char **)tags); i++) { ++ __isula_auto_free char *ref = NULL; ++ __isula_auto_free char *tmp_repo_digests = NULL; ++ ref = parse_digest_reference(tags[i]); ++ if (ref == NULL) { ++ continue; ++ } ++ tag_pos = util_tag_pos(ref); ++ if (tag_pos == NULL) { ++ ERROR("invalid ref %s", ref); ++ continue; ++ } ++ *tag_pos = '\0'; ++ ++ nret = asprintf(&tmp_repo_digests, "%s@%s", ref, digest); ++ if (nret < 0) { ++ ERROR("Failed to receive repo digest"); ++ goto out; ++ } ++ if (strcmp(name, tmp_repo_digests) == 0) { ++ ret = 0; ++ goto out; ++ } ++ } ++out: ++ util_free_array(tags); ++ util_free_array(digests); ++ return ret; ++} ++ + // by_digest returns the image which matches the specified name. + static image_t *by_digest(const char *name) + { + digest_image_t *digest_filter_images = NULL; + char *digest = NULL; ++ image_t *tmp_ret = NULL; + + // split digest for image name with digest + digest = strrchr(name, '@'); +@@ -457,12 +608,21 @@ static image_t *by_digest(const char *name) + } + digest++; + digest_filter_images = (digest_image_t *)map_search(g_image_store->bydigest, (void *)digest); +- if (digest_filter_images == NULL) { ++ if (digest_filter_images == NULL || linked_list_empty(&(digest_filter_images->images_list))) { + return NULL; + } + + // currently, a digest corresponds to an image, directly returning the first element +- return linked_list_first_elem(&(digest_filter_images->images_list)); ++ tmp_ret = linked_list_first_elem(&(digest_filter_images->images_list)); ++ ++ // verify name and digest consistency to ensure we are not matching images to different repositories, ++ // even if the digests match. ++ // For example, ubuntu@sha256:abc......, shouldn't match test@sha256:abc...... ++ if (is_name_digest_consistent(name, tmp_ret->simage->names, tmp_ret->simage->names_len, digest) != 0) { ++ return NULL; ++ } ++ ++ return tmp_ret; + } + + static image_t *lookup(const char *id) +@@ -2001,107 +2161,6 @@ out: + return ret; + } + +-static int resort_image_names(const char **names, size_t names_len, char **first_name, char ***image_tags, +- char ***image_digests) +-{ +- int ret = 0; +- size_t i; +- char *prefix = NULL; +- +- for (i = 0; i < names_len; i++) { +- size_t len = strlen(names[i]); +- if (strlen(names[i]) > MAX_IMAGE_NAME_LENGTH) { +- prefix = util_sub_string(names[i], len - MAX_IMAGE_NAME_LENGTH, +- MAX_IMAGE_NAME_LENGTH - MAX_IMAGE_DIGEST_LENGTH); +- } +- +- // TODO: maybe should support other digest +- if (prefix != NULL && strcmp(prefix, DIGEST_PREFIX) == 0) { +- if (util_array_append(image_digests, names[i]) != 0) { +- ERROR("Failed to append image to digest: %s", names[i]); +- ret = -1; +- goto out; +- } +- } else { +- if (util_array_append(image_tags, names[i]) != 0) { +- ERROR("Failed to append image to tags: %s", names[i]); +- ret = -1; +- goto out; +- } +- } +- } +- +- if (util_array_len((const char **)(*image_digests)) > 0) { +- free(*first_name); +- *first_name = util_strdup_s((*image_digests)[0]); +- } +- +- if (util_array_len((const char **)(*image_tags)) > 0) { +- free(*first_name); +- *first_name = util_strdup_s((*image_tags)[0]); +- } +- +-out: +- if (ret != 0) { +- util_free_array(*image_digests); +- util_free_array(*image_tags); +- free(*first_name); +- } +- free(prefix); +- return ret; +-} +- +-// Validate checks that the contents is a valid digest +-static bool validate_digest(const char *digest) +-{ +- bool ret = true; +- const char *sha256_encode_patten = "^[a-f0-9]{64}$"; +- char *value = util_strdup_s(digest); +- char *index = strchr(value, ':'); +- char *alg = NULL; +- char *encode = NULL; +- +- // contains ':' and is not the last character +- if (index == NULL || index - value + 1 == strlen(value)) { +- INFO("Invalid checksum digest format"); +- ret = false; +- goto out; +- } +- +- *index++ = '\0'; +- +- alg = value; +- encode = index; +- // Currently only support SHA256 algorithm +- if (strcmp(alg, "sha256") != 0) { +- DEBUG("Unsupported digest algorithm: %s", alg); +- ret = false; +- goto out; +- } +- +- ret = util_reg_match(sha256_encode_patten, encode) == 0; +- +-out: +- free(value); +- return ret; +-} +- +-// Parsing a reference string as a possible identifier, full digest, or familiar name. +-static char *parse_digest_reference(const char *ref) +-{ +- char *indentfier_patten = "^[a-f0-9]{64}$"; +- +- if (util_reg_match(indentfier_patten, ref) == 0) { +- return util_string_append(ref, "sha256:"); +- } +- +- if (validate_digest(ref)) { +- return util_strdup_s(ref); +- } +- +- return oci_normalize_image_name(ref); +-} +- + static int pack_repo_digest(char ***old_repo_digests, const char **image_tags, const char *digest, char ***repo_digests) + { + int ret = 0; +-- +2.42.0 + diff --git a/0063-code-improve-for-oci_rmi.patch b/0063-code-improve-for-oci_rmi.patch new file mode 100644 index 0000000..0d8f427 --- /dev/null +++ b/0063-code-improve-for-oci_rmi.patch @@ -0,0 +1,35 @@ +From 2db6add74c621344e902ce28b5e6764f6ef55b8e Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Mon, 18 Dec 2023 16:07:57 +0800 +Subject: [PATCH 63/64] code improve for oci_rmi + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/modules/image/oci/oci_image.c | 11 +++++++++++ + 1 file changed, 11 insertions(+) + +diff --git a/src/daemon/modules/image/oci/oci_image.c b/src/daemon/modules/image/oci/oci_image.c +index 471510e7..9cf2cd4f 100644 +--- a/src/daemon/modules/image/oci/oci_image.c ++++ b/src/daemon/modules/image/oci/oci_image.c +@@ -542,6 +542,17 @@ int oci_rmi(const im_rmi_request *request) + goto out; + } + ++ for (i = 0; i < image_names_len; i++) { ++ if (strcmp(real_image_name, image_names[i]) == 0) { ++ break; ++ } ++ } ++ if (i == image_names_len) { ++ ERROR("Invalid real_image_name"); ++ ret = -1; ++ goto out; ++ } ++ + reduced_image_names = (char **)util_smart_calloc_s(sizeof(char *), image_names_len - 1); + if (reduced_image_names == NULL) { + ERROR("Out of memory"); +-- +2.42.0 + diff --git a/0064-bugfix-for-resort_image_names.patch b/0064-bugfix-for-resort_image_names.patch new file mode 100644 index 0000000..19ef9cb --- /dev/null +++ b/0064-bugfix-for-resort_image_names.patch @@ -0,0 +1,26 @@ +From 39686ee4443400b810edecb38e3891b808e3a065 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Mon, 18 Dec 2023 20:59:46 +0800 +Subject: [PATCH 64/64] bugfix for resort_image_names + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/modules/image/oci/storage/image_store/image_store.c | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.c b/src/daemon/modules/image/oci/storage/image_store/image_store.c +index 1b482504..034268bc 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_store.c ++++ b/src/daemon/modules/image/oci/storage/image_store/image_store.c +@@ -473,6 +473,8 @@ static int resort_image_names(const char **names, size_t names_len, char **first + goto out; + } + } ++ free(prefix); ++ prefix = NULL; + } + + if (first_name == NULL) { +-- +2.42.0 + diff --git a/0065-fix-stopp-removes-cont-error-remove-inspect-error-lo.patch b/0065-fix-stopp-removes-cont-error-remove-inspect-error-lo.patch new file mode 100644 index 0000000..ba38cc7 --- /dev/null +++ b/0065-fix-stopp-removes-cont-error-remove-inspect-error-lo.patch @@ -0,0 +1,76 @@ +From 3d8c6127161acbe35bf03fe495ac43e2b9242cbf Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Tue, 19 Dec 2023 18:31:30 +0800 +Subject: [PATCH 65/71] fix stopp removes cont error & remove inspect error log + +Signed-off-by: jikai <jikai11@huawei.com> +--- + .../cri_pod_sandbox_manager_service.cc | 36 ++++++++----------- + .../container_cb/execution_information.c | 2 +- + 2 files changed, 15 insertions(+), 23 deletions(-) + +diff --git a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +index bc40cb06..4d1d19eb 100644 +--- a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +@@ -703,30 +703,22 @@ auto PodSandboxManagerService::GetRealSandboxIDToStop(const std::string &podSand + std::map<std::string, std::string> &stdAnnos, Errors &error) + -> int + { +- Errors statusErr; ++ auto status = PodSandboxStatus(podSandboxID, error); ++ if (error.NotEmpty()) { ++ return -1; ++ } + +- auto status = PodSandboxStatus(podSandboxID, statusErr); +- if (statusErr.Empty()) { +- if (status->linux().namespaces().has_options()) { +- hostNetwork = (status->linux().namespaces().options().network() == runtime::v1alpha2::NamespaceMode::NODE); +- } +- // if metadata is invalid, don't return -1 and continue stopping pod +- if (status->has_metadata()) { +- name = status->metadata().name(); +- ns = status->metadata().namespace_(); +- } +- realSandboxID = status->id(); +- CRIHelpers::ProtobufAnnoMapToStd(status->annotations(), stdAnnos); +- } else { +- if (CRIHelpers::IsContainerNotFoundError(statusErr.GetMessage())) { +- WARN("Both sandbox container and checkpoint for id %s could not be found. " +- "Proceed without further sandbox information.", +- podSandboxID.c_str()); +- } else { +- error.Errorf("failed to get sandbox status: %s", statusErr.GetCMessage()); +- return -1; +- } ++ if (status->linux().namespaces().has_options()) { ++ hostNetwork = (status->linux().namespaces().options().network() == runtime::v1alpha2::NamespaceMode::NODE); + } ++ // if metadata is invalid, don't return -1 and continue stopping pod ++ if (status->has_metadata()) { ++ name = status->metadata().name(); ++ ns = status->metadata().namespace_(); ++ } ++ realSandboxID = status->id(); ++ CRIHelpers::ProtobufAnnoMapToStd(status->annotations(), stdAnnos); ++ + if (realSandboxID.empty()) { + realSandboxID = podSandboxID; + } +diff --git a/src/daemon/executor/container_cb/execution_information.c b/src/daemon/executor/container_cb/execution_information.c +index 420f08df..03fce848 100644 +--- a/src/daemon/executor/container_cb/execution_information.c ++++ b/src/daemon/executor/container_cb/execution_information.c +@@ -914,7 +914,7 @@ static int inspect_container_helper(const char *id, int timeout, char **containe + + inspect = inspect_container(id, timeout, true); + if (inspect == NULL) { +- ERROR("Failed to inspect container:%s", id); ++ DEBUG("Failed to inspect container:%s", id); + ret = -1; + goto out; + } +-- +2.25.1 + diff --git a/0066-2313-use-lxc-5.X-in-CI-testcase.patch b/0066-2313-use-lxc-5.X-in-CI-testcase.patch new file mode 100644 index 0000000..a730c39 --- /dev/null +++ b/0066-2313-use-lxc-5.X-in-CI-testcase.patch @@ -0,0 +1,41 @@ +From 1c28f9259463433a7aac10733be09f8d47ec17a5 Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu <zhangxiaoyu58@huawei.com> +Date: Thu, 21 Dec 2023 12:36:52 +0000 +Subject: [PATCH 66/71] !2313 use lxc 5.X in CI testcase * use lxc 5.X in CI + testcase + +--- + CI/install_depends.sh | 16 +++++----------- + 1 file changed, 5 insertions(+), 11 deletions(-) + +diff --git a/CI/install_depends.sh b/CI/install_depends.sh +index 5a4d71fa..ebeb79db 100755 +--- a/CI/install_depends.sh ++++ b/CI/install_depends.sh +@@ -105,18 +105,12 @@ cd ~ + git clone https://gitee.com/src-openeuler/lxc.git + git config --global --add safe.directory ~/lxc/lxc-5.0.2 + cd lxc +-git checkout origin/openEuler-22.03-LTS-SP1 +-tar xf lxc-4.0.3.tar.gz +-cd lxc-4.0.3 +-mv ../*.patch . +-for var in $(ls 0*.patch | sort -n) +-do +- patch -p1 < ${var} +-done ++./apply-patches ++cd lxc-5.0.2 ++mkdir -p build + sed -i 's/fd == STDIN_FILENO || fd == STDOUT_FILENO || fd == STDERR_FILENO/fd == 0 || fd == 1 || fd == 2 || fd >= 1000/g' ./src/lxc/start.c +-./autogen.sh +-./configure --prefix=${builddir} enable_werror=no +-make -j $(nproc) ++meson setup -Disulad=true -Dprefix=${builddir} build ++meson compile -C build + make install + ldconfig + +-- +2.25.1 + diff --git a/0067-2329-modify-mount-dev-directory-for-lxc-5.X.patch b/0067-2329-modify-mount-dev-directory-for-lxc-5.X.patch new file mode 100644 index 0000000..a67684f --- /dev/null +++ b/0067-2329-modify-mount-dev-directory-for-lxc-5.X.patch @@ -0,0 +1,27 @@ +From 4a3ec85d707db28f10d4cd5654abf227dfc515cc Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu <zhangxiaoyu58@huawei.com> +Date: Mon, 25 Dec 2023 09:34:28 +0000 +Subject: [PATCH 67/71] !2329 modify mount /dev directory for lxc 5.X * modify + mount /dev directory for lxc 5.X + +--- + CI/test_cases/container_cases/bind_special_dir.sh | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/CI/test_cases/container_cases/bind_special_dir.sh b/CI/test_cases/container_cases/bind_special_dir.sh +index d030bb08..545d5099 100755 +--- a/CI/test_cases/container_cases/bind_special_dir.sh ++++ b/CI/test_cases/container_cases/bind_special_dir.sh +@@ -42,7 +42,8 @@ function test_bind_special_dir() + if [ $runtime == "runc" ]; then + c_id=`isula run -itd -v -itd --runtime=$runtime -v /sys/fs:/sys/fs:rw -v /proc:/proc -v /dev/pts:/dev/pts:rw busybox sh` + else +- c_id=`isula run --runtime=$runtime -itd -v -itd -v /sys/fs:/sys/fs:rw -v /proc:/proc -v /dev:/dev:ro -v /dev/pts:/dev/pts:rw busybox sh` ++ # lxc 5.X cannot support mount /dev directory ++ c_id=`isula run --runtime=$runtime -itd -v -itd -v /sys/fs:/sys/fs:rw -v /proc:/proc busybox sh` + fi + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + +-- +2.25.1 + diff --git a/0068-add-cri-1.29-api-change-docs.patch b/0068-add-cri-1.29-api-change-docs.patch new file mode 100644 index 0000000..97aee02 --- /dev/null +++ b/0068-add-cri-1.29-api-change-docs.patch @@ -0,0 +1,139 @@ +From 6ffd8232929b3cd1873c899d6bf379013959fb28 Mon Sep 17 00:00:00 2001 +From: jikai <jikai11@huawei.com> +Date: Fri, 22 Dec 2023 15:26:48 +0800 +Subject: [PATCH 68/71] add cri 1.29 api change docs + +Signed-off-by: jikai <jikai11@huawei.com> +--- + .../detailed/CRI/CRI_1.29_interface_change.md | 105 ++++++++++++++++++ + docs/images/cri_1.29_interface_change.svg | 5 + + 2 files changed, 110 insertions(+) + create mode 100644 docs/design/detailed/CRI/CRI_1.29_interface_change.md + create mode 100644 docs/images/cri_1.29_interface_change.svg + +diff --git a/docs/design/detailed/CRI/CRI_1.29_interface_change.md b/docs/design/detailed/CRI/CRI_1.29_interface_change.md +new file mode 100644 +index 00000000..f94d001b +--- /dev/null ++++ b/docs/design/detailed/CRI/CRI_1.29_interface_change.md +@@ -0,0 +1,105 @@ ++| Author | 吉凯 | ++| ------ | ------------------------ | ++| Date | 2023-12-22 | ++| Email | jikai11@huawei.com | ++ ++### 参考代码 ++ ++升级版本：1.29 ++参考地址：<https://github.com/kubernetes/cri-api/tree/kubernetes-1.29.0> ++ ++### 变更依赖图 ++ ++ ++ ++### 变更说明 ++ ++##### [CRI: Add Windows Podsandbox Stats](https://github.com/kubernetes/kubernetes/pull/110754) ++ ++不支持，无需变更 ++ ++- Added fields to the type `WindowsPodSandboxStats` expressing stats required to be collected from windows pods. ++ ++##### [Windows hostnetwork alpha](https://github.com/kubernetes/kubernetes/pull/112961) ++ ++不支持，无需变更 ++ ++- New type `WindowsNamespaceOption` introduced ++- The type `WindowsSandboxSecurityContext` has a new field `namespace_options` of type `WindowsNamespaceOption` ++ ++##### [Improve the API description of `PodSecurityContext.SupplementalGroups` to clarify its unfamiliar behavior](https://github.com/kubernetes/kubernetes/pull/113047) ++ ++描述修改，优化`PodSecurityContext.SupplementalGroups`的注释，明确容器镜像定义的主UID不在该列表下的行为 ++ ++- Clarified the expected behavior of `SupplementalGroups` field of `PodSecurityContext` ++ ++##### [Add Support for Evented PLEG](https://github.com/kubernetes/kubernetes/pull/111384) ++ ++新增字段，`GetContainerEvent`提供pod status和container status信息，`PodSandboxStatus`提供container status信息，[KEP-3386](https://github.com/kubernetes/enhancements/blob/master/keps/sig-node/3386-kubelet-evented-pleg/README.md) ++ ++- The type `ContainerEventResponse` updated: the field `pod_sandbox_metadata` removed and fields `pod_sandbox_status` and `containers_statuses` added. ++- The type `PodSandboxStatusResponse` has a new fields `containers_statuses` and `timestamp` ++ ++##### [CRI: Add CDI device info for containers](https://github.com/kubernetes/kubernetes/pull/115891/) ++ ++新增字段，CDI特性支持，CDI设备信息不再从annotation获取，直接从`ContainerConfig`获取，[KEP-3063](https://github.com/kubernetes/enhancements/blob/master/keps/sig-node/3063-dynamic-resource-allocation/README.md) ++ ++- New type `CDIDevice` was introduced and added to container config ++ ++##### [Add mappings for volumes](https://github.com/kubernetes/kubernetes/pull/116377) ++ ++新增字段，`Mount`中新增UID/GID映射信息，要求CRI创建挂载绑定时指定UID/GID映射信息，Kubelet不再负责映射， ++[KEP-127](https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/127-user-namespaces) ++ ++- Added new fields to the type `Mount` expressing runtime UID/GID mappings for the mount. ++ ++##### [cri-api: fix comment lines about PROPAGATION_PRIVATE](https://github.com/kubernetes/kubernetes/pull/115704) ++ ++描述修改，修改对PROPAGATION_PRIVATE的不正确注释 ++ ++- Fixed comment lines about PROPAGATION_PRIVATE ++ ++##### [Add user specified image to CRI ContainerConfig](https://github.com/kubernetes/kubernetes/pull/118652) ++ ++新增字段，`ImageSpec`新增`user_specified_image`，确保创建容器时验证正确的镜像 ++ ++- Added the `user_specified_image` field to type `ImageSpec` ++ ++##### [kubelet: get cgroup driver config from CRI](https://github.com/kubernetes/kubernetes/pull/118770) ++ ++新增rpc，获取cgroup驱动配置，[KEP-4033](https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/4033-group-driver-detection-over-cri) ++ ++- Added rpc for querying runtime configuration ++- Added cavieats about cgroup driver field ++ ++##### [Add swap to stats to Summary API and Prometheus endpoints (/stats/summary and /metrics/resource)](https://github.com/kubernetes/kubernetes/pull/118865) ++ ++新增字段，`ContainerStats`中新增虚拟内存使用情况信息，[KEP-2400](https://github.com/kubernetes/enhancements/blob/master/keps/sig-node/2400-node-swap/README.md#beta-1) ++ ++- Added `SwapUsage` type ++- Added `SwapUsage` field to `ContainerStats` type ++ ++##### [Expose commit memory used in WindowsMemoryUsage struct](https://github.com/kubernetes/kubernetes/pull/119238) ++ ++不支持，无需变更。 ++ ++- Added the `commit_memory_bytes` field to type `WindowsMemoryUsage` ++ ++##### [Add runtime handler field to ImageSpec struct](https://github.com/kubernetes/kubernetes/pull/121121) ++ ++新增字段，指定拉取镜像所采用的运行时处理，[KEP-4216](https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/4216-image-pull-per-runtime-class) ++ ++- Added `runtime_handler` field to type `ImageSpec` ++ ++##### [kubelet: add support for broadcasting metrics from CRI](https://github.com/kubernetes/kubernetes/pull/113609) ++ ++新增rpc，`ListMetricDescriptors`和`ListPodSandboxMetrics`获取metrics信息，[KEP-2371](https://github.com/kubernetes/enhancements/blob/master/keps/sig-node/2371-cri-pod-container-stats/README.md) ++ ++- Added rpc for pulling the metrics from CRI and broadcasting them to prometheus ++- Added cavieats about metrics ++ ++##### [Kubelet disk api cri update](https://github.com/kubernetes/kubernetes/pull/120914) ++ ++新增字段，`ImageFsInfo`返回值添加容器文件系统信息，[KEP-4191](https://github.com/kubernetes/enhancements/blob/master/keps/sig-node/4191-split-image-filesystem/README.md) ++ ++- Added `container_filesystems` field to type `ImageFsInfoResponse` +diff --git a/docs/images/cri_1.29_interface_change.svg b/docs/images/cri_1.29_interface_change.svg +new file mode 100644 +index 00000000..06026d6f +--- /dev/null ++++ b/docs/images/cri_1.29_interface_change.svg +@@ -0,0 +1,5 @@ ++<svg version="1.1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1775.4295814700522 1691.8333333333335" width="1775.4295814700522" height="1691.8333333333335" filter="invert(93%) hue-rotate(180deg)" class="excalidraw-svg"> ++ <!-- svg-source:excalidraw --> ++ ++ <defs><style> @font-face {font-display: swap;font-family: "Virgil";src: url("data:application/font-woff;charset=utf-8;base64,d09GMk9UVE8AAO9AAAkAAAABO1AAAO73AAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAADYTJHQZgAIluATYCJAOQWAQGBY9lByBbpzpxQ1QZOitEWvl1ExHg7NqqxsS6RlCwSkciyuhqMfv//89KOsZwUAeGmJXV32EeijNbRUcYuVUTmWNvG5FWRBfkAW4YWVueXU6tq9d0pZufMcACcUzc7AmXTyBwYmw0WojBScItqLq853lRPMu/i21F/1VrveHFeyj+LdHE3AfsFiWacmNotmSIhpJesVGSFIFWJDgsMxNmRIsQQzGF4rvOX33MPtY+LeOF490dgR0G/l0zmGbP8ku9beGBOf2+EcKFm8Md59ESM5GZKYtFok/F14jmMBPlbsKEWTmjIcDMVCgM0RQv1VD4IZddn+x6g2Ff91p6BriTOHYqRgBPP/Z8O3Pn/WXBXUTBKoZoSbZg6N3qwPP/937uvc+bD5huHSziTSgio5mQNpAsr235cGq9P8/r5vc1H/KatrSSHj8fGwzDhavuia24cMyBvutkjYQtKFtwTcS1wVWcODY7KMSFOM69OS/+4fm59f7vRf5FsY0aMXLQQgsGrRwggmTYYCJVEmVBm4V5Z0ThidFYPeyrtRKEoeme3dvtgT6iOeJnlC8MsAKKT0UoFD469oV+l7DD8pn6lvkVc++0kSEfkCM0Vauqh+RnHNszHtYIqREKELpb3S0eAhMvIXmB7+tkb9/wpNO+5fOjmT/aU+6YYwu6wVlib1xrbGxcwNgduqmiNNOESimjUWmogISEAOOOe086XjvNccGlpjQnb8gjOY3Gyr1sLlmavf8vKSIJwtUoBiEBfY1jpaqrq1ET/z2/7f+/4/hmD/yU4cZxDvyseBGJ7/W9rVwMSlAkJUsk6hzqQCsqJVbd9NbifX/4r6pJ4LDfsD5vl0a3Nl1TSqk09SUxpviFT9KCpj3jFmScMz4t9f8VWXkadd4bSR35QqxNH33Ai5BewBI5hQAbQsZFe2nQZ84A787uLKHXduwYMHUawjI9wuP+7R+jZdW/WpL3hdof92mkcoac8YBwj3gJB7CB3aayqzgrsSIikYrAZYK2u3l6AJfgdm+OybOvB8+JTFAi4hU026JOz7fHCxQujUCxVVutY8Wgje+dK4CA+afpvl+9Nyecky85rLBnngvwxHEpBa9ATSI+MxnxjMcbadI6LaWIPXHlOeusTrrDBkAFCAzMc6UTOFEFKTY1Nm4GuAPasdAah4+DcVmNg/pfU5Nao+e7QykouG0KYR4ewkKQVmtP1n91ike6Kvt60/WySWeBpeOKr7QCg0hoKAwgPPOvqrl+UPI78CqVirROp/RhKm2YMy0C6CJQdAHdBNBOBOmKyGuC5AZe6X1LKQUflBN+2M7DB2VHoBs/KOcRvCZQfn6i0lsdM5Ypw5otoyCn0UlftixbMiaTk2XKy5Q1Y17G2zJOB7nKUm0/PkgM7J3K/X5azIxlS8eShkmeIYrIIuvN47j+8Xk5xmbzq/fd0AYlB/aifvZu/3or31eH2eCosZNM23gKUvMyo25K7fx8x2ZRqLM4/xxLbOST4x/7aOuym35y8olHctP4b8V/TX/YCP2p/1mult99+Gj0u488j59Q7/DTaWN4EkPx7itzRBhgghZ7XPAiQJAIcTLMtFCJKku1WqHHWlvsst9xLnKDYR4xwicQZ7tG+knSE5NnLF69LLqrR0X89vtHVudoGtiw59gDkletTr9hov04IcNsWAhCFGnUMYNrWMA6cqigBQoCbHTgQRhxDOMAjuIkzuIyzuAybuEuHmMEbxC773VEWIigqlrWqT4NbHCjmtiM5nd+y1vXlq5ob9d1c/f3eC93qA/7pk6NEihFn0Rmmnk61thin7O8wilOc4krXGeGedbYJEaSKhN6GOUQD/IKz/ACr/IW7/ERn/Il3/ITf/B1//9y0DjoHH5x+M2hIRCCxhEpTKlFmUZcYipGDGMmFmBNbIBtYYfYXewxq9h9hvn6/vr3thQeZ+KUArHScmqorznNakXbKqkpSqIM+SLlVlB9GtJ+HdUZXdZZXdGQ7uuZ3uizfnlBkASHEFk57pwrbnvKU17wunOuGjNrxbYj2+1zj/u92yM+4fO+5gu+7jt+6Od+40+uMZH/QTH0f6P/F2WjGegMdD6aieagxWg5WoU2oG0oH5WhWgfrEB2GI+UoOOqOGcc5+Rat6/9k/8+mUCCeJiIC5sZapHx9QEX2IMLTPuIbQjZoyoY9XgDCc1CSUcEfzRVFJcPJ4YfarTTXnEEp9Fll4bmCyF6huXD0c2NkyBrUclqFDS0fCcldmnBxq5ZzUdjQ0hBhra3noZkdrY2QIXdtwDXbzuJpombNK3hcpdld/aaHaGW4yOQ0Uq2KuSpqqHgkIlXMSxG5hZWvipLO5Yja+dGe2Ec0Xl7hJVeLriry+vKRHMnapHSSWh3Na/r0xosnKF8vBiOnHHm02DmrmKh7JKa/6Kx4A1EGB5L5gPZja3EC36Px8gVmpEMfpVo7eKal3dCDQk3OQWrtEfywxsP7kXKdc+xllnS10CSorh6UNHRFJNThKpI3Uj12NefH5EBdrYWSRruZkfa+i7mL+aR7b+irdu+/Svdr602Xpz3AJtUl38juYpi/pk0HYWKZtRHy//WcNGHEROmgcepGC9vOyzGRY1+qfJxX8Ka4G2gxbNKvLqrqBjSlQRMYHTrDDGsYBjAs71XrIXW0LYetKxhHqGHoNTTSyi1NsvSW9pYPZAY4gkxWtrU0z9vbRMU08fOJQxc+Syy+gtk9G54gA+xj+zrZX5Y0rjZrdBmGy68X/sBnzCF5AdMtKp4mqPbyQez0lZGifaq47Tf7SIZExoiy9kkO2JjFGs8VnQMdK9nMR0/nKEUbGADCEcjocG6DUGjz+CoAhQFwBBIDEwuFxo4bN+++CUBhAAhHIDEwsVBobBxhCwAAwIcDAomBiYVCY+OISUAQBH0SBIEgCIIgCIKgcvQdCMIRSAy0MCC+M6Y5BcVI/gkYkMn8sRZzoj6jjymGesj9+I2a22/ocz8NNPA2C+fgZtOvbe7ZbLKKuW1NchzK6mJ1Pf/1IF9PGNzuWk+fWr5AJLsGbXx6WqXtz7Sw5sK/2TRigdQEnKWCSwHOJDBxwFWcy/MuPpBExwGTydS++B38ygHYhoswSGJKao0cwRU+cxa87AMKDql645JDKYO+5m4QVzcJ3y7stzuYTaZTHAbNHXHpPfKmuHugSHKiUk/ZPC/TXrC0xqt/dsGQ57sgXd3hs/6nDbLRY/DknG/yxVziZVz+ldwy0Iy6cNfv9o1UVPeO7SRI/Eej3M5tWU/0ydP7mv8W+vb7QPcr3ytNdXF9bro22Y7bj7Zf+/5e7C/73n67/xrB1Iy9URj6uDPOjzvj1fgeZsnDRCrKMR3LsR2VaAcbarhhDXcEIhLDcSROx6U4E5fjZgzHoxiJt3PJ9GZwpV151l4c+73B+fGXExniT6JM1AELXkAM6XgMP5YTZEIWvN+8x96yP86/HmkfeR30BDeCq6FruBPGwn1qRw26oBG6Ql8N9qqpBupSDaqaqtbOeqAPtE9H9a6ua4yag2bZtBiT0RqTSZtKYzeKbdmu2Fors26rtyFbtJX2mzudd3CO+7zBVdzBKraziKWsZzf7OcUPuIzfccG8beSYgWdMypiWsTBjeUZ2xraMHRl1GW0ZwgxVhjlDyrAzchn1jJlUWRX48WpiZWxuhR6qmw57kBDolGzsgvY89rhb7XvJzwQfY4+20SX4OtWreKpuSFaXlECpiOVamcv4JA4/x8nKhM2qKCxtMjn4oyulp5+2+DzPxYtYzPSoPf9Yvlvd4vcuYU3142xsRkmAfbwfK6ODt6039/4JpW8VEqlHIZIunBL+6JmzfHyE3o3mF33fTPtyd+kt8NrwT5yY3tXcECqX0RZRjsN416y3bBO8IjDJcPLMjeX3UFc4USotsEQLOQvWwDz+tuofrKKIxg39EBCg4K6oeW4pSCLqm7ih+kAy7/Fydux5Ldh5nZZU05IuqzLvripqXcA6pZ2RPKH/d7Vhy+50yl+yFh/UIyLAjn8fv/fx2Z7SNsWc2qzj/U7z+cFG30qoPh7Z5qC3Ze5YhCSBZYH04b4xhJmKKLbQSuzNojJnYgVKYcrTV4jLFCCbX7QrsD8wOFpEHcVC6m7/T4/rDbOCMYvv9JxgP8P+5bnnhxblt29k79MERhrZfpNruPWEn9zyaWm9D6Yhlysalgwi4BGLoGwXLltLytTuk/Fts6n1vh2+DbWgbnrIDLRawWQ8nikM/EuPyg9/viTjuTpg3B1Y+hDUoC9ieivGQlh0cicgzznLfcQVZ3dVKCuRSmjBAmXfyhNu9pTGGImlCL53o0K42CnEDXXqbCl182f/Bf89gv5vwu2e7vJ4iWYg3CGwkKTThtD0GoseIwAWO58TjMhWCDTwf4n/MWnupK2x/YCzIig48kILRHftWSKYgi+uRo3xYmrCurpHP+wg/v01/53uutoVJiUQVDulJgTOfnkrR0ZI1k1tENAMdR8iv1SLeO3Yf8I/i8/b9WdwvtwVMZlpeDIIzYdmpyAtzZMJSHtCcxcsWTFrteeNzqfiMmfScU8kIho1b78owsx+mq4uQutyR6w+9Yo3PmDBKfNIugWybM4TIVcd0CWURQ0Sc9kHMwiRyovyick6zb2ikU77TH+AKiu2Gf+Ahqq76ShpCk2KlUBjcPCK11f9dj+F7vSs8H0W2JSabHDxn24YNFev43yHqeq6XfE4R3rTOnZgFh8dEP9Q12qAHDk022XQ1pUSyjS/MSNipVuDYUYp0cM2GyEIDULG3HlLVE2Auyq87HOQvZquGjVgYq9KECHaIH0yk/2vr37SWfBV8B39wF8uFIxNfTsoDBvADTTNVVQ7OCrfY6L0G9yaU8aGZUKY7KcewrBtVffi6ub8GXxbdpI97b8hBoFQUgYJ00i0awM2XFgyVBGcFpXLAuR+2yS9JkI/gib26ZWf37eeNzxCOI7hrUwsKtpDVjRC3jQBB+N9CokJWXc0xw4tO43puOVMxB3S5XQnpNSwKH7otNIEjQiM4uWWjq3nF5hOi5K7xSdq3G9gYjUCzpPda9tRcOW32Y1gO7B4VKSrcYl2suSDeHI+STDgD5mdKorFl+SBLm5wuaWCyjmlZRMIsqBi5kErzkGOJ4nBBsasiHSBSPGEh2GWK1XRkbr0idAO/blDD3UvNXmqJJmxpciGbqkJFkRNIUlDf+tr0yuVBmbNePeTRpvLyLH+s3osJPvlgTLqh3bYTHbiV3M6AGRNwjDRIuSterpZLzTWgcuy62rHrJUFzGaxIq+ArjzfVthhOorC0AiCppSU1uIeWScjvuX2saqaGLe8Fh3SttzX2jZ4L/t29aumKWwl4SBuLsVe6wvvnzl7f2Fj60x2nQvYoDXgEEtrWEGMMhekBIg0R6qwt5tu7aJsxFY80wzcmiOQL9SbhS37pegG7yUM0NERxJnuG9BGqdPXDMtrch5+Nbp6qOhNVG7d8BXcqZcV5coyFjc48CSzVlnjgNh2I9tCF1i0cmNaz0tWcdOWmnZQokq6pKW7cjrIj6FvBr7l+gH1AYTeF4etWFaoxef12lP4cFlvw7p8amqeUgDZqPYsC1MlUuNK8Mrdazc29gBZsdmRjPBrIuC6r/YdOIIzHVFYFJYrS6tbwdJCTvZNziE5kR7IqdUadOyGS3ESUHqK7/9NM3cbbyjUVkSsAOtQcaUIuezfbaVYT2V/XL5KOV9TQ9fG3WezS8MLr2cJBkVOfn9tH754Wao4QR+eb7wYvd2+107VAeA12lEZIBOssvLIFFCpkDpjRTY7Ovv0+7XIpQne3FTbfpFkF4sx2A8e2b/y6YM3OW4SCexYtGl0iDncNe3uxpa942A1SXQP5Cf61mjfYINvow1RZVm/4BbL9Xyj+fTag1N1gDdXtRHSP7w5j1qrmZ5xB3ApbsCAIKQ4zRDLkDfbE3FVSPJxnsxGk+P9wytWacTijZsSdBKq2ytb/prJ+O2mCchsWw3g0mDn9m7b4DN2d5seOaA066UNdPbUzuH6nQfXAunVgqo2ekuBJiumUfT9cxcjX6hdfdEzvjRfml7aBmEgqFVDoUAjb8kGGy6WB4uKKOG8vup5eFWweTsT9ItV3gQCp1IF8dX6Ok5ox9gPGraY2+n1iq7ZkiVZ5Dm9YDs7FyRqz+xpVVUolMUgZ/h2DpOmplbsrFN3TFtxNEArmqxBPWcSCxlJGisugPhwyHi/GDsfHF06QQKjxAP8pKgompEleE9YZHz32WddH0hyGDqskcSu9dgEuaqpMCjrBRHiIZeQsMiKnimZr0syQ+0SvxE9AFhxhQTs/AE/sanpzViT/b1wMD7ZvX1rBxCO8WcNLUCqHPuaarv1pOIBIHGalH2hk4EPA4jG3RO5MbMxXOhX7kx99f2ds1Vs7czum7f7DmvB5YZGY71ArWtIxnOaBSRdxAprUkE9EkEqH4Y6kuSWwazEblObCGPKRPDEqNqV1ZqTdw0xlgornRgwgOezRgQpEMGwA4+rQo3NbYvbG1gBuLaPrA9GzVMv9HbRsnav4EhDT3mZOlKNK+/e+Fx1s1pz/M9+UXf4Ja1DUcdd6aYJ8L++VzqQssWypCkghOsGqraGxEuQ2mgxXrGzpnFHzPRhhCT4J3DIYv5KjPnM8J3hYgyJRUFRERCiihSmv6uN1bjpzcsbV/NL2xfqs00vXq6mMiCiD4gbnSqNoxUtBqPV1/g+LIxqtIXWiofbm1thsPl6Tpp+Np3aKwM27ZYdKGpYoyiRVXwlWj4WDsx7Sn4iIWHC0NlwyGTXXr6wn4uzjyhq6JJ29+I1ABL4ss3q49ZkuZsSPlHwQmoZbRecMh/egNGDZJQi3KCtimV30adH2/PUB1KNGxptAPUUU7ajIqzWOquk8wrlY3tfbHumeo8VEOyryY9PbkV+J1BKHY2KWtbXAMeel6CQTZtybf3V627dbiPd71q46Ddo1HO0RRCRHyo+zBQ6YTv23devSKVABnQx4MJ/5Tlb/kCNO2giJf1rDycrANLWTQHC9x/Z272Dh6XbUfmlokgarA96u5NbV5DMzZEIEqnHX51L1IoVK7ndbzlCfP0n1UX1AtgGWfbL81d6m/JpVqBUQQS48gdViQ3PSvZD+VA4KO8s3VKOzdv7/ejA1bbLR/zvRoPib+DjXfcMJQd2I+MnyDoe/DePZDoN1QCCWscaqmlTQKd1AtMAB7bbbXfVsTEmNmOQ5EIvYqiRay296XeXArEJZJYnNeiPqoaL4seQk549GuJ/GTLKLduXFx+/SCHfYgMfIXRFrLHzo25iixS8ZaBbAuc7Cett9j01p1gzBodStOCUNZCCwIubdyN/utMxWO14HRr2m54XGI7nOpR6FsVAzvk0gLiVrFY4QZKKQSnXCC1RAq4xJbwHvP8Nyy0kkV7GC9pOVkK5M6vCwqa0z74TqahlwTSNmZKWbEHNZ6K6hPrG5FaXGRswXApnlnYWSWn6diVOZ79nOvEtn+Ox0RQ6xrJqA1cxLbjQJHiEZXWqYnb54dX2zEoxAQcYyxtX5JSdZTobF3KdImDFfgmUvYKII2bqGld+gzJ1Sapw1KJKJmbE8qJDxZTF08osFb/EMhMWs+mPDlXMCwmssLjEqUIZM9Ut8a2YS5QME1arzFoJ5lck07VmWSrXpid4zLMi0ww0pN7udPnlN1/hvUYAl62L039i1+t4moKRhtpjPKdhhJMXOPL0MpLpz0T2r4vrH581ODlaJd/z5uxALVOmys4gc4g4tnkm0yt9Yk0AMSu0KvP62glaZpuc/2JzG9suZErr5XUgDRhB1PifHrq0pvMFWDgK+c4wj14B5Om3O0g/F3lco3RR/aQgWEeRUE/slB77cjCLjTzqvCmGSrPM1EwiJDTNyALmpLmIgGrQ2pXdTLbk7vLBQQvjmUcktrqu72KCXYVU0+t1lYKg/gArCHwpVrmaQL0yckjHi1APcQhyykKTXX9tS/qqYwCZr16RoNHrtq9ceWLl/S0rn7jmlkW/waImOdjpXOn4bKbQpUscuG9fGb+xzmZMxYR6klAHLRZ/qJA2mjDlf+3hX0OMsUSsPQVhOPHsYmjyqGpQD4YJCCSMPCQv2q5IGDmI31OLvrQ01dvWVIanBA+Njw15M/XUSGnW/6I+tzKLo9Fj1+xXGPPUprENKxNidYwuiSiZbFutTvZlMVs5l9sqMQaR8LGJZeJUV3OWPWBkUfftVXC87aB2G39YDFYflLVYi5pIQDDNFBxKBlE1ZOZrWzjmgpJnu0o0AaR6neXUqQkJ19loSiYKkfHMwYnKfhMEUXOx02VkOGu+xrm6x6OvNc+9DajfWqqbU5OzpSUNwPN1zKNjWoHzF6kuw+EcRTkak/k0VudILpcSJXaPdFg+zHgJuWznOoTERhyYB5y6SCPWjrpkd39NaLQajfWd/uHWHoASlSRGba2322QquVondGQV20ZoNwLfO7l751szicl4a26KyEdv7bSPATtgJwUb6GRJtSGKRl4JhcO8qCVFZmPBKUkck2FO76rN17PktdM99RuLxKbn5gs3J3FHSuk4aqAk1ZACcNFGkJFaYxvBhvVyQ90LPqguF7fHJZ1+zh76WuBJPmLUuJoaNjBbPhdW4Abx+f51GmgAZIOYVBBv8i7WgQIt4w0bU4yN8UMzmMxcimVAllVR70rk+mJb1x1wRcsWr6SuCegtXmcFbZXI4pazvJwVljqqZGiqagmkGOQkoc5QGbD5S1rE/m9oYZeVhKXYlyqQFwoul/d5Nrp6SiKVmlThv1KiIog52ffqXvnT3xxQlwGT9OGGu9bWtn0+Dyqyyk9G51mb7sw/akWV73//dJNdcvmSQDNd/m6qUZ9XHRhWK0kRVetydUlKiYgFYr/8kEnFR2xXNiI/QK9CACmZ/+YXnf3LiOjMsS8ZnEgG63G1TZsNvzeGc4oeqmGeBLxogCYm2XWeAfNvJLvNO5S/onTtM7bnt9NTY6Rds26C4bbw/urcKix4S9t5hWLznuKpgW+FhAY3TMCEqVDWsJqeaX7XIyBw19HqWn68/u5qx/PVytZMTO8CGZaKfXMnuNX8WinQ8dW260lAKBdV2DWmsePLDauxHGxbE4d0lFBYwuzTG/gjJBPBQ0io/Z12b2eodtoTARgDpdIxSo5+/4l7YUDGSCgX6ha4JVZg4upSBZBjXQOimGEU1DcuUqbPWXsd/MczAA48M4B+0gpp9TEqiOV2NFhVIct11HPA3pfGoCQu+SIU8A0mREVOuSKXZNJys9d6O5kBWbq07svP1xb5aq4qn/nEGlSDh+O2g28bS6CG2abuNYKw7PjKDbQPhnlDt3AYZMLk87dDwAMyto8/2B+x8bvNlNB3R4DT6MZHUtVtlAr3xh0mpQ7aKnJAJSshCpAgdz2uiiAG/rD99o/XUODaTWr5BKoOdiMZ1Y0vug2o1ShyFVF2MZQNMMf84E66z0huVdwVpZcJeHZLG3DV0RwtJ1H6bzdWnZ87BkUEktYTxVcdByuZp1u352nwaA64cpHGX1aA2Pm3jjxjy9gGCbqilSAW9cM8wslgTjApO7u5Dfenl6eYOWrGoMI2wYTCoLyj7iub/VcZdDk6C4sywsci7sbhk8hs18B+32UvzeL7EfPPThkDuoYxqaeNzWhT9pdGnXHObIsHEEhWEMWpZYT7RoYhiGVN4sKs3B466Q88G8K/PDkP9IkN7Iz1rzCAmf77F10WCEy+W1mHfweg8htltqr2EqKSSNuco1kxc6ksA19TpZGFKQrTCvpJwff+YqgZbvttM8tTT5c0rLFVibKgWXl9dQLDI5NSoWLG85xuYziabFhAzljEhIlcrewjQnPG2JIPdCPjGA09JQRnNgLtVExzqGGWC3q32rimghfAFmMmuxwCFtd6EhrP7n49Pikn+aZp86ouacvA1mn3IePGfB5gDn3vkOwjCdNwINloGRy2xsRY2mQHAoyRCV2DVxhW5eUmnJwyej6yx6xUC4AFeVUyJfzK1gSvclTCoA8QxHkuhm18w3WELQ26hn0kcLfuCMexDQkAqbLKQ0aa4sHhljGG4RerELEjFJ+H9wGSr0eBevkxnn9Yj+AAEBlFqaAH6TGe5BUJUrcxwkWEAT4uJRw6HLZjXYjFmShkWEliKEhru1wMFQCNOrbRFVcPSBt1Aqe/GuP0BfzzGz9cW3I+s3JH6d7zPNHtRqA9bJS4IkTpohqK7cI6qzTpkji3uiYyGhZTZXFVE3ukSre9cPlO7S0ip6Bh7JQaNYUOwXblfCEELP1CJYAPUlOVjV22Q3ttPSpzFb4McMBEtXLzVyjNsgKRczcssaTDEjBDutnTgaqiTweDrT8SZi+58DlOCJA4XNTg6pG3U/rroamj7WTT52noPBolRUWR0KhujBe20ZBi/F4jVCOhWJwwKeXeqiF7lu/lHJE1eS3Fgno3UW1k+1QXR85btD3d3fid4lJjHUo3/rYpFQj5gWawRvTge6osbDe2V7rAwld3NSiXRX4eSTWdOV7Ob2nz9QKO9coZJknLzPQ6e/Jt+VZ4Mt1paA0aQ+thM1FdlCP1gXeqa7C620j7KLPb3JG2iqcmK8GUhSfWUbuXEtFXMZXtSe1Z4xlrg/nFGcjYOAWV63RehdtXnTvwHDIGuUElACLra3wCP9kk8z6Kp+485PnsZb+/93UWFVKh32aqSlfTEFNvkIBF4hRgVPUdi4tcKSxQDROKmCafwvL+3tp9tB7gt9AuTstW7Sf33LtyyzO4hcy2GMQ4HlBH3PWx/GOYBBg6pAoCzzBh+wGv0xsrLcud0OmyRYgILA719aT4ChbQnzhreVORUK1hc2QgKVp5pL8dMgHUrY2l5sZyFTOYl0Ie3AKZEO/arV6ApISCJVZEp2ShrijMQSgQTlGz9Ugrm2pkBKqe0ibpmTOQh/rAHv43T2BXA3BtpQT20woXrkiKQLjapGZ7h4vfq1XxZVLDHWtO2+bnFdWc4i8PrmBwRK1beJqSkhdouYDTZVGSrLkdCcwjdqTN9Ne9LNbR39Pzd4rTalZ93HbVhLdXXCCRZVqCgaAr8Gj87tPL7j3n7G9njhWADt4REmjnqRoioDdkDTLdubxDL66vjczXBWQu5CDUuenOC8ThncfkIbtmnAze4Cknur452dvk7onBc28W1MBtfGGvALu64uQCktJsnRNQHztGC4oC8VS1egptGIEFYKWtUI0AhPxo1jBcxcFaEp+t5ZUtEbD9N1IWtncO9q4hTrnohxYZcT1vNB3L5gxTN9tEM7meAeUFsOfSEBcB1tYFzCW0oOLCSvJiuKrd7NS6xIm6O/wtfvYGmRkRmJ6hTbcPrV96dZKYc0uxRmqTod/YN1QnUYBQWZV30MiB71ndge0w0Xx02zV7k1tlVu8PnhVE2BQEaXppK1/ZeMwaq2H9yW2jO5kVAjeIOQT+/dMN+LKlBy5ITd6ND6GVkyUH1cn9l0p1t/C0VvcJqUhZXU5qo/0JIKKDVEucSEAUUmwEouWEXvoJNWkLHblbNbhmvkXKnKyVmaxl+irxlHkwL/5Ra5I9SgxJVTBQwMcrFwu1tfhmdkFyAQSgwNckiY0Rl64mycf2XBdaxDnwBMstw5oUfzL21uQzxDz3RzFe8C1Mzaaqzi4Zh7UjbSzv1cbSQZQhONLcDC/ev/rS5iHAw5nPly+VltPTfd2eU7+qmzN5wlkJo6boBQ0yCh8oXmKKtEACgJtluJp6ALTs2NRoia7QexrLVMRfjfgkEFUqJmHYye/+FBiF7+HvZiIxE+IYVmdEJUI2HdQDDFyz1g8Z3yjfb5NpU4wr89e8J0aYqZCzVD1CY204dedr+ZHwA3nr3NjNrgs3e42TyL+kKQnl7yaf1wD+jxs3TCvVV1rHrbdjPpCNr9wSoULmayuo1SpVxEr1WkYulKtpEAt0bw7Hu4mSZkGtVhIqiLGUikkCcBAAcijrypeONwGX9CUfWlmqmUhN1zXFvviwT8YbCvtLxD3UHczdsNP5nHTQVGI/JR2+t0U7JK0P5LTmTAONui5V9bSYqPqsW6fzUlGcxfUq4IQyFqaxDSNnZi9TJQNrFMQgLI3JJoR78LIpAFfQe8H8NgFSaHZPhjiWRkW09lBpfpXIt4Gx/BH8ouFFgpmqEdCqrzRly+h5gK5+fVQAX6JqVtWopemdzmFeg0jjeBvZice2UQGpMHlDy2mBM91p0soCroBx1/GyD4tOEsKoFHT7ndb0n3iHBclEgKorBFnGKUFWMEzGd9xpIFOV4sTu+q6vH108hVZa1xwrwxOfZBVqPOVBm28UF7/OGhEQq7OKAZhmjNNcE+hyQTGEZTdRdZT1hul0ZzWEX3bNlsk7GdfaTeiWevtnKtqfMOVk1tegShNkeVSCRADUxkJMZlILhLjULCWm7Xl5ycQdqVJylBWKGpQRQvWGHBQH0bA2BLh3lzx1iAGLGksqHCV0bDyP0tzXVOUs3izGtqJIkq1NU70al0hAq+uiATV/cRFplJfYqQkspiM7UPViNi3jWxLH3z/JBY9ZuzfnIJFlu4uzTHYuY9ulEtHp5wS9QlvNZykyhU3FUrzVSjCNuRKm4rDMre2zU93UGNwL6RmwiyHvdrRl+bFONeLAdUsNvRUnTtXp/6mdEZrhQk8ypNZtbJYfS+mJ7TmOLQTYE43poPYIKwi//Qino7vU69FW3xQKX6mieISp2kuwQ8hwKOuRoRcfYcbSFzpdnVdPiNe4JHOgdMFTsco7JUmeJr9GX4pwBzsT+VMH3cF7JZoApHN8GUTX8mlDdsXuUj/tTm9ctLy9XEoYrUnanfZ1OvSVh2GrhngG7h2b22A8MMjgOB7bvhyRiFOuALGSfH9Fd1+DySdEwIph4gg9j4yBh/gIzpdNp8S/QXy07fKAqQpu/dOuGm1hZdocSc5cThCbiapoAv6jjzFDot/8ocOukugMCyVJPDrJtAopaTE4eaC+igv68rTrlyDpI/UFnyK0oBbUpttnn9b/mMxvM5kUNSWAmYENQVUB0A5kArSLFgF+v1Iq0AuUARuBOiAbdB1YAxQDfUAnsBrYACwGHQCzgAqgA2gE5gHdwCpgAFgIbAU2BWwHtgDbgG+CHWCQ2XlgP7PnwEpwABxiDgYcZv4D1oOl4ARzNHASdDHtwHnmjOAiOAcuME3AJXCNuRy4FX49sA4YAsPMbcFd5g6gB7SCR8z9wGPQr/0E6Mbv1Ay6+wnqFd4CevABGsLbQG+/gCPm7wEd/gCngH71h9OAp0D/3n8oAqoVXhJQDlSZXAJUgnerTQdqAF+ABMxVeBZQCBSAOcw7wFv35gfkBN4ErADfPLAWfOcbRAcgUAhoMFjobQdeXB7mKJfzlL+39BHAl+uXXvl/so5N/2RJKH9Sj7U5/fmhxseQzubqP3GyV3+yNnYcHb818XdREE6URRU0WPiFd/AeFsudUi8d0iU9clrOeDmvwh/o5/j3I+uDlACCuSAn/MtJnGbQK5KQllx0mWyGALVe/0NHdV5P6S3mr2ab+WTKzRv7k31i7dbLI3zgMT0qi2XGJmM/4qcTH02OJJk8mxxv7pOxZi5nVO+uXE++5Fv+Yx5b48U9xfcOczlXcPvO4MKlT5b+tQ+tuuv31P/sfKK5o+ltzbT/2rZ2/tWxdnO9W72S/o/9N5O/Tm6fLE0+TSV+8sNbXT16oe9i+pq/1lSu2V0rvLzm8quXT1xJy36Q8+DUZC7kKtf+3wTt3+9u8Jn+3Y2/nf3vQ3mz6dxffR1zIOSnYSML3sXCxcriT27//tLS8v/Ie5LPW21f/bWCduFHC+8WLhedXv89qlOx70Z3YzzaWNKxZPPmNyX/714rdS/tVPpqx8mMNptWbsr+FKedkpVby/m5k3l2ZbvKPpXh+ZdVn6iaVRAKSSFYeJ1wi/OFmlY1FyqFlX+te6L6/1M718+snaSNq7fqUuOPmz7YdKn1VvOXrFNtmL0QE7C72H5sPKdJ6yLiibarbQvtR9v7tP9DZkmW9Mkf8ydSN6ljtID+GS7VVeYxeC15vXhDuC2O5Eyuw33H/wX/C4KHBFHwBYfwSsgV6/t90ZD8jPxnknsURvk36eelRWpDlbVPyWq0P5V/Ux7X/2ZRT0WvsW7cNd5R/F8JytnK5UqbmZhr1suWaF232tahdU9VUq2orhW9J//ZTd2+2cZKRJ1qsiurIt89p76Ke843onpSvvxW/hFv4V/zNaJGxERCpIkMkSV5sn04J5nLOXN/cN58M77FhFwvs/VRpThMfU28hHnkBZWKr54RT6h3tX8vLZNeN/8snzE+biwZP2h+2XzTes76Bf1LesLJGDONa+af3H+2Br2u98D+tX8nmA+3w990RuJ/m5qXWpDKia/Hf5w+mv6lx8n9X1ea6unfujvZd2XLLaXdbnfb/fawPW5POjc6XEfoSJ2d7i91Z3vfVXq6PFeBDctGh6O36k+vbU6+0Jzf3LP+S+uP1rsba6eg81dTxVsT7i+5X7j/5+nxaDxaD7WzsPslb937ZW/K96Ozl2aUfmHgSOB/ULBUH7pywo/8Yu2X80+iumggmr9Tv2uIfyauENNH7xEqkRJbxJ+RWEG+YFCwST5PJol/UVgvfEg8SFSTv02+k/rctavUakpIM9fv03/Oo6irWTH3H4s+5HZz+7n7+W/dXimyd36VXOX5yn+8/302NfeT6t99+Hn+9/N7Cx97Ip8WycyaeM32n5fm6OW6L8pJ77/t1m/ojXI4q7U/EMXqXibzWQOct33KO7p78qQWcK2sTGv+kkDXjNWp2M49I1Q6L/q2WHP2BWDFnVwzBqFVqTVW48phjWgW1QgAq2uyBhveX4cDCDiwzY8u4tMErObflbyTvAPyR09/8Vrtj228o58s7AB/nZZNdnyWpFfGOpHuAfL1l505dxgxrAWyDzLMOTeGYxpvdRXF6aVDVqrJGoBWyAWyvl7CNcAdbjReNhsNNjaI79ewimqdXvT1XIQQYG6A9LGwTo9MAmeLkxvgZkmGleW0z8L4QB4biCACM6Q6Nwwi7OqW2+6F7+lOABJYISyTmeKVOsaMDKH829PqIqiLLw7X2FLIcZsqmBjpU8aYsnaVU9rmU2b++BFIPTHEGGm6HegMNZ4tcybgY4uhCgmKaDXkzOC76dpWsK5sS4DmV3nIm/OcndbuBmHXfJJvsTD+sdRuKFZ2T222Pgk0olNlHwdc6GaACrecfCB0VwvmHD5TsaSkXlNAr3mngRIfgBxbDY9VVFEFpwHK7VYqVwELH0G33Wss5cPV1bCslXILKfg2kHP+5RrUnzUSh50HK+9mhw+RztN6hTnR/xx32e/84W71GRAWuNBih/zhvlsGCENv0CWHUFbKHqtMJRP0DHYmvU23MNgfPIPzSIB66UhwB4TY6wYiwMKof9r1UcqjlqO+UvNJebyXL9okq0QcOxkmOrHV0NzS7zgb3ckuv976+mx7Y3LTnPnCfrNBJCWLbgHtXWurUmpJdvGoMwoqk+sm+BEgOUqR8UW62ZzHXHreFSGOx14FDUX7mkCJV7Nn0dVPNN/mu3d7xD7XEzdse8+Wzfjc37VU5IZ4LMe7kiuCMRhfJ17Q5pJA4mriTy9/nV5OQLhf20HsYMeyAzf19y6ZmcJig9+S6m8UNlRAndgQN+DZ05p5gshG6UbPSKFmKoKGvl2rA/fIHzQtcaKNkkSgCpWwBEaRe4I9m0dz/naSkFW/pwPUTGHwvETmjRclZYYI5U3VYsUcT8Q9XNxziuUgIGikhWj8/AIafZRmBY0jM7GUnbmFLFBkP5r6QLWpAdb272zUNGoBTDNqR3eN4RaowxPBagWjUW1Rcch/v7zyUuNtMhs23/gIkMQfsdI4V+a80am/guRmhiA/+R1MghTO3rpSA4EMWng7xLo+Y0MGC68IMN/7WpLVtlIHgT7t98OjKDb2GhTI4MyGe0s91U8bi+EWAX2ShzuIgKc4roCd7Mz33B7K+3X3PWnvXZu1EDjwb7N8QIvLJtjNUItzFCpqGs5IBcCC9itFn9xmXf4/dIhriZBnqyUZ1pFzGkHh1yudZljbjoBWtUMkXOaEX7xdbErxhW9KFBMPjPEyJ7HziGbku9OpneY97bc2JDJDIapthguGa/1MpaP+4q+szFdH5Op3URYsmnPCFAns1NnkU3GkZ09orWn62jd3Lu72nlLxexunYJG1Ya8AuDOBmsAZlKb4/XkH5Xpla0DjDlGIHJYzgkgUrKTZ74x0Kd61xe7T0lZrdvkrD3U59jYXwxBxrFCQIKJg9OMIyYq8+ilkjKGQk6lgVrxzpGDX1Sha1RJdSeo6jiKtrTbBO9oz/FZp3VAju6YpQBpSqYoSuhJgX+8vamqDCxuq3Aa8TChmHDlW/Youh4ZKXDf1lZhMVid1UuHblWYO0EHxUEIfIzZGHMoGdK9ay//aOOlXV4sikKS6wvapZIvQAU9hCQ5I5NyQkSWnpIo0tlaMxK5ZFVdtk3bYIoC4fJFzmvdNv9gpmmwt+dG/EdyAIpAFWqNSks3wOXndgtrphouiZyv3FAuQx31ihYZwBAvZMDSRVFm17TQTH/P/WHSwKtenDHx1uxjtpw6rVmEZ6ZUpPz9TRfahB/1nKNgXb+5ijOxxmdQGTwFtqiLtw7QoiRoC9fzx5xkXvEwt3EY6bKk0J20cyD6NVWjQlzEbBqtIfp3gmBgtwwvvaHv2qzDjxPTH3ITZWDUBpKukCof8jlRCyaAuyiJx+Lys5+vxdq0EXOSEWfW68pSK9UefUzldBRweqbbr8wMxwoBsnE733ZEKZkBOWKuktOnFjh+pUS1Y9g2jotCE08QyTwDdhrM21PkrQwq6GzGrk5RQkMRXUxGAwkogvxpfw86oxs00akFFXmWXXMUfqHhMRO15Rr/TPzXHsDfA6bN+XMFHmvXa683j6P1xLxiLyb4os9qZ01fSVCpjTriaUQj1chJ8+gmzMxT8Dake7+S+C9EbQjILkBE/bDlakqLfbu2v7gABfwBPTy2Zaf83aFabWmtZmf4ddn6M79SPrpbdU1GqSYnovxh+/gRbAib0/300uO+LdzdhW6vvXEZWi6qiGYRRVhoB94lK9b90rkT99iZQHjZ2K3PW86SXaE7bG8cuMMdG+gAu7135ygfo+hZxZDprXeEzeeufat9iVH+48h2ZNdkXj9K5Ao/W7n+jnXz9f7CmxDivEBeENMmyWUkXMBvNOJILtTShNjKyTlsKARilp8Z7FwgJBwesZEEOMYW6TRjUoYW8KWFWoGs4YanKfPYmWrdl9UkKB5TYXunDiVO+ZSKIKaoMMbvKo6bxjAuoI37cXIKhmt5P6Q9qMWL6iyOoAawnxEN6nV06fP8mQKfuCwOYKGCSopywHvvJcBJSb+OxRA3427Z/+Lzp2kXNNRO7thsKgX707IBog2dnyO6M/8PfLged30mDZafp5iYwA1eHTejpTeIraIVX87JfEWS3DYKgSm5K3d2OomhGDMgL9mGtQazxO+u810A4e1eqzq9+qe4OXz8QevXVkSPmcoXE62UxX90GFjFqfrg0li9ve2NoZ2XBQn58TAj1+BARufLyUqCKgEBIBLFTjx17Sm+a+CP8wcC1mnI6IaaVWHd9cHwsvk7PPx+qHeZ3Xtrczo+pQLABWjPv2OK3TVk5kaP/sX576na9XV0v1SOVU6vWqmDskY6cGpmhU4fmAXQKihyg1qms+B4ChJwrDZrLX/M1maLaLYacT+p7w8rveoc1rzNPOYmHpdMLRbIqMxogW16Ka7B+5/71L6L92VKnqDGlPi/jwHUvh3NaW7PN/APqrOrYEmAZcGj5ch5qgD6eiyTW/fHLiiY1JQ2DibypqaxqOqnD0FOloD8BDbXjjFhJ7lGNgtTgYYyoyuqaEb/ypl5pUpLECwWPdLPrBM+p7Fwq8FMvfcG3spNUq5IC2B7egfTBC0kTdnX3hxcQaSzbjjsXf/+Gloz27t4e7eoxF67aDBFqyvKVKxN9bdUtK99p8tUR1xIey7oAoEJH15ne3/y4BvnZoTNW6finnjU7vVvRo0zoVv9ucpd8dSAK3pr03AdPdt1KK6X/RMa1PVu2sE5TfNPo+68rGiAr/sCE/7LYQby4OOHH0AVfjzOwLg8YaVxtzPlyLGBaEEoTTPe5tA7r64a7zyMNze6ZMIRJgb8MBPZ8AZLDFVUSl6fixvWZhMF0RwdcMZZimKmqaoruhl/Hv5v2K3Z1eOUw713+Sr3UpEvn0ltUS89vAb0HP7wKu1X1+X1O9drnwsBSugt3LDV+GBXImeTD0bJeQTm1oOB1UhbFWFmH3MWPELhx/wuz+btnP6uoW5fZhY3o3uzeY6ulYeA2PX2rAIWi2F9GrVLFbGdv/Fxxpr+7e2/Nf/vW5nBPPgpBPk8WEsSXoH1aIKiWJXzqcoUhuDMr8zIbISoDPrmA6+MZeRQNuuvakN8qPLaf/HuDq/K0RAX0lx+8uogs1MJJSgHRQu99uq3NOUD0ItbUPTcS1uX01BdzigMHVitX9XNYx41CgIXKutwetHzSEVtAwhs7DPterpkpywo9NZNN48AK1N0wMsKExpVg3gVyeK6LwSioPpcnMwlR6zekdvXki1JDBDyvKDw6Ro7xrKTI3BMyrBn2cHgo3tT5+3SoAjITSsnmPaPcNQ0SgqcIOdB+cM6vsXFPNc1zVfX0QJwURMdUo/wT+vdb+dZTnyOeelHft+pTaAZKQNiwdvXrMWILoRLAxdNuaxe1xnbXtGNwfVJqXo/EjRUvtjHNvP7Fbu5gR4bXwKlmWqEhBiTKOybd5Jal7FxeyRSK/KcX/oYBB0ZzMd4NraWkZ4WDsT8IHlUj+ps0AnlJUhEcKxK4xAz7fA9Mqy/323BF+srclLgurTazBKF1iy69k9ktv6qPxFvzlrv6xa2NdAVY6snzvaGbr+iIO5McTvoAJLEKU2q3NMcNza5KRBKUSq59Bau5JQ0QeTXv6UEGWDw3AWXhAXvCnmcM1Cl1e7akO7pGvp4Gp9h3MWh2VAaptFa93NcY7rA1WO4ntOTF9Pm9S7VV6cvD16/b/Oy+fdHVoywQdFlT2JYnSo7lN+OzF3Imtbp92BRA4Oz8VJj2uZVMtHwwdBv777xsWcialW07ua7gzzOyoOuLhTB9cXnDLOwDgu5v5Zz5DF8CE4nFoGOsyZi2NGXxjj3ALeCwOdOAFCkJNTTg95sYB3bSf4hOveq2qdFSnBgklFaYMmKu3rNs3fMdxRcHfPu6DcaAR2bnHYlE6S7RFOXjejSTfh39E3i5pETJNRHqKWndh5Ztsx8haUZJL8F6kSqhkUvfIAksaq3w/R2psvFN2+nsulEwABYMPm+fKDnAC0hhadW2mAMNbTBm06h2oypKYjNAgli9s/XCSWsasY/88S5JrN3RQDty4D3ZGjeM6+Hdl7UIMKYtWdDpMWIVGXEp4VoLb8+dKFg721ycae/rnII38MQkaunht94ECG2qbqr1/SZeA3SBi7K12RVDyq0+NWP5v+bMb2/ONfcyKubxPeVB+24nxUeskxZPufbbbZKoFfSf+RLfwidfXWhkfCUFpvMjbgRlgLhvqq65bkZOmn3KaERyPUTG7ik3QS8QpKJ0/9IG2vMcpchSjPJSUYD8ijBy2G47AurTP4wprJKeMUFAORXuz2u/IZthDXyQb9EJmpAtyrisf85FRLpk+PBxkBDqMmZH9MWzB4CA8x0JPYDPecUq4MJ8DH8LDuF2uMW4Fs7B7Kc/XGdmwqo4dB0xci1No7t7mvZnr4k1p/Q7ifxMcrqRCgPRq4TnLD1kiOl/8WMIPfF8DQ0DEJS0KkrB9MOTaJ8GdPLx0WH4Wh4pua4p4hOGX0sO9AY2QooRGGEBe4Bwr121diEct8KBF7mRhE0Ibc1VVasmA1EDBmO5jHHywdz0EEj6H70vx+zZ6tbVLtgc/ZObl2H9Dj/8ElpayMitIIpFNA+st60L6B2GfqYi2SgZM0xR5pxnn3V8RRFEXnx2boLlRUmWn7lYbnVmwR+IDz043+zTIA67coeX1+Am3vtp0ejsLo6qNiBwMR9l0h8xHmL3GjctLz+ZebzYllD87A9xt2B9SrXH6HVH2a2pMi01LF9x7dEiBop/cUeEBxrc6PNoE3uRm8cLwRW//VobKtzhUCmbMY+FhOxbt+3xJjBBqzOpPXYu06YP3NxliCEI6ymusPZElicujdA2syBrOTSTwe/94PpSAf1g5GDbqIqrpHq9hFs+d8NKQNkJZQszmdGl8mK85b/Ve5jcHg2dra/IrqIgMnYbMDNVqQpDqr27GAk2VdFZ5vAUb9Q1zERi2Vh2QNuFVMAz88F+39rd6a8lXacZ9GsAqpKE0RFx0hDqJbqpEx1AVE6+PNj1CErlXtZk1zOdb6p9xVQt0Ve6O8ACCNtqs4MteqrFyPqTAMjh76caeccfpvfuX9o2CxnIpYw30PGMXUBQDgs4Z01AmsM89OKOUkWVvT361G6SGif39qXOsNR1fmfF+k7gZv1OTAu1XXjqCAeCFgUPAnYq0ljrVNGwEOEqzuHTJ6WOqUx1o3warTj3lKiSoNvVlIYUFJbMDcW78kM+Qm/pk+xhbIjhaONP/evFxgaxVVgcv4DFjCB03I0Tl6/EyU8+8+XAQrZFmSBWIoq2dOZpa/fGJSavrtCABcHQjUreTqTrWs9XJkoXKDByBL8FvcaxDtXYKQUIylakwrXC9xNxV2++C3V+GkvdeQ9IWH5Hh56+KJlDK+jywkwIWLaOa2i3NgYYvi5BtiD+ehK7gli5BgM1x4xwzNdshJCjFrbfogdqHVVWJYfLKJwpWSTUvYiPQCN5YZ2EGlMkBmzb6/U5dJCfLzWM5zz3cvcf7zFVbdAaFzMukJf40bt47edOoDqUIm7ZnrsYDgFMYQwnh7THjrNz3Pq+FlN2b+w3G4vjM2CZo9nzlmgAnqJUQqfHm2WMCOV4AbMiabvp8rq9tbOes/TA8KyYWSBYO6Nsylv74AFl6k7wLpOf8x/uv7UZjv2wgNJgujR5whx3NxJ3JG4V2lhCCZH8yooflYoTvHg5n6afuTTTKuan8xdn8YvNK4Pzh7NHK9/NN/bbfDlZXSFr49/1kaAwrlaaqE7uGvQFrZKj1YoldILCCnD1RRY73+fFbPyR1X9m1FIC71PVSPfMnBLA3dj2/JF8y7gz9K9DW9+uHkm/Gw8pP4oCnvSNws9A1SJ15rJyqnurNtKGhjEEZf3mkII2Hr9YTkn/RvjtD5Vv6K+IXXjiWKrttggIAEmURBH5A3FPuA76h7TEZE0xqFsYpC1x3ItdiiorOgI6X0dsQOUpJ5NkFBe3poSZgMTJBtvP4ZYMm8uldjUJ1Ax4wPFoKmI8H0tAEhc5LhCxGBtSQHK0deSKe1YTHGwdsYYAUvNPoyMQk7wXtqJqkjsLLqcOUXUFo2rtqMdBt15z2D60skWACbwX9TSR5ao1sfP8VvZevT4RsyN2O+309tod/3Z4p/SGGCidIk1fn3fKoWC5vgGDoSjKoh6c5vRGGhYuJPGXz+FMOUNi9PNKvTi788JMa+K+/bw+BdRgU27Cs2dMYwfV9dH6RAbtN1+Ey6fcdXxEQ5KYzgf/Ickt7xmtJA1KrX4TWLlQjaFb4cUYLUaOVwf5NB+t3Mx6ZkUsewupOVLamm+qqmyUtUIkKbRIRDwwo42AFlR7dScUlpeIPiUCNl7ulOHuhlfeeBa5ft0UK2tyPp14fJL8xDwmneLmu76Lapw1GtsCaA0ND5ok3KvWohN54VucbbF7xRofQACjsOdJ58foyJrkG1ybhpkRMw0M5T50y9hwkbdCVM66tbPViscnoP7k2sBiTNeB4/k7O3vUX9lw3KyvE1EyyoodrU5wu5Oi60uKxy/RBUEr0so3Qu1cQTJZWmDbYSMHWDIrydDXP9jPov3IyT8YRq2zNJQtlfAFx6y69QuSDUTu5SdYxAKisPcTI6m7dl6Tr10APJzqVPS4Dqkx6Ux4tTEfz42X0nn6QVbDirHOcI06oRv6LoD1ZYMGpkA0BhpTfO/m7b4XSef2b2UCmjneNU1MbQtj2637FbfUwQRe8aTVciiAkX675rSunp8fceYtU8ROc17lX/Ue+kzYH2lN+bKpcTtzhDeUoqkAF/otmfz5B9fzKkrVKtj7gZOGrr/3aHqpShWSuXolbZNzvuAuoIp6uuoAA88PLBiLhV4ZBYJjInMNa9k4yUbT14hN7dJp0ajijLgaOHw5mM0/3/uN1S3jTkuw0Vs/mXx0HFjsSb1uiB0lj8MzNXV13zXPktUNOKq/OJz2Jf984eQLJ1U6F6/SdXcDlrtICH6IffTDx1+6SNWNOqTOt6X+ir8LHFBS436tbWx/cdQmNanO1dWytEWINZCE5AXXjrVN77Bt73bSwsiL9XGmRXFPfcPQSqns0NRteq6ndpaStkZjX8kpsjKShv1m9KGRwbBE7LnIVGiZDoSIDNpGW1ww18wFbSNfSNerFb3dzAs40MerlmyBUsTfvhuPha64LTQJ4EZO4eQjkHDSUcon63FqderCH1YC3uFVJjBaAZYPwKPN5R296Q0yrF+0ZQKcyOq+AYVSXbRRPac107KHd4b/9eI2YFPwl59t0I18dwou3bOAWVNlFdqB1NUQ8MuaYANiLfhl0ahf6zypwIxlOI5djuSuMpGbVibLGK8PascMxUdFa4P2rUe9rd2jJtNeJw7dCWlX0nIKChCW+IYUA+FjFwxepmkvyyH6VWFc8ks20CPZvgy7O7z4KqqjN6sFVazVqv51TpEMR/O11GjtAflsosO88vuyXVYVr4Bj7We4y/yV7MjKhdxa2myVwyQvBspl7tsTLRIjs1XnBnAtr3HwgmTXCjS1qRG3D6QkE1t6Q+X35eZLk/ZJY4OaUF1qQ+wLRW6PWqQbLMvjvMAIAPXpqkikotSOfGAmedtBi08c3Dk+iUgaxcq3jpEj2uPFyIUthB5xDq7Zn2zLxXh2ynrNh7ydLGvY7rb0Vx1bVmji4dz0ym/9ZPN/E8On5zuoYf5LtN+0C6CCOfRKqEKGkQQeRckSFxeUT/ttIuyKxL/x9f0MlzpXbv04Ryr8sSNpRTd1WhZ66Ywv0N6qDtnyhg1I1QlH7N+tYAn6hLgsN2F0Sv3f9/Ce0nyCF8qdslgCTCKvqCxvsx12RK6bHHR48Y57FiKuJudd6eCZT7Eb/iBPx7NsrokQe75t9VwcSF87Htv7JiYkKGVAdQomKGGkYkhCmC5LYoqos/f9QdjKYuqinOy3hPRmC3cI+zlnLXkmjRsT4U+EttZSvOP1/eKad+9hg+o4J5+1XRDXrpCAVaOW2Rh2ymTT8ZRUS8qx4Cx3QvOn35q6+NRRhxhbGWjQ5w/kHBrtLq8PClK5XCBV0KxMr0ABxgxFicXOdZWvFe9pDk+V3BKNATHwkpny049pZnNn5tgfOtBlLF181v4axl9ImuWmCdWODqK9pjZ6efqRo7UD42ZtrT23HXL3dFdqPg5oDI8te7jaamnT1I5DvWvvLLb8IK8YmDOjjFAmn+q03xRTQIaIcTNzXCYm8bS21dZ7ipcSe3XsW0CKl3fKsLdXNTB0d+yw+0Q6gUt5VVbBmcTuDiMJB5LPspXKmyIWL+fXON7AgVQr4YrYsMvA3qGFMU2WDaUpmHOKQ+FaxNhqI0lo0/O6diuZCLeNTaCCJLosyGBIQdeB663BvWRxq69Yh7je28Y5/VX4oBgOQpJe9nUYJ4yn9Gi7Ssslr2kBK6P9XkgANE2VotqzrfiZBAZkTcbwLyCpFmjMyv5OkKpNy2lGKSaqphmbx5lTMFXDxHFu3zdHgWoU+B2kblXXulum7mpxHIqpSmISSRHvP17t5Lb7u8Ixd3xzX2/IJkPBCC7xhkw22TreDnYzf4vECuV2VuM0siTZOwqnFQ2TlAVRCO+Wu2oMiAgdkR8Tw0DqSawYuNcRsXqgD0oT/wUglYrBbDU6/4a47qpgqbjdUVrXPaCRgL4YBnizDcJJS5q6pn3htSqFPPR62MCOqFpyYEQlb+to83XvAp+GqVY4jI1AagF1Yr4tOECq+9zeBlybYxXn0FLRVuP6PM6k6ycCUEQfiC2YiakkRSuB443DJmiXXHgqNhSkHftxLpenRAQLomQg6BYUig5uu3b1JXMH33LCNUBAI3z4sUtzcY6ONQv7uPjzuAHEliSDB4YD2T7rDaB26Tgmapm1VtiOxE7QVUBq0FVDhNA8I1p715TXm7RRC+TNqwKH/VJNkcLaBRxc/pzff7YMOKqm1GBA9syyDYafHoBJzoLwWWPzgJHMSG/qKZ4R5HtG2L+rzuiKYVqW3mnQAc+TicdMIr8cJbioXv1v0dv/elwmJ4Wm6fCaLmrZdGTvwtCfq5Sqy8rp3l7PrlOaOZOPO2oYNiXXb+Bx2KN71BBIgfgAxNX5POQyV5fOnPw92vK8Y8sZt0JUqlSYY5upmRnWHv88y508uMB/3KN7KuDyjZIPKzo1KEqUa2m5c/Fo50i/h6llp1cbE1k0U0mWgWuSfkJImbVOOJTdD6goYbHsSu2KKdl531VV6Wwm1cMqwAr9ehPmRawm/tGosz05vH/DU1hoYXLe6ziMizY0ioRkRXXKV16w15kStlArbC24Xkm0aN42vmrYjMdWPslQyvmxxIFTUlhc0YArL/Zh+K7xnLU7bDbC56/HldpPTe7QU5TXY13Z8Q3OJ55sGI7VTpvDdmfQ60y2A1UxCJW3b6yXdzhQjbf32jJIG57ZZGC+6tdczlrAFg1oTk/opx9NSFpdw+pbt0q67fmyhymWhV8WF+bW2lUtU24IQGIUCbIcrqODh6Wj9mZg5KMEA4GpahykOVkQUZQqlJMy2APT4SEeqbGcGp640qzKLCXbiiH4VkXMeTZwhSCPOWuyoPmCWlc0VOOmeE2tY1hRDkv9gWmGoeAJjed93QLO5l87mIEtckVo2AmuWgpt9YbeIDzuf3fJn8ZhGKkdhWTiY8VQcD5v3ZC9FjomOk1HY8NKw3G4UJIsnk2F2Ny0XrV95IH9WuyEg/jdZwVhtRglP/n8zPL5miW5PboIPAbdc9hM/CAa1gARc+X/i3H+rFG9JoGRuI6w/m0gwUhwz4aX1WyPUS/+tfa+GFGlSIhhqqxZCbLTui9EAAoqZoZKvMY5VTs7kbf8Lu63GsPbh73x4WA8frfX4zdWVcuMybVmTEE856aKL+68HNJtPXL7UXp48HuVuGN39sxR0NKDRq1ZcVcMXQP3q5/f+7B3c/HSDv1TVvO43qKIb3M1jrt50/UWa6KBabJD7O4Mqm0nY6qWbPON0oZLll+ZXcWLa8ClmqcHzxz5oqPUmfc+KPbyFcKJrYE6iIeG6YdDp2WNlbaWZiM7as8akLGwpL+3Iy7HGwlhoAQa8YSjDa8MeRuIvEAEOIa6SRWxQrWkxJWfM1cqVzvjmFMfaz9KpWMfd3rxiNM40HuE7OywYtbGNiwOc+oQ1VOtsNFZ73C3uPWlA+bAO5Rj3PDdN5RI7nYf8BmPd+tX13YasgYmzWpXTcvNUggS3sWUgniwl5ZRXK2YnKmM+mvOSDvi+rlb5L31bMaRUE6v6sDEsrsKRNHYLaGg75hsiqG1q3ve/uV+yd0i3ItEKFdNPi+6y9fKkXRNbN4q6DWQbzJLg5Ymq6IkcvFN0SFARZf78KlQqU6TCiLcyJfOy0QPpB/3sTP9c5Zg4a5svueCabLzqstW+lK3y9gadS5rSxkRiU70QGhtuLa90OxkNigzQcIp4T4TAWib1jxodlZq9qUEiS2GS0NLUBZkkFwZT2T/C2RXz9ufCVCx+/+0MbGP74od/O859pzCf9NDClLEe68zaUTOfsNUCv2TmToicctpAMmxWXfgo6ZFDfCYzzANJN/bP34nYjKdrUqR4ZqICen3MFcgLdJGZq/+ggnSZQmy7YNK0sBiogH14ANABw6v2ScEvLzTpqmwbaZVyCLzenewu8qyQFrEn/+YAw593NbBnmtAjWVADV5t6JVZXjEEDGsgNeOrFC0OjP1NS/0kXZA2a2Fmt/N33VyovqABLyJqhVFCgcTzhIcRQpQ4dJwTX4pI0Me/fHUAI9wx8ht29Pe7vnIPKR/cNyWt19ZL652ZBACqSilyhlVZ6yu9+UjxfJKvewoYuhmTBVS/Jjr+IgojSlVsS21gSbfkezvPM8zZmuAhOFTS4PzU6452MbQ19IfSBmWZ66zLQyUCMuQLuIsDUNLHXApno4fCtmrGmVLItQcJyEyFTRUBpWkywpCYrASWp3A6KehFp9bIpVl/u0avFuyo5uS1b2v7AJWo4grbR+PlVgHw4WIC0+BJUuVioVzK8JjcyzGugsf+YlvKMoOicsc80R3sEocACgJMMSTjF9wJLiwk8A+ghXSElxgO7k//AKawQtraGgKnbGMLURD0aqn+VswRfaiPABhbJzGsgnfytJpjX9sgTJJ1LtgIpMuawFNcY+kWdlQDNKopURYQLJCA+38WzYzJCkyIbWlb03HG0l1V0xw3IxlamzC8WfQo25LPwCYBiAlNZfiRKSmhqgwAj101MatYsa9nbFfll1JsJekhPlHGIgtwTR3HZmIaqlM5nB8LWMqTmz+5TZU/6TRADEpyIBU2CpOeX8R+ivLP1s0Tb8DuYLTrIxkhRw2hbPVPdzDG9KBYCGQJwEweYjX3OuwyKcUY9SLWBK/owhNjyRRJ13mZIfbXbikAsSRayPVGA+tLm4wlHap9gLUl5syr69HcKdHhPrlENYsLU/v1JST3f64FITEsRM9w8LJAKcPZq1uyCgziiuXAng534eNoZnvxyEbhgHtEgXLf0QfZcaAAUCEyY1lvfPNTbV3dihG3hbq5HUUW9Of2VO2CfqPY/TEm80E69vu1H9pAyLQjjeQLYUiQ6Zh5KhRCAVl/m8AIKKB26HftmgsFk0aW/tLsXIIuXzGnJ9lp4K+8WR1AO080F03DX79IJjtccXA9bFVBEOtFF6/fYGAsFukraLOXLOyVwB+2MvXJdTpy3AW7cV0HaDjYyUJWLLjROiAsSCm19eyMdYVDEMs9+LwF57C+OrJd+8Bdw3uyosgEZZgpQaR1hRkYuaMQmAJ62rFQDfK6FEu05E50P7Fprm2Ck6meFzOFAf2Ny6xyWdd7rubPSKa/YTBlX8SzBDgdKokYyhSoGkQu5aPT/uizSwSofZLk4H9i667CftfWg06+vdCBy+GGHwhABlXDjpwJSEPVM+xe/qkPdpfZlvLYSLoc7DGsKpkKBha0Jt202wVQTdYpLI8HE9MEVsnwbEblnCplveAAKnPlcmBlPG2lzSAAsQ0vMp1OXoCI7vgRZ6K/TduWapAQ38CvXZXRPd2av+ulvUVeL1qJUujc3Seo3312OERUF/0nAizQOuL5PjDohB5keU82LUzYkdl+ZR8j3uSA51UHf4Y2b9ItwNingP6kCMSDC45rlfaOCxaIvWeKEEEJK0sHH3hF1uAFUoOROO/zaJDmBYNtb9hNd19IbtTMFtUaAnU4l7wFp0Xe3IMvvcjyaY18XoAYcuN+JbFuwVMBRHh83fdZt2/U3WBkRtbb3P4PaRL5pu3WtndX2S6v4VMtQIDJAqntwUkCNwRtOcaCjtxf+RjJbpxo0RV0NpE6xQlHigD+jyf7qlGm8j4pBUvwEyTV6M/KZmBBMPXWTQzz8lQ1uxchHTaY8RpjwKT+FrTANQBzvEKRCbJm1G0w+UDjooXEgdDxvzQJeo2xCfYwYVahIWStBHoBFcs76C6Yuy3tsmN4KxRhA+glqVUcCqldtcKmw1OMadZ9aFuCQ7qH5HWJuCQx7TQagEMTtJ2wNZD+WJZA8t7Raa8Fp19ODpsIqBirqJKagqmhZqgmF9dP9lxFhSEItN3mzV9uolKMu/Eyfkni63UYApyP1eWnr3M47M5x13gHUoDctYO5jkEMEAeWmqgLam1h7NTxuSCor3+/9v0cQqsv8x1YGjfMNgKKQqBYbRoGeo1hqGkwKGmYi0Z0TEUr+ydlkBOc71AYri7CQYoRIuQ1GSIdKg0NNiJt4TQAEum/IkHApsgojjCoELEq//IeATu7QmMVs9KS1ygfzmCVBNRW2HfOiG6um50PdpquZrkd4tXeaF3dB61tt4OAlRF3AvtKD+g972w0I9WuXzwoQ7FMkMXvE/o+mo5cJHAMCa+RvRusSt5x8wzPTX6Nzzqs6qSrjI6TMxCF5dyEgoJ9Gu9cHmLBc/ws9ktTVr7m+BxlxFhcDC7HVuU0hp16iExN2zwOnKHyMGRNRktT5PAY/K3WEQ3abAej2bT9LOVh1lSunF4kBFxyUls19FbwOfAORkI4aLWdZONY+cELzkxrgoDL+DblQ8uf8LjZDdPQBYOQd3QepgrcvxzAJaewkzK+u9SNjhLMBZsBqrJCynppqAAWnMDf42TzCeGMZz0ZH4E7321Z1Z3pYVB84N3y9X2SztzWkgI/kzjnPTruSyi3JpPLY8XdTsHh9IcvQ+/WrSBFNdsbzMf8tY1tBm2odi/Ff34e7Utjv1qE3QTHY8Kj7tjP7A8iYjSK82MHv1FcOH++3lrYfe8mEFJ1MYT5RLhpoxWLVfc6G4SyG46dCeKb6fx756NLr7pB/AaDAdAfi3aynwE4TPXtCTxUH8nyu2ij3rZUhwC0/SQoMay8IpnsLpsxP1q7mITcZTkE7b0BT8W1w7kYngYJzdXVULolhJPbbyIooOtXfhM5RevEZMDsE+au5fdAc1f+jTqMRDwZDa/9XjZRseAZSeYl5uT2EUmyjI3AhZp+YW7rh35Y+GI58R6K1hrPe50hWTxw2DQonPxuToOAanmUdjoZoNoBxnfuZOzycGAxamZ8oJn7+7e/3bA+On/sQ5PHQavQibYMDRkAtPDKdQeDu+815nHy+fOqWtrs7wIOUIKvK8iBgN121eJgyNSCvxuPsn3GmA/EuHhuDuqCpFQwZbbJekisy451w0iutQDdAsnILENknnVK3Vcm6uouLbHdTkmEtgZTnnoolQ1z24ohJ2cUFeM6hro8BUHxD9StgU9kXCpsegzUqIspqENvecyMyb0s7BA6r1xHJb78s7UAjJFWUYIJUdeQAfXBqMHkyMEFF+kHfVHKrkBUX5m/AgXHa3s0q87wHkwDoOBooUxySpAprjOgNOsEgqQkIgLo1pIkhTJX7j7hGZb/PTt+igOrm4r+F2hp+NG0C+WzD9d58ID9j5RjZAT5TatjfuPNTgPLj52KkEz8RYHrFOpASSoYTmVVap+ANEyH1bsvQBxwwmotkQT52uU4vjF/5kGCAKmAm/keHObWYvS3yLz/UU6AjIMw69XEK4CBXO6SxLhzH4/2viJCeyC/5udpGqHpSzeSBDztVZ0FRo5OCbM7D3TPj1naVLj5l/sRX6TSBJL95j2MoWA2LW5JJqdGMCinBVG0ugXLwK0oKrzod2JkwWeeWs2+D72DCBJVdDY6M0UZFBUzDX7yjIX0gWi07eMMBjnjphGwboh49tMgDt2zgiKKCmxTjoowq04/UNuptuk/DyHrbBD1xazXPXH2zlNDxtsVA256IpMhWn4NiCaTra2MO9gdq85Xbx8bOXx46Fn9rmXNOON+rwlTOcuwRwAkMEEAtmIBZSJeEOnNURQaj69sWVZoZ5UxnitWOzDa2R9ICAhYJuxIZkwQZaLAiLrY7DIyrExoXbUBZBh8pX5e02O8aEYUZBuu3tZgADAYH0B3NQnFFw2sn7L9dJlN0ppUY8LouUhn49BsUaUgArS+zUlUwxhOhwFdLjVjMLuZGcMLYQOKqCP1x/FkGGemwlhBCyAvFHQeSyyZU/HFuCdiwT2RQjI3MklP0FA6FcCYd6Ry/05v6DGFxuna22LmrtHLFSismVIAp7V4DdVgK7e7T/UAM8YsHvgboNoP1JSqUw/bTQWp+s2bz0J/tF9eQm3uqsC2afRvV9losn6nYoOcXzqchwezOzeHckzhWgodhmarnrkjAlHbzpYlk+1Sj9Qp1EBhyOhBCdRAayM0xG+GGJLWcfERSjOvJ2jbVKspL7gxq4Ck1rQDRpf7tPh9rwcI4nmtD9sr1VLoX0c+cWYxNn8I8Gb8KEfNlp7qXx2UC9CgStSAgo5wcB5VVHR2cmhK97wo/9RCDkDzozuV5aFoRT29n6yL3cfaxDuOZ1re6sfOI0ZmPuChTiuLHDotTRSp/CuNEWd+yxXA+lrkbz+1ZYGYBtyzZbsCR193ldnXtff55VYg7uuLfZgPKGtrRZm9i568IrR8yCR1y4YEO9ppwRQjtD30vQ+GC0Kzx74f2EKys8+2zpzmTOhFqmLMmU601O61CMgUD3T0GsKkL2951hdOVRGzpb8go4vCaxKG3LQJLGCe5oQssggKd4lr1gZ+oZ7j5he+vPbuO9aTcSZKV8pFxnqJu0BIYMy+A9wcXNzdJzErIBb2XHussr5Ufhp61kExc+c1W/NtMMTOxuao7b9/izU7FRq/OT9CfQ/T6M7HBVxeWv7m7gMrgKY/W28yl+Z+D4PmEyL5YXgtnkH0q4VSaeYjH1HcgSLJkym3899Ce355DvFaU/UnKPGpbhsVYfT0r34ztoup35dCYHRFfWa/JUq2xgRaKnmIzdoy7O1UeAvl5WIJ6hMyZZMNbuCLjctXRoYDbACzB8tmnmNaN6+vgloCrwbHR76sHwSe9pRiObYuwD5p7B3cfa2+vOQHRJnt/Sxceaa+bjF7DrM3d7jnwZ+/EEJbPlE/TVvYFsFI/9bFngX8lZcrHXj7/qWwBwlqkhIVR+pmlrbF9iVSBt4eqtGGWQbTGnq8oH21HGLDls+bbV56mtT4RrOcZEGzfH+1DQtyHaN0xvLMbjBsTjoWkJM6NaCeqpVs9P0/lX2d7krm+6ZY8mnzJr8c38DeMbffwUblm90UYwBX1TANKT/DA9tcz/2X136bl9GAnvZ3fWn3DSNw/AbqsFzNl9nW3O+n0RNsp4fh8SmpCcNW2gmA3a7lk2XwWM9LItQj6cg3jkGz5h1VQXv++NlBtxuji2pTPW0yZkCibTC3Hp5PixpVk+GwFikRoBiziSY6PSd63BQVE9WDevhr1cA4aPsWpbiEyQnrfIKbAmlYTMUKWK/AGSKSx880ndMvq9SSs+89GwBHbkzGvQ39gdQDwK4hBKetCJWf9L98O4GAdfYSEi5i4Fq/EEmYqcxCOUtZPb8Nuf7XF+1IwiMOL8MIHxbKKAUhY8zRCLQ19lroe5LQERkspdTtCWOPJc4GBNoRUbKRnALaQ5VA7fmy+BhWz234u9edKeS51+OH/eKszKihl7mFTFSSiBXKngrtxx+k7NAp1XMuGhjYURyYAGrR72hIhrFqfXdVB9lAdpCFDYBtyYYwy10Sg+oXiAtB4qgTy+aneAW3MNPInKp2JCH32L63DfW0IluIMw1Fh46+YVS7IB29u8JeL9zku8nnaAGSVMcSCtZm5RdqF9IU+SJJdJ3pTznxCmuwxtU8dB3UsT8BJnljUkJURJbuRnN1SWaImZ3BE5dP+uLae32wtiD/n7UEHS663cd/D9KFB3Sn3Rt9OAuEUTfdQIIyiDCcBm2GHo5nZKgJxWYC4+BIK6IaXt7rP1r8ZWC7dpPg/oPjkO9vG8faZGLMN+LO0KRxufGXvQgl/HlTOGK0ZAeYguiV4JLUxSsXYjC27XoX3UYEPYlMuKY8yMUwMFMFD3M1JLLuto7kyeuJGGyZFxY3eiDitwbzyP25TiQOXWblmLQdsWsDJ8ZdCYxqt7IRPFv8zgTbhCvkl61V6P7Tat6Tzy+PwF80q1gW6k132/Os+/eucynPMQTHZwI8C5V6PH3wx7EtiAC7aTvdI2ALJTajlfXKbx8jzoMPghkJb3twW0L/ypTKdXYRgV2JWBr/4zakFOXAdnxt3h6BU8cfrEzg7LDYvtmXd2e/szIAgu0mga7+bwLkNjZiW6rHB38pPS8dLL/WNS74A08h8M+tOoRCdkGBpVRCFgx85exllrAbKRIPioaTa+4YWjmZ99Co0ckmiqSXrAz7vVKPtVDe8EriZWOCu3dKwU+4KdcfvRJ3qKGsq0CaITqH4uB5WiMF5uBw3Kixq/x3pmkyLVdyavF3wrvSHgH3lGTzouhczo/wPeyvAtejRZyPY5Ft+puesfuzxFWrW2iBqb4UIQUAz240GujF/cS8RCVkr6kTYKitcFjBl5Eo1iU4LI0Fw5wAh0J4wM8D5WoskR0hbpoU0mZOU5kvdhLQ/LEMMtQustuB1m/y7QjMvZcVTxA069hCLzl7QNKoU9iYbnMj92tJinK/iwHNyWpbxTY4tr/PYvAK14JZEGe3Y4SstOcz3xNErGxMdbE+Fkvg+BKTX72Nxg1Sd2CVc+8zavPhr6UcBn3RgIY2TbFJH5sIdLg/lpzYm4P93YKLPg7JI0s2AgqKBpcFJ/iN++SUJFFk4VNp8d7YA3cwzU9xPbYKRYWXKlV4Z6RPC1FGrGaEOM660sZFLQz5E8c9sWYEL738W6GEZ4SfkVihfvfUcYfpH0EW0rjE+5bRq7bjS9S/MaD+n/qt+D8IWCzbdhAMxRi35IHe00AffN7gcTr0f3L8Q9gUt4Dmvz7gIIqmmYIZifZyhwPWFsovnNxOiOQ/a0PvFU5gwmS4phDSpp94QaomtXA50sySQhHAAphEEiipKFNFyD813AF+4v0lE7NYcw5IaFAryoBrDYNODuTA4shv/fPxdSgFltk/fGpmBwhPGNrWjcD1fqLoP6JtIb5qzuauYXMGCBGJnUPNWTe9qQPb35eqEuDGdQoroM2eRWZ3Cdgy+1YYz9V693bf2jSwZv56CSx3/YgJCtU0aOd/voAO24LcfW982yHJ3eIjWBjx3Bg8GnePRRy3yH3/GqicvEma5UBj5mKqa8dEAsotmc59qBmai79hF24oagXt+frQ8t3WaVzwrzGZIW/0Kubgy4cSjEUCs4j23dtvwyvuPnCD57rddzs0gZzHMq5eG++EAYN1sdB6YxMT6H0THMhEKTvzA9fAjqXLoezft63rN7U/bUOm6S1pE66c0/xdhNDjR08ev7YHKiMIIW0WGlo15Bwcj8IQZYOTgUmBMBYRgvgieNL9GBftlVTOkwvBBBgQsknBjPi9uJ0r+tllTABGPBtrItwXTV1uHvDLl3RkbUBf4yHJXgTinxZmrxVQYjDATTgEj5VP6Vs+Gm6c++2zW//5lW27wbqH7+Pu63po4EHlzK4/Qt89fraoQXn8ZAt8DVzaxDM++NzaUeEP+0S3Fzdc4Nv1NjeGn3xXiMFaYnQCQomvDedpQczPVTUgWVhYSx7W03dSRb8ICg8L3G0v5xBGQaBfqZoEmcOiGthpf7UwVsCmLPhc226kneypiQGkdBdlBfiqh9pLVaIJQVcXFX2GDuA/O6A1XWwyMr9SBG9JgGMvmCOWqKopwAQ5rdnyGAUROZNrsWPRsd59jbXjA21pwrV1dv1Cpx5FtHK3MAYF6JAC1+gi3NlhclU07pNlHWYOeY6hXR8a+QuUVk9TwMsFEr8KGAh4rahAFk6T0Q01nek9fLzigx25BYnx+309hxzaancoptvAUqQO4FtvjI+zshJgGaqAHyxzJiK+MMz6VyefmvvDpJGPOpCJOAraPqfTvrJVfn+EHk8CPaGn0BpIRo6+4FeLclrxkm1pCc74rc3LhwyADDUIrJC+DOL3PeNZ0fgdFnFpS3Hg3wGW5XmUPX9n+qAIBSFfHqptUoKTapNDDyy8CMWjp5FFH3trqhDVxpdjAZbn374lopNNpFNsAqTSHst+CwyzjJ3/mihTgrziKpcK9EtODmDt/Q8jbTvahD0Yv8U3hk+UmRt2FjpNF9MfvnNs1H5rrKrHg5ofpkBBRzqrYkjNmYW2BozgF5mFGv6/oHAmWcBOaGaGuq8/XeTaI7+4J8sW+wmxEVKfvvZuGLSph/zbstHbWj7u7IEcbPrXGAuRtGVSgcMSm4rQhbd7ILI6Q1S6+kKuYcSK3Hwqj30g2oxj/EkmAlq+tiAjp5PikoID1/65T0z65/em0GPuCA2oZDXdQXoqjb1erzOTHZzidnBfMsomfGtjbPjBqHbYNvQ2yMNA3rYz+qZbt6pbIM1Zq00kEc0NdYg26G107yr7VuafOLLtOq979sxlFsEGqbCX+bTaq39q8D8iSReKOcATtZLjgnjWrwZQ1jHVkWsGnhZO9h8umI+ACvnKsk/U2LJILd9YzukjgGyDh/4bkzBeo3WDZ5vEBf8PxP6/YonaXADWFvzCDQ9zSG0dfhiMemz0PUZBvoYA9VQL/fXGAr7CZbEPg368ewE2NBhMG+WFKVs8dYoYYstsqSPDQ6z4PCpw9MlMUwhegEqSf6UgvK3OuD/2CTHREVunW8oW8MYFkYKaqklXYAGEiiIhXq7sKq5dBsADO2FQRacsTt8BzUiONMMZwtZ273T3YKvn8+dW8KliFWUdcyJHNIEDnjVrl6mY52w+DGYY1VD99J8/+DCMXt356LNvo16XHhRGMk5N9MySSMaJx0PI/IwlEBlDBb1X2Kzwj119zAYENV/02h2XI2Ua5LwjcSO+KE7K6ObzqzCDHI7CCYNWRlNXjZz94bTIUczxUbxsVeCp64IHm7OTtRJiapcTacSBpUsfe5zMu94hH0bxyVOaiJXGy2qSXjrBEwcn10/BPJxd80jMIH0v+lP5fbdffoMZBSqzGnXgAyA8NTd27g0Zha3TbRWqHFxUmVagN+wKNbQPCnoRJ+zq7imDPZUSLXjBGNzV+7zNNPXln520kZdzEFdJpATOHvPH28jUuodJD7SL7j5OPVdQvbWB5DSpR/At9QMqKmyb/+oyolrL3Dv2AiLoor6P5ipehR33ULaqaDVqL7zNrfk0LokNtjSyN+ncl8c108wjdgyRdz/T2HqVQVOg6kRmJLRf2ZoYzCbjq1uATS3uH/Wz+x0mmhL0sgqeQV87lBD1qL7adPWZUPCgn8nrujqlFBRVZTX915fE2qnV03+UTrpjB3GzrwSODIBYl1ivzhRUJ5KeLkTpswsXq3QI9oJosJmi3ZAZKKoOS8G+ruX5yAcop/X8lo4VgUVYdioaWzEkgVqOkaoxHvTAuWwvZnInG9uHFrBeC54/QkP+HVFUgcsq2OynH/AY6Ev2xBSqGXV9rqxkEnmtbmsmU/wnHl2vmvCN7ZHhj3xTw7auNUAWBrO2nVO33IZRWwcOb6ymiB3NDQyINZgEun+TfSP77x3gMZ3xcL4SkY5BmsT3QM4/Wei4wFt+WG3BK581ZlxIUOIQUxyow1VKSPh1RgOhfpYgYJbFlEfP5IxXax7Bw1xRJwT1ZdqUCKydVEGHfLfQhAFcsz0nZTpGw++kg5YBJFMnGtSTfNVCKEHVkA7yo+nneK3RY91Rya/oW/UGAUI2DDlxRadaMNzYNGo+GmIMq5mFEqoTy3PsHjjN3KUKQpNVuYeTDsJIzJdqTXh+Cm2lvVI5U7CigT+kF1vE/urX8bI+qasd98VwT2mbI3Of2xxwXXFME72XaAmxig3feMLa9Zvu1IaZSRhbTJQW7945VLzh3vjortfBLbWVd8JALlqm2DmMFGDKVqBZZKxs4GK5Rw3amwV5xp26XNzeLDINdwawfs2z0IhFgg/8MKSNUzIwlPPDhQFjaob5AOk+b7uzr2rEMxP1BzcZ+sCFfrNEe/qlIQ06xlaOkOy1fKJprgLBNrKE3RhpobkTpb+VBO2ChM44NkRu7bry1ko+XBV7vfUbhu1CG6sBdsuC76KHGp6fkW0E+2vlRRTOVHVQVvHmSHtKC7o/vX2i1rX6H77qh/fpvIOA9EA9NsTY7aoRMn97f5b/UkX0ddQMa+deNVgqw29eMIZxLR0abJCBeNs04h+eaxKZS4fMW7LAe71Pw5IdNsmi2XUjyA7NmIuCIlTB8f1H5MA6FG++Ihtg+5xN+g3oR+RXzo/zWeKbTdNNB7vZmcf8NTztPZjVqeGGrxF53b7FXHRKALJeseC7m+OuTa9u0DW01h0IFmdLX3erZuXfYdIYkpStDnQor9QxdMcG++bWP0c9dnRz4pr6/4s6NlT8Qk+FuKULNv6ngJ+2KD+e+Nrr/OJk/pxqFBx6jO/XWYUfv6KmW69k5KRCYSqp9tA0zApCZcD2nTuSv/wI2PF8R4MoiHWJAnrx11voSQ+XbXge2MaWj24ayjexJAQaNrBtvP01OX2PUgW3ZhU04AMRzKyTsVDBGsNfWrHABTvSls6ct3S/wV+B6B3WNnqe7Uhu3Ohmzp59+3wiMlxR03wjlb7Z5SHDyqKA+onii3K0/FJPes4AoZAiSJB5aTMcyYg7boagzamjEMFk2GyHFjeGJn/gNEWbp9aB1GRnx/okn5KDmrQqSbKjLxFbaviVf2LSZNpB7pGdTp2f8oXa3XYgAir061vBD68+aBtak3zQsb+i5ZyW2RwAxhBVwvAhwcynwI7lhgZUcnQCrdhk38j9GxRMUCEvAgvYJAMXHmj87+waq0ZexFL3vQ6CTweXti17SqR1Qt+JnDiK4u/+oZA10Rg90UbJtD1hQAixNj/xnb79Rta4eUq4+IBJ/4+4DjMqHZLNu3bDcWynZaWkIQQFXZAVvj3Ha06Y8kj+ejxxPChHDV02ygXD8Qp1o+GNixN5IiwaoqZrtqTiRxpgueesWRJt8Gk2L1WMiYj5vGwKFGa4KPB8omdqf5gH9vP7yADFg/5IFLnTcqLPj/ba1CTeNv6twXVjNRJnH6RjAILcSbs/QaXDv1PSPMAU6KCBpkRKnX0gXqlPswAHmKpokfbcwcS5osF3PwUbwLXBA3rjaw4Te5MrXa32mUflLnTzdDhElbO7Cisow0AnbjpfITV39PS4HkiqfhmfwHGlrVK2jW2GIbBgmu4KMVujtKOrEBNKr9wSgJ2gDeNtF6JCQbmFA9ewORf2/+PWZHJDN/3Tw5APfQ++8vYHPESXJ8I63LgiVrvokwPWH8VWNYAHvfn5qQbXkEPjc2cdO2HTM/Jcllfn1kJzeTTR4sklHkBX4T6OWV3+/bX4BrskLD1Z2TOfCwpMMOU47C3jHT8nfB/DD4G/BYKk1DWDqcGMEaQpm63KI6uzxN9mpGHN+rPVOyv7LFu4TN0RjZeKKVBUNpIXHefCTd3lJOD0boa06rWHWcR6lwBTeYfxHuil8jwJ6bgFSzpRS9mR1LUaXGRe0u8OA1Dfhh9uTrh2vfpBz9A7Wv6gZW/oK27FLIqSRKVZHIK8PLlq8Qbr5oakgPd7DNIEEG6sg07H35nt20ASf6Xag3fvt5wJBZCgpBFWSUDHWdoSm9cIDQS6CawJswwhNfTogPpa0cf6DZczmoL0Am7weL0Zl0Cbcj/f8f+zu9pp2zFbQa817JlAEpzo0EjxFRUc1HZQ00gH6ad4x/kWo6dywtmTAF0Hlk7PF/+kDXKbm8iNgj38HDzRqzRFNtrqWnXIIacUs44NpASndAiyUyVoSq7pngrI5KR2yQuMjGrKaw480aG0UNhFpSD6sQNzgXqNIqRzBIZJ8XgMEGUew9va9gog5TplDYZQ9eMYLSA6kV8vQpmdqXJFYMLO+RzERzblz5DvsmiM5kVIjrZyCQ2nuQpoNzv6RK2o7gduFaGNVL589mYna4jNUUxr9KpDnhIdUP162YTK47eGSMoaluGAT88au3fRrevJRd7VMTe9VR69+8BUqy6QYbiy1OIE0DvQ/dSnUA6TrLLzv1SbWLVz41tbnhyFHN/ocBkcttEtocWesZbDMUt6w7ctoiFukHsa4bCiFWggyhBPR3AEUIzviwgtQgbkx/IvoHEo7/vH65ocAOrVDiPy1DtbqVOKFwRgYmjbillD37qL4YSJKUu0ztFzRO5DvkYJR69JcAzoWttB7LO/SlUdoB/spVe5DuLF7s991Uhgxy0HdQycqnujZPBavN8wW8CCoZlzLwtIcOTJI/txntcxmfWuenAOdnnaL1zOh0VVMEEEuYwmpp51LsB6pHwjKBTNH05PqJl8HYrAhEwpU26P7Jfi1ZccxFtvz7y8CZrGf2D+GNYzmNio8eEmr7E7amRZzVWb9lTgWsT+6zd+TEn/iGoQ0vdbDjwZ+kpq90aTtidsYmASEpgsdP0g6HlhwShNucqqp0GGkQUBTQiKb5JiqHpF9GM1leNA1/ZW0kFhlOPLUJtJFIS69IWjGeX8my+924Xcs3ycyDHcvKRMTLGrLzkSSQLJ5MV/e+5y2rQkLwgBPTCR+DylWH2TzXRd2V0wxW6VUkTiin4HdgM6fImssz5RtzS29OCzNtq1caO5/kPOo1jcLGb9IGkOlylCKVdq5EVPmHFtn1PY1JeZDZ6gZt90GrjNXmPGLVBcsjAy+GHAszUSSkCyx5hux/oK3d28+b2gx2TkIkgQZ4ZJoxV+zvWvLEUnwPwW2Ej6+s5TeZN3GolBtKYFxi1lJYEPgFnLFQMeMATDagTd7kpyQ9wTXc/m3usnbIfZFoNtA6XlRaJcabYbmIy6/omHE40qqqScu5ezn515t9PFC3ABsJ9bKYdU/IgEQqto4dMHsFr4D6Pltr3imj3xPAVBFwMNdxmP2KBPH6q3ZPXNQfWx+5Oj+ov3Sn3UasyL/pf70CFeUZkX/vjWTVcmzp+98Cf9NNrcBoAmpOVViJiONz/yPWNUhDW2ZJhFM/Ag2disRfNvTAZPj8cvf5oe4xlekSH9MC1cRwRgKEmqof6QGxd5TnAibbMMpLU1LoDylrNPJzw6O3UL4sUf/ujvEB2j3+JK/JOTwbJ944MUdLLaQed+49uLxKqqIzUNIr7jB19bTq6hCmywRdMsGr773Go0A1egmsdP/jiUTOzyJ8g6GtxveG1tfJiMC3wizEQgw4pg396ltPVqCGWgvaMKYQNg1O6uYxNdqSWAHT3xwMps3bMIOkX8gojeUArgF1dunLahHqRXGm+lV8Vc/7WFIjmnoPs3eX3F0d1FASSNLvXT6U/s+zpgE2O20RSGZmTqYT2f5U3gfBV4zGgWmXu206Gu623fRONkJxMh2z4ZqFu/pDc+88oCK9CoqEqaeA5L5Kr5DqfsGGbbAKAT5vhNIlz59Zz/FX0A7mCgnstDV/8kMXTjS/nIKwY1NFWPzqKfGYgkTsg6RiLBmvV+vOJ2rw21h0h1MAF/3h9Aw+Lo7yfhcpHro0/Tjar51tLArSF49qhPuvNFZYzlvT9qn0j3mv6dML6nkPUOiI/B0uPj7iAK+Npx0F1rzwxG+d8N3O6KDb8ZSrmuz9s3HOzPtfsRacj0L7nmgwVvRn1jcSQskgwUtmycjqN5PuatpZloAIzvig0a91H3E/QC6LAS/5dvwQQwrhvo2V2x4HPfjOuKxDAYdGDSoOlfdS8Git78GZsExZKgbHxH4d/mzVXmrIQHhpnKUneKYneMPGkLGZp+I+M0PvhqdqiK/AYsia8So+4G1UMVHEh5n1SMvcy9hPXbFhs1mzh3QD9ZJjHbJnI32IPsr0AR5GZ9qlcIoqAoiRSAbwGMy7JkfGsD7ODTA0v9rj/o31vTzp+DOPLJ4Q7pzs/rfzuxpAmW2h1m3uKAtO01vKkkdruTQpPk8fb6hnBJiizJfoXApzEzdV3orejfGZ1BYaE8TOcNJQLR1UUMA2nK3fpDEBnOhkSmuOb3zkGMD/dRO2niw3IVEZzpTrQ90Nh3H6j8B7TZb1f6Cd6EkwH2hEqS3FjVUDLcTKotfCf3NV1USIW46AHICliEXp245KHCRq/KbQAdFNEy4xiARTsUggIKx/BlxvxX97qJGht3A1CYHR7q5PGz2baPuJcQJvRMEp6z6k5hTVgH7iu2U2o/EDlF4dFetxVkBXPMejFZQCRgaUnm0emwH5cFSTlGIZ+oKwrLcV88ezkLabXPuVAiHkXoaF6fAoZWJ7AYnH0xRkMjd5pQOGlKpsmsyV9TtRNle7WowpWNcrlU86/k4xLMJwLss3QncwC55J7NID5qh/roG499ld+15Pq3QQF/CQKaSo4d+WtiiiO5QVMqvZCZ9UVm4IOI+3ycRr5tl2lHdKsldISIdvyGHzXUlIsWI8XM1XECWBHLEPUnRufy4gF5/nNcCHFVAQdqYKEvxOSuWkC+eF1gQFuy8Ng/d2SrUYwtN6haGMPx+gfBNF+2yqD00i2ZvvdiLlp4YzIE6LUFD2YeYfi9BRUI8okA6XRXMRSktLVqIOtu6huqCNZ2xVYNtL7mrD3bzmjn3OR84XUX6uczQK3upbLDjgVv+M4BysMyWzZKJUt5n8vHK3AjELqFGHoBs6nkw8xKMZlDxA+eCBWX5q8Ua5udq+RGnAdloLysCGE5OGteHqbPHjkLXWhyc6LYAEqEtbeBTO38tgb9/RZwMJL+tw7X8eVgCjOp2Ll5IXfWgnqQiKyfSOfEXP/l5aLx9w+/IeCCci+A4kOS6EKMSocELWtOzIidI4FlsUikuQ8pw+45EaL53q/vJJ5h/VblrpnShqyzDqRpCoycWcZrRuEd9MtPb7Umbb+qwkH22B+cRBNtuqKx1xIIf04LwkTxCa9MfNbIjDXA4sUkMGRJwiwypirRGEh8qpUmY3Wsu2PLzFC2SiIHNWW7f5pYn33qxKWIfoIUOoUOOlpjwB4MQA34SRutuGmg88Uzd3eKaPvmmSTzYpH5AUSqBjUnMM8mYgyUWaRCV/KO2No8G+8JWuPmTKL3XxIhfvjcL+28IjO8frbAtdhd9OQfE5Jlk6UbKbRwhHljdTHKNfbQTlimfdy2oRmOng6WJO8IbegLadv9uL9FB9jU5RJ9U06IDV8ModzXCvcOR4E2CIeW3QuSL3s4fBJfjr/SQ3XqOA2tgkHmreWpaP4f/Oact3mJH43bs1IC1IoQMEUiOYRfX79ywoFCSK+kr5FeFnPjU7PFI+4dv83lggKyuySCZFGVATadHm8PHj2YhDHcEcITLampmm0v7186KyMiP+5AJj035t7PQsq/W6Ln+01csGo/Zt6xH0/vsNv2VRC35LPHv8J1x7DJvmxcwQIZq/iqoiG522v+YAm40prMf+6cjtGvjh6Gfq3P8a+T8QSPgO69S3asn94CcHATU9Y5OqWmhdBjZKHuVaFPNoiZs1VlQzPYcreFKRAS/UaRJdOu6EIjo+o2ElXXJxS0C+02zZ2itdJ/hEhrSH3x9kTwzjdjqOZUzUVmshn4zVFjJrfwekrBjwtTg/2AYdA3xDP5s5a5pi07ZaPQobNJNFwpF6J7kmf+HVut62v/3QG2qI2ms0Ic+KSz+t6Z4xTdkOz1Hz776Ywa+PoEnB5pN3f4BKpp3XCQnolTb9Ttz7hDUxCIetmEr2WFd399WDNsm1oPlGGw4DhPGxm/adcOQIa3cx0kFcsNzcqFoVJHDzwYL+WfIDJuU70XT7mdP55Zc+JyH70oTSJpGldHdfmk7ksJY2qZXMsltmCYeSNws4WOAZUiDWn03KxDJRuKya4FfBsfwWEgsorCocdpE1YMn/xIioAL+gXQfbE8XBHgOODJ7SjdE5mU37JRXMMGFz7HlkYIaSKerFd4ZtYV7y+tZFBHeQ0iwjHSJwii/JVJflLb1jYBGNlTDUhk1ui1UuNLJAzDPveEuVPYdJpNgOnXZVdK3oQUOKPKVWu1uUr8EvQArpuoSEL36VkxQHjCAQErxAlk94jJCmIAx0iSiEbDYVwVZPLEeuHGlSZnQREArUFDDV5/mAndr8ZoMCH6EcHZGzvBRLYWDcMwEQcp1ilb5WQlFWIfhKqvcm147au0J4AUQ9c0ojszIKdNrhNKZgqCBWRwwM9rrMHune415abSBDwcz7le3mXdlsOrgqwXDdvxKrWiYKxOBIM8m1CgQ3lUGMC5lJc2hh03GqkGoKFiDSqmyJuoLVN/eupqVsHX/J+o1YLCWmWzEmZjMfRBcPB+rQ0vfJLxeJCsu058x4o2nKOcLBQkqylM4DHNVuGMk04CGtD1hl1xvYqb90QXtxVNksyi6XhVjisJxspEJKTwKQs65fv5LgwVhVJkpxzLSERYC2Y5aMQqgLqGVSjrkmCg8qVQL+R6pLa1DsLVgnkwD+M43vzUcmxS6UgQGJlfWI3n4SrAd9QiOLJ8bl9DT1HHg8GHmLA4EEhJrO/jhZzbguZmU9rkmHC7Z4bQpo62wEa7na0OThMrujroHueCUuIp/Igx/4HsbQ22la8alTKu1a6W4gLcDSSxrkjIy3FMgnpHEEicEgEd0pIgJ6Yz24VkCdGAzYFIQ0PBIK7wso2RxvA/8qgjVWfVadKdgDPGwzUIzAEhMENpkm3fnxZz/RTMEnSIjKb+9lMq5Dvg98bfLLHb5Nu/+gRWVt6/8bU+Lvjj0V9IKAzEOy5d4mOFlwg5++kPPClinJFFIpGqWwmKJqjGlLdX4NYj40Rcr2yemwjxQjozH6GX1l8DSEFarnvKEi06VOsLuMOXMBSdBRrVZJTsGy3X/dTSievHTQ6wMtRO6aU+CttESmgW55URe20DwAWPL2bd61lIqaslP55bNZl9fH6yrERaUw0yTL/+FMGGBHHpArTd8EZWrBh0Mo7YiOg6PfxE1fyth+T113gvA00y2TkDgjElNkafcNan5YZ8RxetKkgiA/5XyDhfePg2aaz6bc4PmzC8LV1DoGlc+DCBR1YcOHxsBjBOQ7VXvQmytohT6POv8ILLjyBkBjFL5iF3sr9D9RHy5vFhl2ygIjNVA5nRLfsAKqbu+0w+1l1XEZ+aWAZTNTbIgF3JuKddWTDlFMOdaLTDbmpBHR9TuorC8GB/wQVhTaYySwEnx+RyBcfbSBMX3mQpCtfggeKrtZdiX82cKtsuRcHBCFdEN5w58Vj4AixCodCA+SCy/zaWTbjYYRVjSkkmNIGpkiQ7yFS+2Idnz+we9+bbDLJxx4048mfL4b+n2ZDgeg0voi7gXm22Lhy3VuqpomRN9Lw6ZtBlUoJ+SrgsegQEwpX+degNKFz1OnGAZof0PuplVHgfAarOccyAyAksG9/ojGLWzWlr0L0oRF5fgu7tpFlJaoho5xyorwUqkjGlIrl2nNjlcTj02/S4WnBAN62RuU9u7oIsMNNSq+rK5OJmquVTdcCIlhpd9CXQ/99pqZOzIMWLiI//zkaePvxK4r4ijIN0XysMOqPWTCEGHnUbeDtuS7su+47FuQs24S2oN9UtwW43cOaym5LsQIXyUoNECzZbN1b/Azp14uFoXAWw+G9UfwY4V9MekzDPH4tQoZTmfBQEvqL8+OwpE27KWXJ7hWbaSKsuxihPwiKuFqzmr1gX5HAAzUfHlRwyu9uNYLg2mCnGnRBU+WqrKPf1uXFTP9TDfD2mu6TPBwU7M2rg7OWoCFgCIitWamjeCmVkKfEZ76U97Tw/AF9rMsz5LFRvm2ez6gqhMigPhykZyFbiRcSXE0WVXIly+rxpa2Sntm9tA16vB1jvr9+Kt1tvLobCd2Z8yf1JEwXwqK5Bx2w71ZbVv94RTECDcVmqGsI3eu7dtrDpIF1mCyJrNzk2MBVZcux8fGdNjcNd/fBKc5wR8mJzlYQ4L9aH+5YLJuOI+avEs/7V/vq1Z7EtEaaSa3aahHrDBlbG6GuR7mWcbItGIYB6pKrBBfMo0IPwPfhRYh+k6tGO3rox0Lp6bOr3DoeTq+BydBN2ecueXI3OXd4hv2PRXLlE14yrFIORHgQYZsGxRKwAg1KtHOjopu69XA/scGGiob5Qx7Zbb16I7g261M24960MtsyYlqpis5u5qPd8BUAfe1dPjM4+e7g5t3kbz/r83W7U6TXcP/LAXqRe+CkOivQFcpDpTcZCPm9dU8xXTvePuLdR/VuJQJkFHOKBuRxvMxSLNd94w2S1H98qE9YsBhS2MiDUjahrhpKo2FmoXmP6t9o15HdQpyHQYm2jkXMXD8NbQ64VZUu5C/YA/ns44f3i+IyyxlHNVTZDd0Pan6bM8t4LFRTbuOZ3oO95Cuw5BWniCD14kLq9AC1y33swsSKksJzULMWIQtcoD/24olootnyYXVoFCUkQGT68yuyr/7e/rdcILXGutdbB2AjXS1gD1JW64g/ApjaLWgTbD7o58gbqgjJ8gmbOjIbJ3wSlfoL41NzhnvyypBJngDemQ+wwMYlYG8ypspJsgDvaUsipsZ3gINu4iLqAbExKkxn5YFY/7E68iaNlXhs+2Lq9aw9ngCx7WJf1jEnUcRXG4/Xda0hiZEVCMYGXLV0qTbx90eMkSc6Vkg7ggXHYGhpDnvYWOVKu/hdvlhSnIz/Bxv8mxtK2K9ZkucaXV1+e0NApb/pWzYd2zsAqQvTTvxTA0E5e90PMXI/vqglyRfOKDcXpLGh8uncMEV6FJbGi8k/2de2UXeqiyI8vDo0qRsY/soc/0ewonCIpFzn5Oq2kaSmXpKslSZN1FZqqvmFRKFzNRr1nJqWEIjM8O8UdpuBtZyIDSJJ4/AzFS8DFYq2SSWJT0ex1wYWpgk9V9C8386emSIL/tLT2ccilzklQ9AnhZsLKqFPt1U9zdIexZJDGl4AfTjeCNhoatKpesCO7aqDeYMvVk3tnfdvtNvDLucGF3paOVHEsz5eKUfdiLbhL030mj9M3WC+IshYeUDnaVxtoOCW6cOiXtZdkOox9ZTXjy+mCBzECsiDEYHNoC8++EIVHRESXsrVJwEjmDoLvbiCVCvsQ1H9eJd9aQgvKiB0mwAn63wenqz/a9pr9XgHBR15/1sDW902ehHtpDyQwxVIJgGvuT3wzpk4q4zo6CmEJ5PB4aERnpAEIbvJby6l1VEWaTKBArfBrK2iB2mPRryh4ML2MtlxdGx/ShBe2pLkt1Q3qXUtNhvquIoLtCXTbEycfoS+vztT4TM8jeMOvwLm+VI7h558k/dMogTiBRvnTt5eWP52e03f93X4ueMDaXV6EpJbKgGFhLm1p4lHkxK+0MYx71YVIHu9NzILfPA2+3rJ7pJDy49eibtQjVhY8kERJqDMUximGoQwyAjs1ljoqBLBkEC3Hraai1GoBASaraSfp9NdHu40tAORAqYmGWWvOQ5ZV5BoawhhMpKwp4BIJLPmQ+ZIkyzIL1Fqe001pRuWmwkm77EPJJJr+P6027VbQCNOo2d+aHDbW9WamYlSShV6RkzjkI5V5WbD1id5Wuso3wCU3et0EuR/5PT0A64hc3Xj1Ys6SV8F8q6GkXUGOYwIhMdWt9gaeA1Zbw3VErO68eiFn/fnY3ORSglXocI6pbeEbWLp9U1V6InM2G9hbn148a0rt2yv97FOnV5ab53509ua+bCO+OvpzmTDqS5iRcsXWaL2HvP/YZBdN1zGH+CP1dY0Uzg5CMaXZmAGaZEsWAHns6do4moGwj7lCKd1UzRV533S4HESr9Bsg7R33DlLjTKnWjo+AjnNWzQJvjPjcsTs1h52aDRCtYiPahWCkv7MXLqVqx9pnD+yT/ptcLkmZDrLl9JugCrl0Ra/IIAaeExWlWl4p5/LrwyVxYYu7N1QpzVpTsw1bN4EEhGJB6SH/+FvFMWxmsOqgTWfiXBLj33zEOU+An0ikFVQfqzSwMnh1LlNVKCZsaWyytgtz1I7IXS5r87YcglPe5Po8kvml3MPwyxhyVkkmVB0ANXMNiDZg4ts9KW9XV1Suy6RzbecaD1cC252sI/NRnSOKDMvxIbpQDLvn1Y0W0ZkNONYiD3p0o7b/iKA3v2/ESE7X1tDQTgMafGnlxklQqM70+FH6dauAljFBeRj6yk6LOa+5u0FtUeP+A3fHdi7ytu/AvVXjNbojjj9MQg8Rg0zLXn0yCJ+lQx2/+pNsO1H3F/HJ0Bh/xroOjk+HnSl9oQf9Ic0IUMjfTcL+Rm9mOPZPiCRP2YGvA+66AVP2DceG0QGDiDisd8+6rwa3fZtHQZ/RWuhHeoXCocyWjNSa6Ob++q3Zyeiaid+DDB0+/wiehK+uI2unG6DD4wd8St1D3b5WBOjnTs4+G5Z2XbHjo81wHRdWi1zQuRHBbJ6DMDUMTdBx6p/kHDR/Ioj2DLBBAWve0udYF9j2U31lVSi4TxqvOA/9L6V//tkx2qfvtnn/yr163PFDGRyLDMCWwWvK2+G1SXeKlZnxJ4LWbkmT4arzoN+Rm8vF433yY+X7tX173BHfEGUtcpkXiD8qIOXYa7oyNxKW3hDzofr0L6KnAf/M1G2fiPYgZmFsybb9cjAZm989i+7snft6UQDXeGbDMBbwvvvgE6bXiBFi/z5hP63oadAH0cdmkQLR7pLVedJXyqKDWC/1E+K1FkexgjJxAjOazCqcfW3Uqa+GuEZGFy3ahX9E1CH/gTWl9fHHFynN6TeW+8xoEuEQn8Ty+Pa74TdO/ytKBepKMJd1jyBTkX1cHNcJw/sK4ACaCbK2BMQ5N0jmO7kfqNPu5CKw28XG7mJPb7NR6UmPBC1l0qDmG7kQcBiF5bEAR2h+wUOVpXFRWASJhIAFVoxyhGsA+40qj0aINHXsgYVRqa3Hk/C+B1hZM7tSJ1OgioYJIxG3U0S7OYU/XzsT+42XpbWE5FED1clZrVesS03Aeq75IaRuH2PbCBPHGEIUOXTUb0cGHwszgJoCQh0Dc4BVhT0xJjOKPAUNTCx01d8zL+wGJnBK+22b9znjvFKZURRG9xJsuL1bYpY1pnZz+38vmfcn+hsGZR0FZTRxKG7/IHHscuBg9Z59sZ3J0WW9BNGonTpoP63X5kpX7Ak3ISmC75MsPmDDkinEo7/2R94uTGgpiCAt/W9vn2fLE6xUEWvCtYUJki1yqYi5DMOPNaa4IQd0RpZN8l5aiGfqxtfTI/+5kekRIR9OuPbM6ZlkVYpFMO7fnx05YHToteoAvv64HikXMQQdjYikuFsYL9OaWLuOqUDUgmF12KBPzDP3QEh5vdxL+VWPMeq88GLLZLGK5i2DZvvDfC93C5bx2a7VjLqdYT+53pRBDGj6u6TQIy/adrCeOMhLsV8Z3uZM1wbivkfVNkz2qGaILuw3D289EIbi9MAbX1ulmIhdo9T2DgbDE+0byfHJkP5GrY/3255gYLzyAmUIWLvRyeFuvgWLDCdtuWbL7zWGbSvJHujZfHzFRu0o2tHinj1z0lqpTwvVcgx2Uzk9PD2bhu2wg/tHuEEqDQHHS9V878ALGuOinJSfWINbk10kCQ1esA8p6GMYNsc7w3KE5CbSUpxx0dWI1zO+u3rRJNBgwMWgqUUCn9SSuJC8aLGR+PPREHSXEuc7oJu/g3N9sRzDzz5OBbxbmE6wUWYLqVsf/wfi9feuv3+cCx60dpcC0s3NJdGO8li37vn4mfq0Gk5jqHP/S/JZUMN1ERiGu97LjlME+Y3j3EyB6RYL0yPO9L5ELx3j/33VUMA/JcqodCi4ITDu2IhgMKtK2pENjGm36nvI7GgGynXHqdk4ev1GyZMH4kzGXvVXoFGvaJ/FfjmDsyRU+c3aWi0D7PolT3F4WbP/djOvIZtJmIPXbpQpIaDSDDN2FfVFWYKvq0oUtYxP6pN59sLqcbFcsUjeD7pTrmw8upC1ZDEDVkIxVrVKARmsZUBe2dJQd4qV7c8DVgg3sRTjVaogkkk8g3dl+z11pQPViLCjxDL1EaLOw+d1CEitgkgC5jfc99kWqoncsyUllbsB7c+Isz9T077xmv+7Y7z7QJHspsZODXfWNgbf8wwct3bw2ZHdkfbSsw/Zor8OBSxhn5fzs+UJZwBMp39fBEXbuBE/5aCdby6PW5GFrt62gWukPY9/U3UT9XBBFlnuKWtTut+5AgDZ75+UXxzgsQRQNK7A2nRvq7ELlIHk1gNPJDzABS7qapynr04x9ni2s6E3DMACdeKCDw39hQBWJPesiOZA7LAHXyMM2sPWTKK30Zux21214OORSee+TD/o6Xozgni5oOVblyregnuwETaRpNJht8Mg0OJhwNcXooSp5/sQeeMFrMUgustU4YLCdUGuS3fHe3rpm3FflRgHtYvYJL7iCgUHcwcjXZWoY7JkI4YWDALKsjIbnQY7BhMFQliIqUpsG88JWYN2fX/9QPhLv4zzqszI/hObgFykB4BiE7CyRBTaJ/XAxo2vZklfRwugpVZzKKgZet3dzdQfDPszibhrlC+JreqfcO+L9MOevZ4KhsglbAXW3Suhsq94HMRMVnYA6e0w8klqA9Zblbn+Fpr2p0U3HY3RM3mm0UvN9UkU2bN6YL6YRMPC0aijZ3aHhbqhRJ4O83iXEUNfdX33z+IrXyYdXC2PpgvmM/StXbFMJ+OjE4RPO1RGQDZN2jR/AAqtIIVAego/6o5hkHqvFyoz0Djrc+gbg6QUhLrXvmThUfRivwRujzRFrLaLu6N1KNpt443Y6t5PBw5mx7HdngXnGFcj8I4DdcHZRDI+41XcwLwpfMr4UwNYZfPhta3ge/T6ynC3Us7JpYqUKwUuwTB+5uNXqEJ73LnJ46d2DjYOAH7So+iKISuwfCUjDQoxOnJ/vDnLnUbmsTxIbWzYr1aMmqSPZHMX747Ga1V8Nr+wmF0D6IpbJhHBoCx6qGVSK4tnZEkRFNE6snsiB4BNKIXjEtCZ5i1EXPjkdOnMLOpUgOFua3JA3HUyhv0018Pmb1ryZ6Kdui8qIlPAmZI8DEKgv1TGRGD95mQ6G4C54M+RMZNcAyQosB2xjFsreJJ1ZdIBLQinAfQAFkq76IkanpqgkXIiXDRD9cnDGgxcjoSCcOpOn1VaJiE4wEfcMSfDOM8lh9rEuKU7gRV0EIclzMdADS4c/4O7ggtcprbdbQFwchPlvBL/lX4e1Wh+Gy3qMoIxTKz2Hs/n83lsDFbRhKsgw0qN0Glqi1vnswBENOzGYHXw87H5yVzCVyRKDfAcvKBcopyuoIUx4LP6q7/lG5XTQHZCf8Pc4RuXqjZulk7uPOmYbIdInuwumE5zoHuRPVdfinX4r5/bgiuxdjeuJlyETtxx1H0tVK9GwT5zRlboStDKGAXGDnv5G6fYdHZVam08b5vLobyxwnPCjap+/ZiW1MsjODwKOOZuNg819NsjpKALNqy0fTJ+YBLYiEqoiu76hsO3OfRXn6fqr+UyObduZZRS43PjabbSehkjxHF66cMIHG/+p3sZF3M7s8Ct0OU4buCBqHfttN4tNZyvCuv+ql6cMbVFw4YaNW9W0XCHWsMtdcadNVbAN2o0tIfZdeRcQHL8MIFKr9wRV7r0g4rURaVDYnt+BES+Qoqwbq3Ro1C+ab2wIR39jNOUFTvRb8ghG2hG2omYYtcIZ4VOefFo78ZybihWB7vebxr+38zv5LZokeQKrgx05U3bx5Cx4zX6Igvh0CvZyQMo3zRS+BdByrjgBKyAZ1d0jZwSlDqGkQL70nwOCPUC12BEIbkZsyfXCC8pMlEWEOKOZ8j7fTK0V3ZqRRm+yyb65rih8w0ByM2PPtJj0UNT/I+EyampWLt9Z+64PNv12H+J3aoP1n/Ux7159ghUJOmEJ/wx2s3CWgBe1Z57K1ahSyEN2ZXKyxffWtQywaJ6WVkENfrp1RLUZj+otKAzoBkTxNYtjXjhTQD6cAx8gZLq6DTIUKEtLRKa+YB9pJ5rTWkFGVVTUh04pg6tGkrxLdymwo3mZpIDLAjxoxjgDEIsBBQrT53QfSMT5EEls+UeX5LqBA1KezsKCgCLzcA4E1jFRtzbJd0JDrAR3p5Mb4OlbsywcfOKK8B6Hb8Qoj26zVwN4oUru321Tg2WjGmfKGOntwWLCbVVl5wFAMfxqeDRGoU0AfkpJTWQ3jNlLwX6fXtphNAJMpqmoFlQpYqcCgoLKEVpbdvP2gUA+tXGdx6YHPr3W7Fo5/kLH91f0/JqvP60kz7hmLm006Frda1lxFIzF/KPddW+tG6kUhcHFlQoA2VbKMqzMIP018GJssDNp1QFUPmqDTBHD/THCdtNxif/dJrYZev4tW2EQpYvnkTuhLs+S2HipwGtBMtR7hZnkLoliAz+bJEmaHSiRmqylydqPp/rap2y5a/FY+UNbbjdhmdOvPZsEgtPzOB8okWm8/BvXzCw6VKRFtYvweFr/EmMFGQ7wuh4Yc+ELCvLAvLFYoKnuXVlQ85oBtViG9DFRlmTy8sJ1lRkZ7SEa+KHTprLqOdNycR6t26VC2MuKkBHtcdKGK6bdy82g3HUO+WHnuCOlh+TTmarlhYjACIvD0mIxj2Wpn9LzC+3GuWmoLVz7e3aiIjeUh2NT/RNFjxJn43vZUI5K7LYQXAjYbZaZH95tlwMxQa6ztPgOuJHB/slJEOJUEKXWVVuEpOVq8rriszvN38/a+vtN71ev92ZqFs+kFIBiWBxUqg00EKsUU8K7enFq+6USD73TbxPRoi6JDE0vchXY3v5dNBxejSiPC2yrKRRy0imja2kwVuwwA3wkAB7NPZNJJ+9OWzj25PoIP0h9v/zqh9PU1Yx7bTJGlYTI5DinC+ceK/3h2I1QytTSrzYt4sGHdnWY0GTMTkv3yhGVeCisMQ/3RpzoOvOSbs9kW5Z8NOraO63ZxyWuyV2Rm5x3tsy03AXVStDfBOy0FmVOSGbXnTrFil25n5nf5Edp+wK53zNxWLfq5vKrpKdyMEcpqmvAjNrPHnXvddVKipSjNTkW7JMfWFzE+qbUWLkJee549q6caf++mRnD2qb0uiV4Er+iviSW6JcLcsYWHhRQzyNZzURpUYPJd8Kljiv8Z8AeBa0CYDacGjlhmvIqzlupw1YK4kqLIsA40wfIQ9cSjEoIAGmlgF1svijIy/J0gjUwv88Nf0fG1kkg86Nyw34CsN/ZlovmwgRZR9BiFauwNQHRND4MERV5HsKzClrFEvZO6q1IQITyI20rmAb6QCax7S9pZ2PA+caFiP/Jsqx8bsPXT/xWI6cUhWdmNCO8p6GkMFwxyjTZ0QuzZy7bjkfrQYe4I9RP0C1Pa5vBNidOefoIQ8I1Ju5MZwJmYMAcX00whGACaoyIDaVhGR0hKx96uWYyoY9jggsMNXrCgMMNSzAoEHbnbPkgbKz7EKGwyS2y5CfRE8VUhHWNXxZru8KtIb2Ia17a2sIsRBbjFkQVkE4YIdYXtiIQYRhiR/zXrQzDbggph9k+3Bi3OoYSCxzjZIzJB07ou+WGpNlPW7E4xFPu5hxV1IXqht3+2A2uZsg4Ux+42gXlD19mm1WWhdq4Te5208t9thNe/UnsaVKD7yAfR2/3DIkP8AT+C4CwOnk7Yv7ejCwrb12wbIs9GoiTN92R6tWuSMAHlSZ7qjNMNMk1PqghkbYZUMQI+QBfIGfU4BiEt1m6sGAYeu58FNaFUQpp3yNqSFkBTZpurLHpi1xSLK9TRjgaXbZcTDE5FMM9oacJkp7ct3ONbD9Nd2JbFK1mYgF3HduTla6RsdkvosW9G4fKbCqqI+f6GsgxTmU5jmpKgEin9cRwHSraJdE36319KcAbf/t6hGhbb3TR/1veiOSUJ9UPtpI72Uc39dmb9TZgCSEKSDk76uWSt0qVF/Xf6n90nZCTsu33nCKQhEgcrx+grdNN53PyhAOeBvVkbVKjulyrdz9Ue2UdlZmNUR+lymBOYzxe/W6F7H3FThYDuUWrHzOpatPpyxSUno1FF/kC25FGdmHQtsUyHG3WDN7tNF/WdCDgGe/MIKtgWYrRhFHTSdse77Wyk3xkGqVai9BrDVEHZPhpJAd9jTAvvK7csBeY5sUFiOSIGX8BULOP+sbVQOueYy8qfcX5+ClRnbhp6QfKsn6Eg9iI6CXFVZj2rrsAl7B4nRI04RyaKjdaKCyL1QbbijvULUCommHa8PEMxrXgK18st5fmmAIBZgR9kyvpTXagUk6t29n5o6dnPJK0rYrIJHI3ZfZkakpMJU4ttfycbM88p3TiZE2960LtNuJAUWx7ZnFFdltH0vD6TLj4PI5uSgQJqQ4WUDqBt1ULZDWOlYQRz07EA68h1/3IP+x+e1bFdYpvylZuClEDESVTakgEWkDcnDdiqgcrKkTQCzBvt/Aef//ji5GemouQxrRmZwxmUJqhMcLS3lqMC5sIXZoSjHCUIdXGKrDJZM/LsYEMlinyJ8XP+VVPe38SNdoumta88Sf9G/nHx0LgGnjNr6/Q89rMmwXyVcoYfWcek5V4GVyrYoHBHjOXBSJnQy6P6lhbzBT7R7NiqyxmFVBTxffRoUI/vOq1e5RbQlAOy1nIPqubwZsbnzzp1f0NedRef6W0r3nWpaNhMTfMR60p13drjDbYVsE3+1d6zy3U2+QaiJgl0S7pjUiKV6wXlEI9JOeLnYYPUBtv2EKJZ6Nv7Ss3ek/rNhVceglYtDogYb2/804onJFZ0AC3wSsitctB8TSTsmDJWeeSSavF6nJ4N4iWUzEse2tRiwnFRVG44kCm+PLALjTS0jkRb7xei3C72uEoLRLazYY4VJbgsINx37xVTDD0MU/g6rxWGTLStiRijgdWvs3k5oywsNtE/JOW4JtDvzAwh149UrOOEFnIhvbo03Q7lr+lOPoKfI6T9ohwyXCABN8vvHKtVVYpti9UUSsbN1vbRndSYJDIh6cNXir6mFfb6hKCCwuKBvswLRyQK8FNOMzRT5YAZH0GIMtdrbIlAb2JnU4Fq87UUujZV4a5ZQjAlyzpzcYWOaTnEudqG+1Qp6W0cbL14i1eArPE4XXYyw24GP5gUK8gr63n4/B4e7EH6Q8rrJt47G79BHwG5xqqMZW21aaOK5GOT92jH2vcTqmGQO14OU0/o1/2nkhPsgNlIergw10NiQ7E/rLrTAyNj8ClVVgPZPM9FzwxpbhtP4nPnHp+LefRWpJWTPYt+fuOD1swL+4zENhnBki/ABiGLUoXqM4gQHD/NTW3O/PX4CBBMNqgWwvoI8Z4+3mg+6zIAw+JurEsBasAeKjiMUa0hhacv/dF9NJho2C1cLak6FWKEmmCSEsTPzyJ26PHsnBKPglemnPOwNE+jnggX7ftqKnfLT+WN2NdMXSIhoiGw6lIXcO4uRErCXQIg6IEoHhMuTLx/0XEzi7a5VWH6KXyS4c2geSUzSXxEiW+2JrD/dC1ClALo1rnkivvgpH7/zttW024GC3sfTF6w6TIzlo8GEZOTVRplrv4juxnfG+BKLzvzQWCspyVAVRo801YKpHj2WU3zmmVcSpfeagDFfnCNw6WtBU2ZiDln5RVW5nkVRSmu8oosif/oagLDsFcd20S7DgNDrAhRzhm/+nHVV3kTYFq1T93RLafkh5FExg+uHj7i3IxloEFmf4iW4j6DGpBols83q147QljyXoBej/A+Yf/L4D8kY+HuXz8P6rNCA9Ixu8CyfHf8iAGmXnAXL2pLB5yG+DKeViA4NCka+U0bAmL2lLwlCH+5BEZ8t8xaOvFl9tbaksxIfWfdEOeclNdLEvtOZPkdTixwFLJ9O/0L6Y2TB4XcprMCBv4zowNr4PC5GRMq72W6EPeIAYNaeAvwdoTICaDcaT40H0lMB1C0itkwRVWFsULtA1aV7ZDK/JCadU5bkay++94OOD3TbBnF0iHPgPfTSEXU6tAa9T/zpFzmd/+WxvLorQrajjH+lbdd4rls3/5ILKgKUi5xv+Q8sJrsylLf+q95sIXZvSYaev2cRWXP1bZlOrUacXM20zKzMgbNQHdsfYbkGuHO0E6zgYqzZgRsyHPOCz36MUx0vSCZxxEZiCgW1D0aGIFGXWBxduLYHlZy/+GuiNbgDxjffZWsuyI0Juu8YmaLzzeRPvjGibD5xylj7uqJhqfxj5+aXt9XkWkaNiidMaH7Umx19RWk+YRBu+qfRBOKlD3LDf/sHvTgFypDmOFMWipMxtj4eAI+SsBY29o548OY1vyx7dHnkNdgEJXLWurboBfaGdgM/D0E8CTdgHFlRshyRiAeiJ6s5Gk+9fuOk4BQ71Dsm/RUqqp3RzQHAFR781fHHJAProluJ01L9GQjThFqjMRj0Gl8I5sriTNEEq5RoOdCNz52JUzbKepdc90c6CkDEKlZGdbv3QB6UxZwOB/pztGuCdy67SPGvO7+V2g2CW4GIgOEc6A8bgY4gOP0E69U/6dfTOL23BM/EHCwq09PPrkr28afKOjk9AkVmUWwUIBY5aQVlTTZCkf3j/9Mz1EFr4UAJY9w9QPlkiFlw3NTNPjfJ65M7babeTsOlQ0EghARXkqYAM5eN7MLdaaTQFp+h/1O8ikbwQpLaRy9T57/MucJPdsnSj2s/ZOMwX7j0sorNI7cPXZQtJp/669vJjdAq/ucdDjUaWK2ik9ejevU2hX0eLLgCPwczWmixb7chdrSRgb/HD4+cr2Ejuyxa18VuaYA1564Pzh1x0XVX+817aMSNt+bXs5fQp8o7X1R4Pv/ysqBawYk7FMoNlo2tWftYbPwCkeGllowlRkDmf33pi2xFqSeH2qId5LV5Yd7OqS1ruaEk8E/c4VQ+m3ZaloUn/r166+u38DTnu+34QEQDalqrD5Ri+AbuzVe7mEjtj5XLaFWgn1bSabGP3j/6bBfb8+cNbSc50PHRUKyzbqJWx71znB2oOyyQA4/Ab67mDrRYCg8oGuxJ4KwuyTnSC5JKu1KqthdFKOSIUSwTITEVmYDaP61RRhF6tujLowUp2CAoVuM4923Vt1s43jFQt2XL/Pn/JmlJFLMsxBTotd8oJNIO5VrXMPz69/SEDjWQW9KJm5XT9E5RamQIkupgfkbs3D9dCsbfXRPExf617BXtz8IEwgR99c/3xmy9vzGF3MSa3qUbSkHkBSAZATrbNkOUjqoEgJj8OoaPPPL1VmN1A8R9zA29dX0QXek6pa+nvhpPJSgE+b0Fv6Tb5t8gQ6RpQRyzoiAhrP2kDtvutvZksVg1VESoYBNzdVsoUbAMrZZcSj//5aCOuT5dsWA5Gna2bBvvE2B0GEafbru2/Pel8Yt3uw0KsdkVFTwqwlCHjP1T0Y5SFXMnzPVrwMllg+cG8k4v+Wf8rSlGA7LPhLWoNkZxThcFRa5MoKmVchS/IyNNO7xdedCSsGoIC2pLeUEpk46de5tXDc5cnWQNud8gWSC+OHJDPf2np3KLZK0pUP/IjiAAepl9HR7BToBZ1GtBU0PnAsD55/TU9fOt6nWHVjJYfSv1Bl3rxYPSd/J0eaFmheb8686qOXwi0rIf6Wlb36tIV7Wp81Qa/D42KCPkrrdNxMzuE3cscEWbHu9W1U/zMUXWGRUZgVNSCPke3Z1j2jsYttkYwfuzFtAkLfbEE/5xuYmeCF7Y8GtKHpzvVgyz0jV7n1Rjv634HmFT+ZmU+i+wOgYnohD6untiKE8RgwfPM3Rysr7MEAoWmPUlouQpKxoWam/0HWPPwUrO6k4ski2u2MgQ53BP/3nq4qtdO04x/2NKQc58fWelcigEuF0LgmXPzwcm8Hq9W+k4HluLF8wP0v0Bzsm4iS2fVwotK51J118StnXUbmHl5GrRFjcEP4G4Ps+4vZx6vzml1mbZk71gPGDcaSvhjNGm9FJBQrUnFKnpYMfMLdlvjJbmBXfb68eZc7O2Wpvz+7nPTV0pfX80FTshOT83H5/95TV8jZM7czg7VNVRiCBmMYT6+MezCVuAJWNA4L//ZrUVXAetTvjSjRmMFZQPFKZ39iCTj3K8GrMYfhwEceKmILNuxf/C3VlAhWSRPlHwjhIpXli1UOtxaW1906P89a1NxDqUYQ8Lsk12c4f3MlCv+waN9Zbb81RRlm7im3ACdZu/TNXj0kPojXYA75G5p+6BeuLmPwSLr88OTugvssrMcvZCHxB1/pKz2R+zxaaybGzk//tdomHXt+rdwmvv+P3+Vnb1aSdrYL+z5oWuP2cCrtlpNb5pk/OSp2IwHfFyHVlGxrR22aoAHWxF6lohgnXfrCasPu05K8298IKL4OfWIuwWy1KonQk88ICuoWVfaZnJA6P7y/gEbDJEYn/pd2VP08u8k25kNFpjlU0zgioUOC71dZbKPBpfrTyQX7ZGjebE+/bGvP4XM+WsjYrHI18k2zviWkPZtVcxU0y/nlUbPpH18gl3tVMWJxvBq0CmnFPca2oAqDTXk4pyP35LebIRcLSOSFVrx/m5SMXiU83mMw7Xy/VK7CmxH+rpz5oZxoqg+zmOwInwspBirGPdTbWjQhhaJccHlj603yWDFRVHFHAzEnCvnkofTFhfkG+W4Aq6zZ/MF3o4srH9xsw1t5AshS/mluN/Qh7rUxEklWfZqJ8Q7+bCtTUCqSjjoRX1VQIEnpkK2n90ojsrAblC5C1ndX/ozFFl5cuf03NiTBcu698dVpprhihgwesqXnsFGYWT89Toa1tyWD+6/WNFKdopwUaJVjAuNYnulA8YO+zsbzKS6Z/TYxcKGPtQ2tHW9ZY289RL5UVD2WyJGTgiVSqvMVLnabLn/ulJU7A+Lx5D06C8rxoafMpmQp9nIEGioV6wmujM4pqaJazHleMM85zLXEoQK8ZhiAEFScJ2J4Yt+9KtxOCNytILL5SekSw+3YyxhQNZmBAhS6xV/iBLiFFTrtPonF0aQ2B3xW5dRH7bxv5eMnPxnrL1uZYUCFrLY57kGNFSMKg0FskZNYmntbOJJifQNv81ehs1NsfcltJ+87MXznFO49MAHkUxTbEBngKgGWtTHxjoGXELGKTLPlQEbD7sS/SbuJm4bUfrZE02bC2unrTC2LMQJ4IhyPnRAvOhVosQQI2qhXG7o6PH7S8kpcyulmMzwtShfrZUwjhDWykH3uZfAtf+dBnIfPXvNjcgrTmv0z+svo/aCfgqv/mflVzLytHi0xcjZttqGhVGR81Atvqlv9UDTSrn9geOKzaEgy1akEoj2tH0cdnTt37yOcr4bQ1o03pF2R9JJLtnZC11h2nimjUXNHFm/RK+iSNkV40sKO9ymLLmz+F7qYB//YM+ASqmvKcS/fV4NqOJfBxO76T+6qSlXdjZMhr76PV/+QNC6jpIAEwqfVUD+uXULDvOGNdkdxLi3upkbVCVWVmp6bSTsJYe0MFssy+RWZEMDAFrxoYHHNXiKL1KHqMA7qo12XOzRXsP8jL3lVdZeQDgFWjwx1elLh4CoxRYiwgduhJ6KCaHCPwChQt3H79W3H3vttEuTn6iRB04Yowk1kMQ/whiufeWca8AuAcFk+APSJSh3ZklOVhc8G1y0dE9YotGEEdiieCaaCuUOLdiu3ViQcnkAe1cLyLGe0Q17Azq9shCiunZMhl/UC6JQYzRZ9PHCkZX5BtC6AFN/bKledOajOgSDBAfMer5s1/xaXE7NWPdt1RtPbm/0N4AC+cKKS/N8VV5VdIyiSYYB0MDOgPbQkRTD+q4fLorRHYHd3uSGzIzat0tcWH1cijVXU37RtTgfxdW5XVMprm02tlNdU03FBgJcPHVvynnn8OjB3ykIzO2XFIr0WBv/itvObahnSRcGHygbyhbIJR7sZpWR6f/2pUk8utNheU19MQdtRKyShBW2d6ffv3iuY2R2DjstVnIIy1PGgB2/AbGgeuVXmbf4uLmCVIEUUebtqfmjYv1ZnHOLuAurNUmH0ulwOVg8TZr+HkWwS1u92gF3XDE86kPszOmmNIKvvW6/9hKbu/au9a2YsUxeTkYYBsXV5Rp6ruqaeQ3e2TgYXkQC55hJVa1lKJGnTAltQBwSapXw9cGWoqUK6KFasr/XCJRDX+GV0web27g7XUjFG/EkxhjTLsVlUTxvgnxKG7BxsFaK0ZJ2VB4+6BTIN6rzvXmjsa5KVt3sGRdQHUN1mhdiNHa4osTzXQLM23TCa9ugvbHDUobBmJ53cCMdjK3QVCt4i7JVT1AFUBR+7X9fZzs/qpg41uGSS6uSyIpo11cyIcaaR6cZTTBFrd/veh1n7PfsUQKWp776Sh4b84QmRfZws794a/OqTGlUUF9Zaa1kb/xxMhRsWD1n01oDeXEx4eH+yGLGQnmU12s67mUObZ7dXNjaO2G4qMfozg7TELeeVpGH44vr2gYIKMe5LpwoRa0RCtslDkPQ3d+UqyzMQP7iU8uaVqcWy4Mz5uCUmXgMCAmMGZCaAhQKZuGRKVnWNaYwstZPg6+9dIxuLhf4hs0L9VUMxH0ucwHqHtb6SEPyb5mbej8T6OhbhTAg+tz8IbhWMpPqkbckOZx56ue/b08glmWyFYJolGUvP5gFmJFP4Mo5tni09XiWFGe/bh55ZVCXQbc4SvddgbCBWhdxpWF/qfNm8AeZmHXuCTTx7rpNOi/pSQv05IV0n02kIse113p7w9dNtC56ajV+/jkI8kjRC5KTRQ9+Yok1k7iOnifDlXmIdWsd55Nt7Mx3L+H2i+7O7IjHvRE7TNjVsJbQkToyJ/IYICLIQpEQN2oKBHkPe1dJ7K6psnPHroNrxxttU8vM+wo/Kjvlq1cumwwYCuevW7BaR9wI0SDnbsA9B2xMnuywP38HwkR5QSjU2uNC3xfmx1bQSe43zbWpGCFFlWJ9Wl0mUsYN2Qm2yfG2qwrhtPeZL5GgC+kY7hpB2w1cNh2Z4nKe6EE7qDdsBKLkABtmeHy7N/rhxyr1byNcDlylzfpVWoOzVG2welVFr0BEF1/6ZCPmOieAIOnev1wZvAEJfKiDTQlG5ETQe8J/xE7Z/Q8Wwm5j6Mp9kMFaDMulhOsibYq6Q14FuscUrcWUrEp6HnJs9A1UDE2BY1i8lqAMKctMHPY6MPezU36SZmp/Md+hMTZIGhjEMl41hW6/9859o4gte/kIyST8UdLcur9YrJcwYUpRTATkWMO69vWv3bP7eABKtXqCLxSgrqjUf8ttDFhkmAc55zngdPU1Ns0lBfObbxvN2iDXJTBgBSacZ2GDqlURYMQMnlbwnCI+k6pyRczJ1RsLL0DDfhCJsRTJwCIWiOVEykg8JtF6wLOsyKSQRXS0ORgSLX34rrwXskqXAg+DypZcp2roVnwGKrjbYchxfHUCk0+GxxsdxBquLXeXTw+5MEfkLX0jeMLr1HfPKy0gyy+5KsyxarygByDim094qFL2ekVU6xgTe64MWhBCFOg7IsZNMuuqWVUHMt+Hcx5cN3lb2ei4t795OABfTtHW+K68mc748oj09RSbfGHV1d2XrN7pF5fjiPDolJawXealsE4aKhOlg4/K2f/syLbOAL+dCD3xK2L7m/juw+5F5zdl2wu4eKLK06LeX6YoXiKDKsB7gftBZexTejoCdt+1Wn3CcqfxwS3GOH7stTfAxaLs5h42K92tzKbzyPk7652FwWnlajqAH2HaWCiryKDdodmr9kqyVF0B0G5kJlxMSldc8tb70BPlCUGqhezvTD3pSywUcjkA/JSFvUfUzQLJlGtYiMkLSfMT3mr2ArLtlIq5g8GneoiZtzv3wkUAArSmL5V1rj0ty1zYorv0P1Q9SzCTvU45PYgw380+euDRr7fwa/pMWgIdQXz3TqfTtlgw0guj4NG5Uklcc2r/W1aoAHpaGDYY8KmVNwtyUqeMMKiJ1wIJykWSHKNjleacvahOPwVTu3Wg6rPh9hqTzi0LOaoVGIzX4j/2FHCzyeIvms4Qv/YWo16pTQnXAP5j1WRx/cuQfXU6gsvXRfvvvhpvTP56berAAiTgVU/pwUPZ27lSHpAQTZOvAJbftlN+r2Rn3q+GRGsy/E2Cs/RYOErfzcdbxlcxu5i327BTETKNKCiF7GsN357iTir3VjEkSQam2fd2Sp85V6Z3sGnhnOHBxy3mzzhLsi3asvVLX58P5d1y0J2GnaTD0BfU9+SkFKCv8i6wJPGniB9b1mlwHTRY8GFLXK99mZ7OvYOt80y/08n6btB2fPfrRXNvpzfm/tnQUvs0sEBaBhZiQF2QyqOm+IRTiAD+Cu3NxsRHT959vkUn3c0FWP/i2X3MIco0nQAY45HShBWyL6y9gT3Y98pqG872626I7u5ZOTP9Q4wLUyP1ltdNv6M05TkY1u4ZFMa4HRQ59pjJIgVy2dRgufJzbSTKvMtBjpKpLhSW6gWVWDYoG2hxwYURo97WUcaUkCbL39HaessAyIVEHEsZE8FbL6qno4zfwIhlI5zBgJheJlhCX3lzC+Avah7pRJeQ/YVu0X+IMb70pMWkmAJj3dnaO197jw+M43hjkykoApog3ZbuBbkFJLawtpDfpMhD20CYqIw00S1kWyBBm/1eYQATQEIzlZPy9/eanv+qCuQ1zsH88FqmQZwlFEbAmFLKA36/1ELO2N3hXVKj4/YaEchNNtd3GPDqj6/nvhfQOWW3w3BfJNasuhTQd+0NoGZUy2TK/IEoIXtI69mAyI6Q2FcEQ6ZMwbGqSmlJBWSqHtimYth32fXb2HsI80rC6Zih3b5APidjVIzErw/ZcraQ7cwcoPVP1mCWoWoFWQ1CW13mn4mz6iHdZ+VEgDUWTpDJiJmBDlY9KED3x41nitnEwizO8rnAPdKtmDkhigzwTb07pXjr9i/di7fZzS0NTFbFLqyO2oeb6Lmh9yoNuPZy0XuMWvvCg+0bwIVHIsvoQsUKaKVx54rkLWPcB/DEFwwy9YnxMmKRqjQBZWnJF2AoqSzHAmjuWGmTsHFg/iczfWvkz37Hv59rsr7uYEWpAlQ/yHbgJfPuGGWeedHt7YNA5XdfiMJW2QzaqDHC/+/og/QjgA8HadYEv++vvpjJTlXPI5tnJ+v/ONuTdBQqgeNwJnF4o4cIHI956Av5qoQ+BO2oLaP41NE/bpMt1uLrwYz6m0iUOW53MzNprBzH8c9ufOUzszVZTCzdnchjJmnFp6BAKQkOekrN1ym4Uv6X1wfYpcHmaNQA1++JgFbqOi04je+ULmVsbW6NpoVsAG2bWLsICRCKndqPUNzUSZIWadaiA14yP+q6G9fQdfLsKaM9u7miWsyhJxHuSWXNKHk4g77SXmDuONv2AlYJJHtqn8YGw0/kGyZRCw5A05G1AE69l3pIagOBG8dNqNUrcgVVjsYb+wPwjT8pYY/ZsGIqY8/5pk5n/KLQk0WpNgMYVGSKVeqU2n/6RR07E7WGcvDgnrdPo8/9QTMzG69TmD48omvIuMmQ3fKnfqGRmx/5bSxZI+s9xP2tJ+b8W+zoKIHVmpeOMfeEmLsf/ROElQyy9GP6HfUR9vKu67ILa8f3O6jsWUw5qNcTojax25gywLBau5FDUiBaBUiS30X5seUhC+uq8ZqJOrqG+iQBlhGEDvy3XSTp4Z3EBTOrKY6AMHYWauCplufK8iF96EKKGIyrNyKR0hP2Akh0mfrPPeZKfx5WWVeXURYoi1z6PwHs1AclYFoOLr7qzuF/AJcyS8BSrN1aPnk57bYIG/2ILVzFLNQhXmeRyXHXstrWhRk3Tjrf9kXmRuB3HeA4P/LER/5hLkg73U5/pl/r+TEAIEdUwufgBmfpl8ja8rc7Q2uNSJu2wjrRpi83n3CiXouGZ6i9yrthJzgX+tZ3K/tK9INfwRVJbjuRQJYyCYgH2Yw0m1hsoAshASTOBxcjGH31B+2Z8jZ+hAHvCrC/+R+WUjsq6p3ffiA8QuKuUstWc3Fu7uLMyNCbm/890m8pc+i7u5cufXKd9YLcmqdXv7KGXyvZ/dQ3jrT/Zjr4Jzw69SoFqPw2wrVffRBXBF3Cgr3Sa0wIsEek9Q3GoEY4YsNBT3Kh4afERGeQ+OMXluk9oATrXvUY8IXVhIuY0teUq1rNKbCOztWuCiE8s2mkbpVGrkm+8in/3jP/aFlbOLegsSzubGobjfQwkXqg5Te5j0mXF6eD31gEv/ff/QHnoa9JrZsmV6wVH9DOY0ZuK5jST5e6NC/VgMuz0g2g+CAhy7GODuRfJl17olFbY4ADoUcDAI78QNiadb1+Ybn/dvhZ6TXI36M/BP+h73zjOUsRW+rBZz1M5SA4+t8GpfLXachaVDANamqE3Cq04Ti+pbDL2vLPibaCe8OY9z0AKlJ9lkYjs1wLy/+luOEYnGw+fB0Rs4Oi096XP7munwmq7juDtc7fAuPVB5HbPrVzsnN/LXDAfav3umNGgXKoWZqhb03WBsLtG99dOdwkglMtti2h9oEpi8qbFtxuOP951DmHXQxnYp066gKEjwAa7gIu5NzCHiiZrd4YuucELC2jwq5f3r4wBD57CzBD/jZugcXc3q37XUffpR9FK7kXvnff70LB3eMs6rIfCOnBD43LU8oXxUX0LKDXR7NyXmJ5RhH4jGqLG5gokzZh6o6U9Pv8cM9oFV47MIlvYnXkJ5TyAnSfNwhNsEKAwItYhEHUdhzECR+xgIkmDFQEnoo5jQGSpTg/QQY47++TnNigtmqHKhDUJzmjbBFDejIEb5nB8uIk6Bw1THM0mLHmxyv+e5okeKPcgq7JanoWXg8/KNaxL8ZutBkPGwkilCio0fugFGIyak96Li0W60jVQeHoJ3ckD61eVLHmYg8qLiYvT0mWRzWuiO6Tu/Lzpz9rfPA183vbuTB7pvH6Bqa13jr/s9jIfJYJYbpkTuGK7IROd+USN9A2/WFpX3wsqJWffTJc8F8dSzG+ZWrjVhzE7JuIpUIfTvcZl9rKtQ6UpOsD9XIRUnFavsZ+spNp5ytCTQAa3nBSb2gS+cIW0MDjrbflvdsgbXhppa+aDzvB0XS0WAHI5RUg0PYuN7A+RKduLm29hVFdgas/OTe/2kZUogNW4oHYf1AzYSo01kM0m5jcvQPJUSdvubr0qZFwfjGKjyn7ZbtHg9aIMUIcB1OFsm8ql20M9PGsDbpAwq3JvvN9UwP0rOWmjOzObkhV8LDp+WM7BrmjJmLpsaWPXn0aktGSpZx+QNaPQTvocIJDrf3K3R95A8GWC6zCSfWGRtvsTH9V2nrsRpu3DvY0Jk8mxM2Xsr56M0QAQn7Zddp5/gLMs52gey93Hc0WUzdEr3z+IHdDQ+S83s4nn8BDp/7TfGV7NWLyNLgXu9AxdxHVI0zM4ZoBIBa5RYOTaEw/lz3NfyXu7f0KKXE2EUjov7+1ENQqD64cWffaZeDAc30P9loq5Q76DULaROD0BERlfvetUg9euIXIa4fYYIe7MlCI310ODsCLpfsx+8ueiPDvRI+7WHl/Tt9DvLZQtQX/A/pyuo3xS1oaMzWxzb0ivViiRIyV/xtqphUBy0i93WM05O3U5XduN87CGZ8L4RKwRs3uSeT9gZdS/tnCN/kcxHU54P31HRPtqZO/osoIq1Qt/ZSos4adOfxZB5hgFPg8tkEoqA2fRmHkaagicYTrZyAxsQ7ht3I8KfV+RD1UjsGUjH/wcJ+cwpd3OUXyXC0HYKpWZYwFmT2i6f3CrideqREqBuS+W6UEru39QWUY3ZUBb8pZXu4ZvMsB38C78z6c7m+nO/M6o6HH/UzIBOc3LnTPFlGi3C192MnOuIafGnTeybfg6aGvTmrQ5lh6K3Y1+93arMZqYKdwpcIwbfH6zj7gDE1/RH5kJ9ARiM2/KQnVTsG8l23AjHMSBUbpSPpwBbNbdTxL7gjfpQ4Jpmqc3U/eY2t7om5rtZX2EAPYyRYz2uLJRzkpfhaWe+iXdZSznv7Gika0SXdSOgSWGpuXbdQChcEQybAlssh+53obzoWbMUa3EL+yd8Dsjb768i4Mzv0MkTtfxf5sXvoSxIL8UNHpPqzJXf6oFcg9f/F6BiY+xvfB2VzmgJfdeMtKhiw0bfUYGdBSLCTUYsjDyoHEz+XsmCshYoOWw8XL9lPxPYEhTVCW2mFAWnlQ6+sk9Wuod1Begtw/OO+kzAsQB8m/z4zCa02m6ciwyR7k8s9B+6FPvVPowC2VbtS6+Jb0dWm6F+wikjmIdsXvu3eBS7aTyCGH9y/xCQ683ITZZJpNS3UKGS6ee6Q2gyrGfydmR2VtNd9ozxV16pmgGxXjOqD20IWtL33YdLiO/8+sJv8kWqmMxGt2CJ7JOTjHM33RGzkxgfuxo/VjaIckLkBaXW41KSoeUhyFD5V/n4NmF4f5PFJbkmEtuYOrbjm7nwddnsMbW3DdP2gh+g4HQWv+HpgnHftj+ao4GwBzCkgG1Ft85Qamsi6ECpR0ahxKZusXlqZG5Nw5kG2yeXLBofm3Tjhba122sViC6ObbsOt4FM48p5aUi++9Hmvlv6UvW1i6KCUdnsgYtOffT81cAdGDdC4iba7nipcj6uNsIwRjlndCROxRmMQ4Tnz36ubP7/8u7HV8TmPjCkh5kyJ7/NwOvgoed2F2xoU+sklWEJE+HewIm3gbFJibgZzVYhsWkr+wrgy+LaoR7FdbHUaOLK8TAvhd33PPK1H+ilzIH7cZ6NCiq0KzzXb2zAPzCJGjXn/6J6tYzRq/9EJw7lkSIcc0h7xlNU+nFQ1vvJm+F8u+bF/Evjt4+uZX2fdI6hiFUIxTZhKXNchAFQpgHs/+FyhGX3KKeo6cxfc6/3hH4nyn1dlYX/IJVuVWVtJ7gBwlfWaDPBLnKzVunpMVN0UCTjF22sP/TnXkyufyK2PmQf8lI4spowOOLTL7n9OKIBfZRCZPaNWNghdLtl3ErJ/Cx4z3GLlyCbC2UNKu9OFG9tEQwNo/Aa2vf1dev4If6JU64WZuTo45q5lEbW4Pt2/vlIAlH5ZCWGO8mCg45R5MmzVv2h9Y8nXlUWlj4utqzFTVKTFHmzWElJgOXQX4z+2+dO4TkNw8U0QEjhYWJDAPP92yb+zsHD00ZcKzYavNShjlS3oXDr6NclqzFIWoVZ1QgRMkFTQ28sVESVnAho+SyQ7TtI+fYfFvK4aoyyj+un7hfNj5LOD7mD6C1I9b56B/ISQmeTmmYYTr/+jm3ZM2S4CptPnd65BL8a/nAn/Nt7QpSzn+o177Rf/nG1yi+fAxBUEx0n7ODH8ouVcaY1PXhKKRxryY02Lxg5ov+q0X4Gb9D22g2uFnH9jtz/75msL6NqbtbYsft0fGC9K37Ck9D2VV1KZJRKQjcnICTFUJDUZCPjt6BaGuw/pff/zRCF9CqAaDEJvKhaUR3opCQ4wXW/f+3pMRmla/+vid3fmb9dfc2v/aaVwnvRjeDGx8mWf6B4V1ZI3c1w/bgEl9pVavVhnyhuZF7UsvDlfGrVPak1PEfXf2vPnpPxN7gYWU1rnmDQ6wxlbWwYJKIEfuWXVq/e/5HyDwmJW9wP+tH4hntuG233wHRs1j7TaQC34vTZkZmVefC82SizCGqYEiQijDMvHgYAE5+R3+piYL5+6Yo49Nfu/+ToWF+2yhEZ37OmXIfePBMPy3SF1rH9jyRPiepHk1meL3Ec2i4Feavifkne9dYjJ6t24vfu2gopt22eVj+/dB+eoQWQgdArJi063ve6KrYk/06naIFo7cOXROADmzXzohWLyiThpxp7I5nSfKM5M0ZPFtedg8AfNm/dR/MFzO/Y54kvLtYUf2n5WVZ/SpONb4UPAAD89H/nmyDUXi0MqjlvNiXNkELf5Pn7qvS88hu6BRSMB9YhKrI0XdAaPlnS8F0+gVHIEB843gF3cO+fWVSqFam8F7J1xVJjxAhpKJbrCLp6aUM7j+yV/7Utkk1lvBO8igVAm5G1dLuzXWMgjYaGyz7/R+/S2AGj26D/Dzq/cTm75/ZWOphFIMXlw6pR+zHJ4+Pi6PpqDsl8uSzONftnnBRCjGT2Y1Q1kclMoggsbA7t7iREkBT9QJh5ru16t2mPTxuU8WVxdRVtb7r/U5L7ZxPU8JyVdybdjspqaBEHHYYiN03KIllk5l/q4fob2LKS2CffxYhFZjHSPxRJXkd9g3itNrJ8HjjGY5LtOP7N7tyFLRRXL6P+v5IF//xb4yqGM4TI66a2SdZWu/9NL0fJSCTiK8ciUzLSKsL3gb+6gPiH1AvX7jvTBzzLW5gWS46g9ADjA3OQ7+WBO+Z+oICNqbEkTGvP/rFch93we+h++d5sf+PNYaCGgTf3pbENX4RBi64RW3ILZqocwrrpFBUVtXhuyS4+2HtAh5seZ1vohDQqfbZ0XS1kSOLptnoBCQElGWFwwmQnpCx/UYkrHgR0AfrpYhFoklSkUfQYdxDzR8UmES3a7efM2lBKuH//3+SYo5GogQugatafm5Q1zE3RxufBhDhBAqyxdDpvUeeJcsu8ZPAhbBzxAH/cipEp/+i2J/Kpd5p6BuRJvPofhKkz15mXvezOXV00YTLy8z/NCFYwR6NKJ7LhfB7G5zuP9CLWigQDzYVvuA3KchfxYAaPPfyB3eQGLxw5qbglKG2wwC1L4tt2/5EJ1lxKNThZL1kWUw+3TtvYeIFP6l0L1BtphWpyS9vCBkEPAVr8I/lGvL8K8xlDqmD28DVBTLlsX1LSkPqHxrBkaxLkFi0deAZrbmOXuekCrQy+/Dkzp6aY+zzTWsHYGNhPPrDaYtX8vbzgl7vK+7/MoBDzpdH6yM4FLMHfKovV/D+Ai8C2clkpyFIBz14/dvyLP7qJ6gxcCpD+JHFlSFRJuTZU1BWioXttfgrOqfvzXKjgf8WgDvfrxaYDM4F8u1hX++BSrnGxHkUpM64q3LiOl8hRQTwIh1LKKSZ7LegEQTnVBevJZyEjD1HBArfDmJ66EsNld4QQDVdZOBr4brhw+WzMfdhupyQJVgUqNOVskmsCm1hcGMT5+5Ozgh1VMqbgxcDCOpxM40mswGps4ShyUr56Ttqyc9ULK8C/barhgqwg4UqAwcDu3vLdNrnOLnD0jY2e/bN1CZqRHex6VRK6Nh9IipOOAevpXKRSY8rMYBm7KTHTbBKN6lyhAB7eqfaLa4Balimb/zjAiOi1EhB21/WijZ8z87ysXvSlKcycrelsmUxXOkAjvaOofLCFMHesCAFzL7d/lIzC/W0YaykcpjMEtejBrQ7w3sZfT6bfHvsNfQrzkt8zBC63gVtQpRltCYPWj2d4CZprqJpE4Nt8l0+Q9Gls/dYepgsyvt1bfYT9GBMPiV57IzwJFnG+tUj00NlCEYqThF7nLw4SMwoKhptgV0WaPkwiKmkq4dTn21efBwaiYd5Ob/Y8NENiGrXF4iZbq1HXWvPQ06J0yrWwSgO635rF9dlTL1sgcWtnuKV4Av54O5x5d9eOd8ZHdc3NsS0QTn6hj78/9dWnLn8PEXKs+S3WwWklsbDZ5t+wODqu8Mk6c004U3C+dnYWnn8Pnm9+0gcD0d+WWrSjpIn3r1xVY2/kJ87vcLXMQe97/xCA/stWZu9lAWeSI28qnbuMNPG6y8WjyPLwCxmcoHSA8jyHHmPMRHO15iWZQlx0pqedmsjYutHBaI8vxEt7V10eu9wRM879fzm087XouoBbPY//ICZbfv60JcNAFjjLlvz6kBuKvrpBoRlv1o+uG32YQRwu4cx4rCU3L9QYQffMkiF+6OnBzkjfNiA6538FHbBpr/5oCDjY2ecRnVMUJ2hRoVPneMF1Sgljn6RagE89hUCfF0WPrwO3Pe/fRb16nyaK7zWVv56MFar8NOIuRouMyDyDTaGHjbcGgAjL0dTXd5RtSggdK7l0+x5KNyVwi/jHUurChydUV5Ckq/fLNJaBqea07SxXBAf6Gn8+/JMjSM5EXGtUY74oWvRJ2o9fedStGoTuT7jPMc/ttbp1qrO7BgOBp0t9z4Dag+VDy0vZ+MkhHC1bLqnbuMw8mcqbjVECv5lJjdn3b5qOfHZlj2ORdMGNHuwaiWYwmBB4GvwmjU0V71RhCO9y1C81De0gAKGZGPO+/CQB9P/l5bLfmFDQkVLk4TUlpFD3U4CdJXRB5f8UoSURWqTNMIKLwoCrwx2meTqz22h3PyXMpIc4hfgMaZ0nZmQiXGILW2pF/fo4mbWdrz1Xq0aNLK0Z3zYQIMOqgh2wnfbbs5AYn/KuO/me5v4BhalXgasNL7KwTWgD2peyoy8mvrhAEy9agGZBiZMJTPrQOextLFqnwRlqS6IqEgP8aLcl2BT11P8SJqz3KM+e86MM4J1So63rM1vkAip/ZVgk0W8R0dqbriovdNPXNnVp1ChaAHw4UjWHHkrG7t3Dz7sxqp/PRffqnM1B5ua6MwzrzKxzWkGriOVCMMiANEPpJU9rlH13eir4lBa2+0ZQdiWrCBSpdkF+4+1FqLZ9Zp7/LxyilLwvNA9+lYv9OiI2r21m6Nv5OPa2ozFRvAZqpaBY7FXC+PDm4085UiSAiiLLLx2JwiQy6wojWql1AkgmSIRSoD+X5tinJyIsWrVvY0GaULObDH/unIglJPXbpLAEk1BQx1kMn7DaTBgRel1cgKWuLaZz1kQsTK+ZVaiLRdjfNlpvFfdVKtNpYT1+UzdeAaI9XAFueh6OgL9k906t0OAylLwmwf8NH+tdImcI2sjVNOruXTigOw0OBbfhWw9sNNuea75lKcglbisSncOqVnH0MyOAHRzwvpvI0cBnykaOm8rC7+LradeGy72/epXj3tzbG04YguzI8HoYDAQD3gfFCE+tOtxcdMYAzsiPQbSxDe7pbkpizTlyG7WOMue6pPDXiB2ISsYDaWtQXnX3mq2SShXdKqO0sR2NTq2h7VWKl4i4wRGFCSrAogXVVrkOKr5V4Ne6sKzy4RvVy5VeU4g9LACsS4jMl2A/bgVSll7dDUamOAaFrZ/BEWhId+bPUQTpeoGaDm4GDHW0+Prh27h6Dlu/JeCTKMInMoJlx0DF2eWdSmrJI8fIPBlhPdbgStbVhBiNdtmC8rfar4FZqJhRor5ZswxHjaV9ig+cMLGuwWY19DNagqZSumoL8Kk+z5lgljEd8uTbdP/4nRuJpjagb58wc5RlGY+6BgOmxLrAwS+nA3YurQEbjeDXjpnS+zQ46TLIvl9usBTKecXn+pF4VI5FaoBwNk91nkGl9EDATj5xeM/8eBZh6Z/BI3lfjTBf1PU+LH/9iYrrbfmHlrL+nNW3UNu4BR8KmKfTj+/bujvOp/h5zgGb0m1riscpAxux16t19+Q0wxSct37a4Rxo2GGPEPjprLW5njc2bDfVC/Kb/0lJzQsc42X5giz8bWNm63tfuyNt8SLABJKAdtgpQqNbRnXzL7+Wo1XZfaSyAetmpTlRbuiPjwq5RjLdcsnnuKJ0hFL4OQeiS40MiqxESFJzePuiedrWSzu90cFPuD8fI6NhHGdF8aSJ8BFOsJ01UxTaJFIGOiYxUP1g43DlaAGgj5Bch/851LQl93sLDyYP2de93E/agJxpSBybGCpEl1JgPMIZsQZKiW35GqckWpKpWsoNQErqNkeBBmpnhRqSMGHGVtPpFqXIajFFxm/v2hKacSs6Hr1zMwv0+PqWJ7KBzN+WvzYyC5K15MnG2QeHzWidkBRYe7Oul1JGd0z3DB6uOz97mXfPeTYewpf7eziFRlc387vNOasSot2oLJoqwEqDpd61xrAhgt9GzoD8fxPGrFx1DwUNtDa/ac1yKVVzgw75wAKEHH4SclVzr9za65hxFhBrcFNtJvTgyQAULeCRKUKYpGcKTHqmNZQAYH0q99dhtOgdqpbyRlKF/s33UhlKey4j/fX0EP5u6uhLBeDmId0Z9zkdIHgB3L3b3BmPu7PdYL6sjpF0i0/VBPuqYDSRUMyixR7zES1iVXdRXieBKX0azforhLXpg/HejgcJECwaBtNSB7AB1Bqidr7SLAYcQYfCqEJ3KKkbSMrsE2Qu8OMO5kYX24EUq/7Z3YAW52EKBouryx4iwfXVq91gcTQl1W2JH9DqFx8SshIgF70C5zFRI6m8bQB9onM2NQlbepVAEEljjjamJj7rlrFHIj9c0uX1IpoI7cq9kwy3ALUeLDgildRTd5l1M5c23UkN48yJTOee9tYK06UPc7hzgdF3ZxCAnyafNO/iRuVbD+Faikd3VSYQFbBsgvKO77jhJdvZFIz6Dqc+cN2S5tbHXxLrq4WJl1el4nx6IUQgxLpUgArCqF9EkCO6Wdear/QHWEtKhePLJB8rVrqaquqX5ZaCFL06g0tZ2mfnS5MXXqN+okePFJpHyIv1RXYbWWmkM5y7kL+EuE07dmdC3VSDMmXU1cYsB1PAnGrPSlzKofe7KWz8mbg4n612uA7LCEp36WJtWns+kKYNVh3civQJLV022b/fiTRGFnt9pvrwiKo2toFBRmY0yQmvi2YYFFnr1KM14OLQ3dxYDho9geurtF0ubRlp7pHo4unQLkCw0Uiekw8wH9cnm74rOm2orSwQ7Xmz5aC253u7nejOocVmAYjs8Xwu655wF0sxeCoKJJqnrwFNweZ/9nxnnlfrO7NWPx2LIN176ajOooK4vkJdetVlmvoiJ4ZeHWVHe2oxiUX+34hbyiRmSZmdkDR9uT44dXqkDb+TqBsvqiYRLA8Een7vkY7kWxlsSZx224572LFzrcbwimLpOW+kqDLfOP9rJ2S+l/HFkQTVlhwnL9dXIJWgwpkYyzf+KX00fP/xLoGXFaCgsVipVIrYyrTpKUqXd6dSa0R8ltQq6imr3Bj82TJ7AZyIUh/347kBJ9vTE4TLUB0VInqftCIlPBlUhVu6laFEsLm8om4DQ50Jns8ZON9cHvXDs10mqtdzPmQNDMBuHgnMkLsZwZ5+Z+KnIVJXOso4FXaF7HSc4BHF2WKiilrIDeHbi+gWN7Xzm09slaxpZqjVXIYrVYRP6oS3GCeHnlyfhKUIc+xW1iO1PUJGDdActLkJGlAxKrfiaRb+l+oqqRrmLQbQd5IGhC+vB/gb7+T0zOBBfxn7V9PsKJOVYB5HsNfBCDCVMuzdCpyLCtOO3LgThZ2kwnlUfCApeURJlkuqknlODMeuH7uahBpPekYDr6Gu9f14rSxIPbeH8FCXGjJ6Hxwu4fxNVyJ9+3fF4zRG0O2ClalvpC4Yq+kp7ub/ZMZU/PzuQxWwiDpuAmDTps3Nc81OTpCvEAuSS2O6LWLrj0+6CECpz1wkOCVMPGreB2daPSr22qKwsO3CDbKz/9kErBi3YFKrmNsblubFN7zIGxu5+sx+PmHt4mt4w/Tu7nd4rR6NQ27HVMSNdsSQXj2Xm6yzS9X7i9DpsbXfUS+kj37pFBzmZtfnv5NgeYYxNxDKsT1BojoGp1Cqld0hS/Udq6+eILXhBBfteo00dEH6hUlVagP+o5RWSx8AGEw/3dVRTNc5POJv6b80cLUKF8EJCEQFJ+AYxzuyhPwGThl9avHNkKs+zqnyEDBOiNMGjU0LWDcHyq+XTB/WCH3NrgOyyPDIxWZ7ZYZ3ZmGxelM/L27FYT1CD2uOu51FMylx1iVScGcfFb3EEt0v4KT4FCMS4JAEBWVHCJAxxq71VR1DNVOvTlB1kolyVhDr1rXPLjhWmOf7+ft2uRKSE1qZRmeMYbO/Mb4RL21dOpe6rWV2d7gjFxJFDzjo7iwVYytgFU0kDpUFOTY3JOJv6HENPNY5/H71oHUIVzB5Qx/LQFKuHgrbz16g86I7XdjRSkm186qEClopMW0AZLYamBD5oZB3nHvcilFs78R2dH5p9trxJ7j6tflELcsFAEYtDvySjRc+kQg+Fw7K+g9fGlywdT3ouvNC3sKoinNfTHxZnj8QPsZ7PzWUlUJPZEe7uYjSgjVx9uO81u1LKBUAxpNFRQJzguCz4SpQ5e5hX4MoTz4yVgY8fZWA48QNS/Teamjqu4QbEPIemoYduuXrJ8T4z5jC1EZpzQXNj3xFOY/fPB1hvkxd67qwNEIwuqCg/VS5UM6otF/8Bt8wfSQiuLqOyfsrhkCUPPrgQesx7Bv85TopRjHWch25aa+6pa6gybgDhZjsbfT9ahRx3cbETeC9LX6AfUezCTTDBmebG22MuOxZkRz5q1Pphoa0F3xx2GE36v8JdTzsxXBZmriFz0zWh8cKdP29IdnD9WUDw8rtRqb3ZJTDrzetlcvKA5aVUu0TB1z60VKen0TB3/0OcIf9flj3fWGP/sx0cg0VrF4ABfjzc+iU+eCq5/X/U3uBVx5etTx211gFLA9nWOA22udo75ziMUfHTKOxkhZzbdvb/DUs14Z5blldgax3UO5Vesn1LQhJNW90D6N+KJYEM6JX2beuDrcXFMGgMaDKmUyuCdAR2oQ223daqdkqvb7kVeyG4mXf0nxTU5o+QBwwg1DmSsEOUBOVQ4GWyA6uReN2pe+PTZ9ojluqpaYyekdJmlpya/Ruz6t6+7c1Yx8JxWM3nAH8L1F6B2gQoI9mxs843K6xwlZr4+c9KgQ+QCjlH4y23wbbjB/P29BfjYEWtjgpyx5nCL+afK6BTbLzECLXTG8nX7Q7pnvQ+mLzoTB3jjcpoVr/GrMwJe7NhbQ3ZAIwmFd1LaUJv6sH0KdugVw71Z05frSdf4vFCVW0obMDwv1Bj0guyAKObXh12QmRr2kvb8C6t123IcQjmWkH0+hBUgq0qyft0BtSSGQxE+rd9VDmeKVLfeIIA1LcV9YRYVAhD3fa8LrmHs8IQiZY475Y/kd2C11PlGDMDm0+KpwMX7qVhRShHt/EiozS3cnxEz6oLyBL9NJunb+JNStTyih7ftT/6obxyQUXyq2UW5fwfYSxeQR8Twqc8D4gw0SIOcUBuS8bGB3hf9Dz2ioHnfdPQtdWS5lvP9kz/TmB0h4HybjbCnmuy0jKUn9tkp8vUJ1RbIwLyGYk2v8RwvwEGxIBM4CTNlBS+mDvKSVK8zfRvichXdGbjH8lNxCNmEN/GakcpjlZ5XmzJGn9AW/daJm8e8B7UkVU2UrwRtZ3jydrP0phUREyahzvJcw4swSTkv6CzhXdbkAAnb14iWEgIvwLrQmz9wAwlZqSuMHBAkIsLxmCzGKNNjYL7dwdrVgQHihberG7A+YyYFPPZG31G2lLEykHvynZunQUHvHsxO5lUTUiwlNsGZN2VnAxVgWagaVa5S42E/P0OqQjjMPI/wRlaUmNPE86nAi23j0GyV3d1Dsysd6uQ6pWWWPmFBu8GY0nNU86Dqo7rFE10PU7Vv/bP+f9eDESxA9OE/1ZS3kapRS/jouP241I963zy896dviI4VXVeFNzFI6lGYMmdqdI7FY+X5u+nPSOBFc1kxsaSre1873GdWjY+oKAtsdczwzql3niY5pveEXXld2piLCSzwIifUrq4+I+jaSF1lFXvLv0Ce68jlv56T823txp+bcOk4ykG/2hLkA9t1UaS9hs9BCh/V/wF92FVZoU9r+i0AT+3hPeiDoaxg8ZTT2Gs3RSwYiCkWqwXuZnZtAL4U4Ua4c1mZzho39TXZ0fY7VlzVWc4wHHi70bptfSHDMOSX7fMfh+qMlVqDHxu6Z7nci5AIwTkQTVpBXxBz8GnCqnxYbC7UjnzXlx4XkJ2xZQf//1/rK93mz7Wtgv/rT+8l8f+vPxub73/hxxXhoEjL2+C2sCuTZTnG6Hup0MlY5UGeHHUOF3pRpyCjD7Na0MaH9wiep69tr39fR85VxYkRr1t6/er33ggh9Kot93zc133RV33T9zziD/7uf0UkmxQVXZKlUGqlW2bKtbJQ1gpen6jTNWyV/Qt9si+O/8wkmAfHZ4GztsXJyWArZ2JmZ2N25+dQ8FurN3u8GfOm983uNye4037v/P2DBv9usLaB7/88a/CyAc3/wXnGeZJzT8qplFHnb/Rv6KfoW+vH6XfqSXwf/qH+pf4X7n+w4RDMwHZoBg2MgQXi8C+UQN0fowDjBkaxjB2cxm1YgS0oQj2OohS16EAfRjCDxfgCq/AdfoU/DVHZcGfkY5zsGoDMWUUPlnTapneCi81A69x5ay7SGgML8tOLzXaZWuXbAlFZibqXPgfmne2ALIPOhJYalhedfcXg6M98cO+SnxmfTwVqbkRpWmPSrWVqwMnHRlMFQKsTodidS1O12WsE4+gJfm2SYPRVMSDSnCXpgLado3Zva3ftex4Hbv0kor34hxe3eEcXsf6zWxtQqMI1ftgcjjUWTw1jHRFLTmOkT7Qtcofa1/Z200k6bO/SbfXI/cPMbnnbJNfD4j1Mm72i4RZgwUlKy4YUWM19u3cL7PcP9pdRbKA6eNQdbhxu3pYSeV10apukMNppg0bgxVsV2NnuyPPoTv/52XCWLNzeOqEBPWMgX705rZrrQMkS3cVPVzcKmaMXn/cDqVB1Ge4z2SNqCDSqQsvQ04TvFpEr3/mQM/XgaHWQ7zK+y0eruGws7UrEz8y3vUMFMdNMvIbcFoO07T3fcCtOzQK4zVDogfogyUojN2ad8SjL6vRqeH8MKVU6NgBTZf0rPag9MGeQdaSxe8g0LM0XrtoyFyIE0c+0Lnp7WJun27gfx0cA8VHT3yDwL6fQ8JOLPMBWNIVMFWlbmSx7pfN55bCuCTY9DMci4Mz3aAGkzx7Lx/D81RHxq/ete9uL7u+6wYWHndPcsN5m+z9+8mjzrbo782C6yj7CnpFhApRkN4dNUsIgzl5oFupGyGzRiNhg/GXagTqjysm8whk14/2XH7rGToH15sgksypbPo9/YnBMCizui1CpjL0Corn8NGSVlVrrho1rpYsy+W/mFt0p3pPPGx7OYhHrHOAZXEf8GyzYBb29eS7BJP3IZqkngM3lT81jc0AwN20DvqRPS8RwxH10WrpR+VYbDVnNl6Q8zRMgpmXHZQjc+unnxvU1Pmv0e04waf/LneTu+cnfbiR3x2TWsTyKuEXDwaDuwEZG7L933+dEXnpTo8K8KEn0bMhgSYugnpJF26g2r36ZTyt/rQ2JVU35JUBMd7qON/94onSudlKDliIOw3375YCL5fXCcNggA9louHE9EFNF5T2JVo2bpqW3u9h90gYGxtpP0XIzDutcifbePmM6frVh2b7bdXbSTCITx6vNeHQhZnGsZMZFLWd75x7admbVarpC1VutNM1QGnuBriQ313Hq7deP3QMyyQijw28/czpUztdiIZt3aI6vXeBHBhuXdVaWn6A3KC0TX7eaSaCnDjCz6oAmWvWba/nypIh7dFQgBq/4IlBK1s5q5rin0mJDjWzPjwI10QOzT9Sgu/16WbD3uZPp45X10qRwSLheTHwMFDWMqIaNeqwYxSHVETvb764UtgqANguDHSYNl565XExYNIO7UToEm9pgkZFEJZcvrFZLVEuub9JuUFFKpH5EPij66KNkfHVfgzStiDU0zOqX92XqUEGreuWHj0ktW+9juoyDelnhXr5AVIk6yymMTQDalxihAlSRyHWG8Dk1cvwoZ5E0ooH9pJqb1b8+M8Ep82z6JqfAs1e0VyZVhDh/YKf2tf0Z+5FjGx7u7N7uP+T7wPHGiWHI0Ws71YrhRQ4Asko4W4bM9Tyq48s4uw8GmUdUQbgF2HvKepTFeM2AbNFASNqiBRWDaCpSP/fQk3HuTdv5A7P4WndXs+PzkpxpaFUCGm/UVRnAkqrCkiGvCcnc2s6otp09Odr9tB4qmeF4as9DRou37VZdvk4mMpgHxGClHSST491Xzws91oa4beFijVFssb3UUlia0I2CBng0F+swGNq7Z5v3KUrlyHF7miTZWaZ6NqohtCt5iAytagGLi1PaCPwic7R9lrkJS2KiJtOQdYjzoNTykV6HHuhwkyzrRhfUMs90QgyXxZ2Y3k65ySfh4dYYTsuhn0RPzdZUT66xzIPqHxKrxZitc409VkxsbFpkdhouG5K6DotnhKYp06zAScq07/Z4KNPnWCAHPFUptiRb8m9xQS20r1qzrrMos16QNt+Ru/hdvpQomG684wxZ/0JlvbwSUdAtykq5vFDZzi8PWZnReCfUa3rVaNVnBFrZADPxjODJnpKEbTfc+Z0bWAFCqSjvIP/EO6UJbOaw6qKLzvRGAuPffMAxTVRsi7K0mis1WxXwxlymqnIBak2sjw9ggdoXq1cUmrNF+mQ/NRdYaIsnxwFiW9g6DU4D6u6St4osJm7NnamP66nYx4v4hbh9dhT3x7uTdeHnPB8TJ+WS/4/n085y/78RaReNMO/h0lyjXfSs+o8fM8OdqeS0QEi/Aa7/37dOiI4q6eMeQerrobOvvG56vHeLquSz3lpzLW13ZneCQHTqvtaLgnYw6LzfnM9UMfXE2iktPg+gMSExC0sUWwxINt4CfHfnxv3SAF+1L8S0kuE0pbTw5YDNnrbw0JoJRroRdf3S2unoUG3IR4Wv9eaDc6GaGKXIsqTUlGnOyQmktZFv2avsZfUyeRmIzEtyzI73HR1XLwCXtRynmJssksc1z2vlC2Ky04XadX86+8lG8ODZU4mJ+px/z4tCkVNB96UuGAa28wZbPtbWzQwM1peOMREfqAPhfnVlpcIeomPjUr87H3SISI1rMY3PeJMsueViVX3Pl4YS6NC+YSLyDj5pSTacLPf7KFY1A9IERbSHLTpGaAdRH6k7QjYUdRP9IPubjZGZrUdFqpaqFdPIMZ1WzGQxTwqQb8dtctW9LoqUksyr8Wa6kdnPHwvDoUmi9ePqwZAlGpwu8HoDTcOwkJidwUCIt81W+qAR3jhujLsPK1LzS2TL2vIxZ3HLBHnOln08spMobW09WE4200nL/b/jjxHSfwPwMSGqqre3M6JBNIyTce5uRWUl5bonlMtls9J86cz+azeu3t6aYMDQYN8ThkJD6SsBuJqWu4pJFaJxUcXONQxMy2bZvplmlxgpX1vBl0uraVq4lCSrC62yQYF5a51vG0TQuvLJyXkbnH/XH806ezGO5AiXsC1k7Hf6YApyuapRlUEc7Bd0pVXIWBU1u/sktXlWQfMzRq0GHMh6qrdixt8b/yli3XP7M/Df9dofo2myXGMiDRSmauLwnBHitBUZoXN2dMBR3/cdXkNN2bxZ0J6NrX3inrnZmJqKbYA0AZ91FRciqAaNz6RQ2tIk5BlZSiVNAhR8zmb2uKbOpi1rWP+itAfYWh4DwhPKvWVTUNaXQx+s+q2oysjS+ZKiAxrPhyYMR2zLw4cVhHdgnErbXvPQ04uyP8kLZ1tRP/kxhw4anFn9ljb09Ky51J8XN6Xxjo3/uXHHLM7DZmDMUzko9xyY/e8S/Th+GtEIfCU8o7QW5T8+2OBfnKA3/pGsivlOJyTXrEv8TLBgdaRL0oBh7w0hgovHTsei8tAkMa6namxlQ63vgzw4c3LQ7aPWgalnnMc/cZMdU0V/s0dG7Bx+NRkFzuhab8zIe19oERWwW/lUjRRtKNteXtaT3Qx1u1gCi5gebRIs6xODCp4u02swYPRArVYLfDl9ZzMNTtFMrTBrppoEtuwomeVFF5EXHZqKCnN77/iNRc9RUdulm7n6iaAxeSFJKBil5wKb2jTjab1ai8VKJcNoBAqq5INWr3KWKdJFJzV58mcfSHNLfSZ2QDxlCzZUvaqpI2e4McpOzHv+OEvoxqgRt+0ioF1M1mZ0s7aBd6T+wpYU+UMhph6tDbB7vb+wi6cg2fP8fhk2t4T2PFpZU685M4A5uEBkzO1/La/Sk2d2FlTvKMX4Qne4/mi9eSEEAl7ZqcCIVhYLaAuvjAlXAVATHzQRLVFSb9LcX496ZlBsyIfFdnllCbSrMRODZdLBfgHpXee2IhMQVcSBfYv7wYE0XLi917s9i6vvWzi1EhChg6cG0dvZ6sZ7Et+Q/MRaLgGU/TIm5xZSHb7b1DkKcpeiTdp/nx9E9Onodx4ZAntsm9d7KI38JjHKRNbBY52tXdaCRsGFdCSU2wzlazqyjcG8m8j5sVu9a2kKI1b/Y0OQOwZeleXthohhlZE64lKKZroHztCQ/mkwwzg1z2HZfp8Q2LjPy9YOrjhfmzqmhbN/3ZhVAPXhw8X8fX8NzOJnWmAuvBchdE57bpzl/fCjt5A9MDZ+nFARnEL1aDHZ5azQ7nARcePk7nATNPV2MGaB26JMxrt+w0Ba1AMXI2l1NYmE1RI2kG1bRD8GnMXnC1AodeR5YktX2g8FZj57/8ZNSyEfLS/nN/TKUE9HjbDaxc2KxbkF1VJW6jYGmp+Lz8NS8qrxospsHfe+UcKPP8cg+/d8ixw+5ilaOELT7ubxp0jkSGGeDpDDwovwPyLTZTcdsWmmVgiOQlGhW4GHDVMwoVGAhOq5ulYDivZRAK0wY9zfdb6m4G1KeqM23emKiHdDw9x7H7leTAAWMjY0CGIDl6HM8rxjjyFkLPJ0h0jvorxKjoVaEu8hmQquuLZN3yJ9iRlfmEyzGK9ZJtCmnWJN8PtdRUmfm04XEfWynpYeUubMZLtWTKWHLvECw9TD6U6yyRdYv7GrHJjT5GdSLbdwB6efZZFlYtpNw5UNidZMCS9SN7oZPyat8e0JPfHbQuRferR+iB9u2l7ba6pppqzwzeHedJsb+gAloo3k+QbNf5lMFy41BT+9EePOAITdab0NBeZePwc09WuKM31vP2ASbuEH0v7uLWZn044FxNPjKPvdfaCERad5I0MpyJwjYFqPKSGacMItIDhU3n7Qqn8XOL/jhK+ECFGa3sssuTrGPGMGXXEQ9Pnian+POTptoKlN7/S9wMKSczUSTJz/Jc+iNgN3UrLGcT+RJdAzTG9y8Bpesy6mzpbm6bPQgodPZIA2Pk0wAHxwMQAIkjo2AKCejh8AwGA7+O0ikH3wHQBM9CZyCpO0fIBvx5GC/yZsy1hJAQA4V/8oAD29XwBQgzFEtsPnQCqvKJohkPDoeTP0sX/eTzDM20YxAgoGDgEJDToMmLBgw4ELDz4UAoSIECNBigw5il9tGEExnCApmmE5XhAlWVE13TAt23E9PwijOEmzvCirumm7fhineVm3/XA8nS/X2/3xfL0/398fSI+qNYGafhKnzUar0+v2B6PheG19c2NrZ/tg//Do/gOAfFZJQfoEOuVq/k+cmgJl2gEwnViTNJGa1HAAQPsAyFJ4AAwYtEjyYve7INu8/ADAYsVawoRY0Pt8yJd8zafU2w2Af6MDoP9RYwdenpPP+RayRJUi1SrUqlOvRpNmADRaZrkO7/TLeHkZm4AEmpsJGRd/aGcAEMcHhgIAWBQZsNYmg3HpH0xF+QvLz+VO3/l0L1dmMlPZm+N5OH9CJhrGHIuscJdfcWRHuL7x/xmglqmVaiO1Rm2ndldvZsJQY4QTc/77t+N//58YxlHTZ431ttgbtwTn17mIsU/d7Za3GtXQ9g504wOhbDTimSZQTc/t5/nPYv1JLYwuxj8F5bVZt7pv3a6v6xDczd+1tbpW1tI+wnEOhd8nJHVsypGjYytG4dFrpB4+pkvn2+Uvyuh6J4ZvdFwnyhff+h7rFwVSVTqJAgUUKKJApc7sBQXG1c3dm8VHn1FQqJgRjmJQLDcnMwdOKrWG8/tWf0n7p9uKzJWrsmbz9RLWJn51K8qqbtputz8cr641noylfxqdwWSxOf/vH3+TUn9ESv//94jEkk/1aWhqaevo6ukbGBoZWzOxbsOmLdt27Nqz78ChI7ccO3H7Wemiu+6BEIygGE6QFM2wHC+Ikqyomm6xBnfsDqd5Bt3Pg2PD3KPyMI+Onyd5egyYCT3L84zkRV7mVV7nTd7mXRSwETAtn2Oh67fwiNieE/cjnNIMAQL4+Dhk889zj38AmRfi8neu9fzNePf5FPA37Z+NlqdQqxmG2RAgfT6UeyabsyZtz1LhFeEjCZgBfZVevMDL8WzGAEBOs51NrLvMP3SQkh50J+GGArKZ8gMBjasI5BubL3vXdnZyAIDvCb0NOBWjl82sddszGZN+i7skhnwCEwMD7+IhZVH7jMJr6GMq5ppwGppaGMEwAb9+4rz2+EJkZtQZEraApDTWReP2R/S9nFu3CGun6CiitaQloo/J7nWOXZiEXh2jopChgDsG++RbYne9+tkv2Kmxxn01Doq7VSh/EfsNOzdoxmQ+X61fsX1HyjCk42d2ZP9fECD45tN6nShLQt8SZ4TuVoFWYee04lyZxQC8kiCpI/AmDSwPwjz5W0/gB5yZgM9KHE9SDQvUH/p3SDqIvmm6rgbXf0lqx2g3drn4/YTaybMgLNdAzrWE4sDE9zupq6gM6I2blFThVoLRJUR3kGP7+ZA9WvXwYQGoJpnTNhk05gTxcLyHUQWxEMdgkkgkRyDXmGGCYQFet2jvGFo7XrZ7iAQzzcArDqepxGAFBVw3YzaHprhh7bDCvA+g53Mk5HbW4pSJUSd+BTDug+suxPBKg54u+RKaH4F/YBzE97HNMAEwv2OmcDNFDwJCjIQ8IWZCnK/8Kt0Ibo3i/OZs7w+M76vP7T2EAscLqWJfcCMSnv1LHG857uwv5DjN8Qors1X7bhuTz87wik2OQCpMdYNvxL5bquIwK+6TiVKPbBL3OH7h9Rb/3oqJagkQghHv0B7bLBjlHljo2I65fMtl7Dlvx0JotXkSr1/acoVwidDYcmaksjZSjgtwGWqQiM77iXGBPZuW3YzaXRHdGqZ2oqkE6ySn8sLG+yauHAeSeLqOAm6EBRWzmCiuxgFvbzht8SruT26xOwk3eCxb6v54fiHznH/yOhzYyaXxdScuTxcTpAcH467LwvxEK2M7+w/Ly8jYWDa5DMZrXwzelRoIUlfxb4CetniasVNW8J4pcrjWgEN7XGoUdhGZuoLrBBj+Cs8Lka7CPXChvy3H+zmI7Tc8d1KpmMUtqwkeU308ue8N4d3XYO4Aaf+62QPQKfQuE/oenB7O0BeYuzE9gmgDui24Lsb3P3WvT5FuIawKM2qnK0ZVSmrHyEsifbuPH2/0Gde/f69kOlu/AkVIuX9hOX4YMIrjhZbuDV/C3Y6b1gOKLDRcn8PSapQf9q0wVEYpPQE8ZBzpa+ojxc8DKyy2mSAABb/LhXGBNL1m3hHKydyX7hJFA3w/Eo529nz2LaXP9xHNB2SeZC/ajTXEqPXamyOmz7GiASgmwEKByENh5Omcx0Qg4u5HMA+VOgIJiqtns8d1B6F/sAcB5jXvwliy8CqRH4e2j0JLg/jZ0b8+Fwp7EeEZwDAkw+s85zrSAGQHcYi6lYFMxdFR/B63tdMod2Gu7ejTiO47ycPXKq+vQxlhEwTC4obFfvg/l1NnIs+2Bm/PhLVdMFUE3Vr4hgJW5+FxQiMfHnfcje0XXe0apeCGA1nWZHPo3Npf0fJi4jB4mnHKoGJ3InmPT/B8ddJ32xftb9UY4QHCwUcAoDT/N46ML7AvgmCVAhgMQShD0PKP5VgexcAwOCxlOAw6GYEj68pIhCDAoEU/MDq+wRgDmYEGM5rIWLRnwdZsZHEAArTRAAClZyEkdC66icGwO8kQou56DYpodBmGcrwlDtF4RpDZ2YzEcoYZDcsNY3RKH1vGwHHbIZkUfcZYWr6kD9m8svO/jVCv+dDs4tveUJ29kdsblexxWt/fbLTC/dG/XNZj8K/eadcz9tqJb4uUrT3t/yOpic9MnM3sAvRqUup243FfKT0MKOc+VMVHasvt2TUKIeqNECoWdF735WA8cqSRnZSgt74lse54Tu8shUjoPJmmQ81IjFGfIeZrjGFCidYcCQbl3RrJcRjhcYI1AYczHSLabJvWuq4PehMSXDI42U+xoLrGOgPh/ihbOMGFdQezf0tcfIqDAT5jmcBLvQAFFvT5VuZy7pIYaFzy7zqMUlT2QiIiTkhJf+BEUHOaNLchX9LGhTdILfMM8WGAjERslyaOObp19wMe3xvT5+e4L1yG94Vn1vtJ6E3BzqT0x/iQOGNgrggGEnmWJVLrza95BFYFdZr0pqn4xxnAz7ZPOrvxu9ln9GCHuHCq4qq2L77Vp8JuQPrgG8R7iwRQrDcvyLhL2T1n5FNGYDi1WhnLFfWeY4GC7lB0PGKUoMbrzuuEsy8uljTmTmNQAmvoaLLZQOmok1jKn+jxKpDVzuCgR3FYa2Y1qrBICINYO3L1m0h4Nbwz8Pagv0tjAAAA") format("woff2");}</style></defs> ++ <rect x="0" y="0" width="1775.4295814700522" height="1691.8333333333335" fill="#ffffff"/><g stroke-linecap="round" transform="translate(432.4951915572758 194.2386962872556) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C38.27 1.54, 69.71 0.86, 123.42 0 M12.25 0 C57.08 1.25, 100.41 1.21, 123.42 0 M123.42 0 C130.05 0.87, 137.62 3.02, 135.67 12.25 M123.42 0 C129.41 -0.25, 135.23 3.57, 135.67 12.25 M135.67 12.25 C137.2 17.84, 137.08 24.32, 135.67 36.75 M135.67 12.25 C135.63 18.87, 135.53 27.61, 135.67 36.75 M135.67 36.75 C135.96 45.56, 130.18 48.49, 123.42 49 M135.67 36.75 C134.67 43.8, 132.19 48.93, 123.42 49 M123.42 49 C95.98 49.68, 66.49 49.35, 12.25 49 M123.42 49 C90.86 49.22, 59.02 49.84, 12.25 49 M12.25 49 C2.25 49.89, 1.02 46.81, 0 36.75 M12.25 49 C3.81 48.52, 1.25 44.93, 0 36.75 M0 36.75 C0.91 27.08, 1.15 18.34, 0 12.25 M0 36.75 C0.47 29.41, 0.25 22.37, 0 12.25 M0 12.25 C-0.6 2.28, 3.59 0.5, 12.25 0 M0 12.25 C-0.87 4.69, 5.95 0.6, 12.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(437.6813459675752 211.34101807252205) rotate(0 62.647987365722656 7.397678214733503)"><text x="62.647987365722656" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ListPodSandboxStats</text></g><g stroke-linecap="round" transform="translate(430.80662958052994 271.7021675857842) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C34.97 0.97, 59.23 -1.52, 123.42 0 M12.25 0 C40.77 0.34, 68.84 -0.06, 123.42 0 M123.42 0 C132.18 -0.16, 134.7 5.15, 135.67 12.25 M123.42 0 C133.44 1.32, 136.16 6.03, 135.67 12.25 M135.67 12.25 C135.14 19.88, 133.97 29.85, 135.67 36.75 M135.67 12.25 C135.08 21.23, 135.94 30.75, 135.67 36.75 M135.67 36.75 C136.32 45.43, 130.5 50.16, 123.42 49 M135.67 36.75 C136.02 43.05, 130.87 48.21, 123.42 49 M123.42 49 C83.36 47.82, 40.33 49.98, 12.25 49 M123.42 49 C88.35 49.03, 52.52 49.67, 12.25 49 M12.25 49 C5.04 50.61, 0.69 45.22, 0 36.75 M12.25 49 C4 50.41, -1.68 44.04, 0 36.75 M0 36.75 C-0.96 27.04, 0.16 18.04, 0 12.25 M0 36.75 C-0.03 30.79, -0.54 24.97, 0 12.25 M0 12.25 C1.26 5.34, 4.4 -1.81, 12.25 0 M0 12.25 C1.86 5.23, 3.09 0.52, 12.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(447.3833250674895 288.80448937105064) rotate(0 51.2574462890625 7.397678214733503)"><text x="51.2574462890625" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">PodSandboxStats</text></g><g stroke-linecap="round" transform="translate(732.80662958053 231.7021675857842) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C37.96 -0.25, 58.44 0.26, 123.42 0 M123.42 0 C130.73 1.63, 137.21 3.69, 135.67 12.25 M135.67 12.25 C134.46 18.36, 134.06 28.96, 135.67 36.75 M135.67 36.75 C136.93 45.64, 130.11 49.46, 123.42 49 M123.42 49 C101.97 49.01, 78.66 48.81, 12.25 49 M12.25 49 C2.69 49.52, 1.03 43.94, 0 36.75 M0 36.75 C-1.43 30.68, -0.02 19.27, 0 12.25 M0 12.25 C1.09 3.03, 5.16 1.26, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(749.3833250674895 248.80448937105064) rotate(0 51.2574462890625 7.397678214733503)"><text x="51.2574462890625" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">PodSandboxStats</text></g><g stroke-linecap="round" transform="translate(1036.6301589922946 190.46687346813712) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C44.85 -2.05, 71.72 -2.23, 123.42 0 C128.58 0.43, 133.23 2.22, 135.67 12.25 C133.68 21.65, 135.98 28.32, 135.67 36.75 C134.29 45.33, 131.56 47.33, 123.42 49 C87.17 48.45, 48.96 48.31, 12.25 49 C1.79 48.77, 2.42 45.51, 0 36.75 C-1.34 31.01, 0.19 28.62, 0 12.25 C-1.68 3.66, 3.91 3.54, 12.25 0" stroke="none" stroke-width="0" fill="#b2f2bb"/><path d="M12.25 0 C55.92 0.03, 98.9 0.39, 123.42 0 M123.42 0 C130.81 0.13, 135.11 4.37, 135.67 12.25 M135.67 12.25 C134.3 20.91, 135.22 28.03, 135.67 36.75 M135.67 36.75 C133.87 45.1, 131.94 49.86, 123.42 49 M123.42 49 C93.16 45.99, 60.25 46.4, 12.25 49 M12.25 49 C2.17 48.99, -0.81 45.19, 0 36.75 M0 36.75 C-1.88 29.42, 1.34 27.06, 0 12.25 M0 12.25 C-0.65 4.52, 2.51 1.14, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(1041.7808290886292 200.17151703867012) rotate(0 62.6834716796875 14.795356429467063)"><text x="62.6834716796875" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">WindowsPodSandboxSt</text><text x="62.6834716796875" y="14.795356429467045" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ats</text></g><g stroke-linecap="round"><g transform="translate(571.640771356552 212.4519444542433) rotate(0 78.99999999999999 20.115948601810715)"><path d="M0 0 C26.33 6.71, 131.67 33.53, 158 40.23 M0 0 C26.33 6.71, 131.67 33.53, 158 40.23" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(571.640771356552 212.4519444542433) rotate(0 78.99999999999999 20.115948601810715)"><path d="M133.12 42.72 C142.86 41.75, 152.59 40.77, 158 40.23 M133.12 42.72 C138.9 42.14, 144.68 41.56, 158 40.23" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(571.640771356552 212.4519444542433) rotate(0 78.99999999999999 20.115948601810715)"><path d="M137.34 26.15 C145.43 31.66, 153.51 37.17, 158 40.23 M137.34 26.15 C142.14 29.42, 146.94 32.69, 158 40.23" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(572.140771356552 299.0891267760677) rotate(0 78.5 -18.431952859407374)"><path d="M0 0 C26.17 -6.14, 130.83 -30.72, 157 -36.86 M0 0 C26.17 -6.14, 130.83 -30.72, 157 -36.86" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(572.140771356552 299.0891267760677) rotate(0 78.5 -18.431952859407374)"><path d="M136.08 -23.17 C142.8 -27.57, 149.52 -31.97, 157 -36.86 M136.08 -23.17 C143.46 -28, 150.83 -32.82, 157 -36.86" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(572.140771356552 299.0891267760677) rotate(0 78.5 -18.431952859407374)"><path d="M132.18 -39.82 C140.15 -38.87, 148.12 -37.92, 157 -36.86 M132.18 -39.82 C140.93 -38.78, 149.68 -37.74, 157 -36.86" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(866.4749131325741 251.4676269031088) rotate(0 83.74469381787131 -15.503183999315013)"><path d="M0 0 C27.91 -5.17, 139.57 -25.84, 167.49 -31.01 M0 0 C27.91 -5.17, 139.57 -25.84, 167.49 -31.01" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(866.4749131325741 251.4676269031088) rotate(0 83.74469381787131 -15.503183999315013)"><path d="M145.95 -18.32 C152.59 -22.23, 159.24 -26.15, 167.49 -31.01 M145.95 -18.32 C152.88 -22.41, 159.82 -26.49, 167.49 -31.01" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(866.4749131325741 251.4676269031088) rotate(0 83.74469381787131 -15.503183999315013)"><path d="M142.83 -35.14 C150.44 -33.86, 158.04 -32.59, 167.49 -31.01 M142.83 -35.14 C150.77 -33.81, 158.71 -32.48, 167.49 -31.01" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(1335.2753504021923 866.8002068014706) rotate(0 67.83414177602208 24.499999999999993)"><path d="M12.25 0 C48.4 2.3, 80.59 -3.48, 123.42 0 C135.12 2.24, 132.67 3.45, 135.67 12.25 C134.87 19.17, 136.02 27.61, 135.67 36.75 C136.03 42.67, 132.87 46.62, 123.42 49 C92.33 51.13, 68.41 46.87, 12.25 49 C1.35 50.21, 2.35 47.1, 0 36.75 C-1.53 27.33, 2.88 25.21, 0 12.25 C2.65 0.66, 7.62 -2.14, 12.25 0" stroke="none" stroke-width="0" fill="#b2f2bb"/><path d="M12.25 0 C36.15 0.99, 57.65 -1.63, 123.42 0 M123.42 0 C129.76 -0.09, 137.47 3.94, 135.67 12.25 M135.67 12.25 C135.04 20.78, 137.39 28.03, 135.67 36.75 M135.67 36.75 C137.57 44.31, 130.51 50.01, 123.42 49 M123.42 49 C83.78 49.44, 41.91 50.36, 12.25 49 M12.25 49 C2.33 50.51, 0.22 44.23, 0 36.75 M0 36.75 C1.06 31.42, 0.65 26.8, 0 12.25 M0 12.25 C1.66 3.74, 3.45 -0.23, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(1340.916888113273 876.5048503720036) rotate(0 62.192604064941406 14.795356429467041)"><text x="62.192604064941406" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">WindowsSandboxSecuri</text><text x="62.192604064941406" y="14.795356429467045" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">tyContext</text></g><g stroke-linecap="round" transform="translate(1623.2118583387003 870.355762357026) rotate(0 67.83414177602208 24.499999999999993)"><path d="M12.25 0 C34.82 2.38, 65.14 -2.62, 123.42 0 C132.55 2.71, 132.92 7.38, 135.67 12.25 C138.41 21.02, 132.75 26.92, 135.67 36.75 C137.13 47.53, 130.64 50.29, 123.42 49 C99.96 51.19, 68.85 53, 12.25 49 C5.74 50.39, -0.82 47.01, 0 36.75 C-0.92 32.77, -2.39 29.16, 0 12.25 C-2.39 2.09, 3.17 -2.12, 12.25 0" stroke="none" stroke-width="0" fill="#b2f2bb"/><path d="M12.25 0 C50.77 0.93, 91.65 -2.07, 123.42 0 M123.42 0 C130.18 1.71, 135.66 3.48, 135.67 12.25 M135.67 12.25 C134.3 17.75, 136.53 24.4, 135.67 36.75 M135.67 36.75 C134.72 46.54, 131.53 48.22, 123.42 49 M123.42 49 C80.17 49.12, 37.41 48.66, 12.25 49 M12.25 49 C5.97 47.09, -0.29 43.38, 0 36.75 M0 36.75 C-0.68 28.89, 1.08 20.7, 0 12.25 M0 12.25 C-0.26 3.97, 3.69 -0.55, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(1630.9055887377692 880.060405927559) rotate(0 60.140411376953125 14.795356429467041)"><text x="60.140411376953125" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">WindowsNamespaceOp</text><text x="60.140411376953125" y="14.795356429467045" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">tion</text></g><g stroke-linecap="round" transform="translate(1046.7347341556933 867.8655662785945) rotate(0 67.83414177602211 24.499999999999993)"><path d="M12.25 0 C37.23 -1.58, 62 -1.35, 123.42 0 M123.42 0 C133.15 0.48, 135.69 5.67, 135.67 12.25 M135.67 12.25 C135.12 19.67, 134.75 28.09, 135.67 36.75 M135.67 36.75 C136.14 44.07, 133.24 47.34, 123.42 49 M123.42 49 C77.28 47.6, 35.52 49.55, 12.25 49 M12.25 49 C5.37 50.85, -1.57 44.77, 0 36.75 M0 36.75 C0.14 29.29, 0.47 23.19, 0 12.25 M0 12.25 C1.74 5.87, 2.69 1.82, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(1051.7375542154068 877.5702098491275) rotate(0 62.831321716308594 14.795356429467041)"><text x="62.831321716308594" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">WindowsPodSandboxCo</text><text x="62.831321716308594" y="14.795356429467045" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">nfig</text></g><g stroke-linecap="round" transform="translate(727.2510740249745 894.3426904616014) rotate(0 67.83414177602211 24.499999999999993)"><path d="M12.25 0 C37.2 -0.34, 62.75 0.6, 123.42 0 M123.42 0 C129.99 -1.92, 136.93 5.45, 135.67 12.25 M135.67 12.25 C133.93 20.9, 137.11 29.21, 135.67 36.75 M135.67 36.75 C134.26 46.39, 130.57 50.7, 123.42 49 M123.42 49 C80.56 50.04, 40.98 50.78, 12.25 49 M12.25 49 C2.24 50.03, 0.11 44.66, 0 36.75 M0 36.75 C-1.34 28.67, 1.3 22.79, 0 12.25 M0 12.25 C0.16 4.26, 4.27 0.95, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(744.2831528127153 911.4450122468678) rotate(0 50.80206298828125 7.397678214733524)"><text x="50.80206298828125" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">PodSandboxConfig</text></g><g stroke-linecap="round" transform="translate(439.28489749700736 1004.8299090607029) rotate(0 67.83414177602211 24.499999999999993)"><path d="M12.25 0 C41.72 0.2, 69.94 1.95, 123.42 0 M12.25 0 C53.15 -0.04, 93.92 -1.75, 123.42 0 M123.42 0 C132.89 -1, 137.49 4.47, 135.67 12.25 M123.42 0 C131.04 -1.8, 134.26 2.25, 135.67 12.25 M135.67 12.25 C136.19 20.51, 134.65 31.33, 135.67 36.75 M135.67 12.25 C135.95 20.31, 135.96 28.48, 135.67 36.75 M135.67 36.75 C136.19 45.65, 131.15 48.69, 123.42 49 M135.67 36.75 C137.02 43.77, 133.83 48.05, 123.42 49 M123.42 49 C84.56 47.53, 47.08 48.26, 12.25 49 M123.42 49 C81.59 48.51, 41.9 48.93, 12.25 49 M12.25 49 C3.5 50.58, 1.76 45.9, 0 36.75 M12.25 49 C3.56 46.83, 2.06 44.36, 0 36.75 M0 36.75 C-0.6 32.61, -1.2 23.85, 0 12.25 M0 36.75 C0.21 29.02, 0.27 20.46, 0 12.25 M0 12.25 C1.57 4.81, 2.82 -1.56, 12.25 0 M0 12.25 C-0.26 6.19, 4.43 0.37, 12.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(479.6303307769357 1021.9322308459693) rotate(0 27.48870849609375 7.397678214733524)"><text x="27.48870849609375" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">PullImage</text></g><g stroke-linecap="round" transform="translate(435.0734597032681 757.0450143461597) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C47.68 0.18, 81.9 -0.96, 123.42 0 M12.25 0 C55.01 0.52, 97.37 1.22, 123.42 0 M123.42 0 C131.11 -1.56, 134.44 2.49, 135.67 12.25 M123.42 0 C133.73 -2.28, 134.21 4.96, 135.67 12.25 M135.67 12.25 C135.95 19.14, 136.48 23.83, 135.67 36.75 M135.67 12.25 C135.96 20.58, 135.48 28.02, 135.67 36.75 M135.67 36.75 C136.85 43.92, 133.54 48.18, 123.42 49 M135.67 36.75 C136.78 46.76, 131.04 48.87, 123.42 49 M123.42 49 C98.14 48.43, 68.75 47.03, 12.25 49 M123.42 49 C86.86 50.55, 51.76 50.25, 12.25 49 M12.25 49 C3.63 47.12, 1.79 44.43, 0 36.75 M12.25 49 C2.42 47.92, -1.53 44.1, 0 36.75 M0 36.75 C1.17 28, -1.25 19.04, 0 12.25 M0 36.75 C0.71 27.69, -0.71 17.13, 0 12.25 M0 12.25 C-0.23 5.92, 4.38 0.32, 12.25 0 M0 12.25 C1.79 3.02, 1.91 -1.87, 12.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(455.83142197489565 774.1473361314262) rotate(0 47.07617950439453 7.397678214733517)"><text x="47.07617950439453" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">CreateContainer</text></g><g stroke-linecap="round" transform="translate(1336.8654531099419 974.0551087622549) rotate(0 67.83414177602208 24.499999999999993)"><path d="M12.25 0 C52.35 -4.22, 89.43 0.74, 123.42 0 C129.64 1.43, 138.33 4.63, 135.67 12.25 C138.06 18.55, 139.66 27.73, 135.67 36.75 C134.88 45.27, 128.09 50.38, 123.42 49 C98.1 46.45, 73.98 51.11, 12.25 49 C3.8 52.04, 1.37 43.85, 0 36.75 C2.85 26.61, 1.27 16.95, 0 12.25 C-1.45 7.27, 5.56 -2.79, 12.25 0" stroke="none" stroke-width="0" fill="#ffc9c9"/><path d="M12.25 0 C39.49 1.79, 64.74 1.93, 123.42 0 M123.42 0 C130.44 1.99, 135.17 2.86, 135.67 12.25 M135.67 12.25 C136.51 19.42, 134.81 23.8, 135.67 36.75 M135.67 36.75 C136.73 45.25, 131.06 50.21, 123.42 49 M123.42 49 C81.44 48.4, 43.43 47.57, 12.25 49 M12.25 49 C4.55 49.68, -1.07 44.07, 0 36.75 M0 36.75 C1.17 30.68, -0.22 26.71, 0 12.25 M0 12.25 C1.86 4.98, 6.08 1.4, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(1343.8967918464132 983.7597523327879) rotate(0 60.80280303955078 14.795356429467045)"><text x="60.80280303955078" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">L:nuxSandboxSecurity</text><text x="60.80280303955078" y="14.795356429467045" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">Context</text></g><g stroke-linecap="round" transform="translate(730.9177406916409 756.8132786968954) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C53.01 1.64, 90.95 -0.14, 123.42 0 M123.42 0 C131.23 1.35, 134.38 6.06, 135.67 12.25 M135.67 12.25 C135.03 19.65, 134.52 23.45, 135.67 36.75 M135.67 36.75 C134.71 43.96, 133.45 50.17, 123.42 49 M123.42 49 C98.46 50.5, 77.5 49.35, 12.25 49 M12.25 49 C3.3 50.27, 1.51 45.55, 0 36.75 M0 36.75 C1.84 29.94, -0.26 23.85, 0 12.25 M0 12.25 C0.28 5.93, 2.15 -1.26, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(754.8042582611201 773.9156004821618) rotate(0 43.94762420654297 7.397678214733517)"><text x="43.94762420654297" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ContainerConfig</text></g><g stroke-linecap="round" transform="translate(436.91774069164103 894.6760237949346) rotate(0 67.83414177602211 24.499999999999993)"><path d="M12.25 0 C58.17 0.56, 99.76 -1.1, 123.42 0 M12.25 0 C55.14 0.21, 97.84 0.83, 123.42 0 M123.42 0 C133.11 0.51, 137.25 3.81, 135.67 12.25 M123.42 0 C132.68 -2.11, 135.25 5.99, 135.67 12.25 M135.67 12.25 C133.87 16.84, 134.01 22.03, 135.67 36.75 M135.67 12.25 C136.39 16.47, 136.11 21.86, 135.67 36.75 M135.67 36.75 C137.41 44.84, 133.49 48.66, 123.42 49 M135.67 36.75 C137.28 46.16, 131.75 49.66, 123.42 49 M123.42 49 C81.77 49.04, 40.23 48.77, 12.25 49 M123.42 49 C81.17 48.25, 39.31 48.23, 12.25 49 M12.25 49 C4.43 50.82, -0.82 44.53, 0 36.75 M12.25 49 C3.42 50.92, -1.05 44.61, 0 36.75 M0 36.75 C0.64 30.08, 0.41 22, 0 12.25 M0 36.75 C0.21 30.36, 0.45 25.9, 0 12.25 M0 12.25 C-0.4 2.9, 4.7 0.02, 12.25 0 M0 12.25 C1.4 3.04, 6.07 2.24, 12.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(460.7983378509639 911.7783455802011) rotate(0 43.95354461669922 7.397678214733524)"><text x="43.95354461669922" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">RunPodSandbox</text></g><g stroke-linecap="round" transform="translate(1048.80662958053 782.2577231413399) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C53.17 -0.17, 99.66 3.51, 123.42 0 C133.03 2.78, 138.79 3.97, 135.67 12.25 C138.05 18.49, 135.47 23.91, 135.67 36.75 C137.28 42.77, 133.72 47.28, 123.42 49 C91.97 47.37, 61.74 48.94, 12.25 49 C2.17 46.75, -1.92 45.16, 0 36.75 C1.25 28.71, 2.24 24.61, 0 12.25 C1.12 3.82, 3.98 -2.06, 12.25 0" stroke="none" stroke-width="0" fill="#a5d8ff"/><path d="M12.25 0 C53.14 2.92, 93.24 2.02, 123.42 0 M123.42 0 C129.97 1.52, 137.59 4.25, 135.67 12.25 M135.67 12.25 C137.63 21.04, 135.99 30.09, 135.67 36.75 M135.67 36.75 C136.4 42.95, 133.46 49.56, 123.42 49 M123.42 49 C82.53 48.79, 44.32 49.78, 12.25 49 M12.25 49 C5.1 49.64, 1.14 43.29, 0 36.75 M0 36.75 C-0.23 31.11, -0.08 25.31, 0 12.25 M0 12.25 C0.56 2.78, 5.89 -0.68, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(1086.555779901474 799.3600449266063) rotate(0 30.084991455078125 7.397678214733517)"><text x="30.084991455078125" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">CDIDevice</text></g><g stroke-linecap="round" transform="translate(1046.6955184694189 700.4799453635619) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C57.66 -2.9, 97.28 -1.24, 123.42 0 C128.23 0.19, 132.14 2.39, 135.67 12.25 C139.02 18.77, 136.39 25, 135.67 36.75 C135.85 44.51, 133.5 51.82, 123.42 49 C102.86 47.96, 72.95 44.33, 12.25 49 C7.56 47.85, -3.39 44.54, 0 36.75 C-1.34 31.4, 1.37 20.76, 0 12.25 C-1.65 2.62, 1.11 -0.61, 12.25 0" stroke="none" stroke-width="0" fill="#ffc9c9"/><path d="M12.25 0 C35.6 1.34, 61.09 -0.93, 123.42 0 M123.42 0 C132.93 -0.64, 135.43 4.41, 135.67 12.25 M135.67 12.25 C134.04 17.51, 136.97 23.66, 135.67 36.75 M135.67 36.75 C137.52 43.33, 132.51 48.11, 123.42 49 M123.42 49 C97.49 48.32, 67.23 47.96, 12.25 49 M12.25 49 C3.24 50.27, -0.05 44.08, 0 36.75 M0 36.75 C-1.5 26.91, -1.45 20.49, 0 12.25 M0 12.25 C-0.88 5.34, 5.1 0.05, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(1097.2605104651675 717.5822671488284) rotate(0 17.269149780273438 7.397678214733517)"><text x="17.269149780273438" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">Mount</text></g><g stroke-linecap="round" transform="translate(1330.80662958053 663.5910564746731) rotate(0 67.83414177602208 24.5)"><path d="M12.25 0 C58.12 0.86, 99.16 -0.88, 123.42 0 C134.62 -3.12, 137.92 6.32, 135.67 12.25 C138.63 17.79, 138.15 23.93, 135.67 36.75 C133.01 42.43, 134.2 45.68, 123.42 49 C79.18 49.43, 39.65 50.04, 12.25 49 C5.38 48.18, -0.89 48.07, 0 36.75 C0.73 26.11, 1.9 19.72, 0 12.25 C-1.48 0.83, 5.12 1.81, 12.25 0" stroke="none" stroke-width="0" fill="#ffc9c9"/><path d="M12.25 0 C41.74 2.79, 72.94 -0.14, 123.42 0 M123.42 0 C129.97 1.57, 137.65 2.11, 135.67 12.25 M135.67 12.25 C136.7 18.4, 137.67 24.43, 135.67 36.75 M135.67 36.75 C134.88 42.96, 130.75 50.35, 123.42 49 M123.42 49 C88.15 47.43, 49.83 48.54, 12.25 49 M12.25 49 C3.32 49.5, 0.79 46.35, 0 36.75 M0 36.75 C-1.98 30.03, 1.14 22.69, 0 12.25 M0 12.25 C1.01 4.65, 3.52 -0.8, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(1347.8682875308684 680.6933782599397) rotate(0 50.772483825683594 7.397678214733531)"><text x="50.772483825683594" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">MountPropogation</text></g><g stroke-linecap="round" transform="translate(1332.0288518027521 747.1466120302287) rotate(0 67.83414177602208 24.5)"><path d="M12.25 0 C39.75 0.36, 62.6 -1.44, 123.42 0 C130.28 3.26, 132.16 1.7, 135.67 12.25 C134.97 17.6, 133.29 22.28, 135.67 36.75 C138.81 44.87, 132.82 52.21, 123.42 49 C87.48 51.09, 56.58 46.48, 12.25 49 C3.29 48.29, 3.52 45.84, 0 36.75 C1.12 32.79, 0.95 23.53, 0 12.25 C-2.71 7.46, 4.42 3.5, 12.25 0" stroke="none" stroke-width="0" fill="#a5d8ff"/><path d="M12.25 0 C54.57 -0.04, 102.05 -0.36, 123.42 0 M123.42 0 C130.8 -1.82, 134.23 4.51, 135.67 12.25 M135.67 12.25 C133.8 20.14, 135.78 30.32, 135.67 36.75 M135.67 36.75 C135.71 44.5, 131.96 47.25, 123.42 49 M123.42 49 C84.35 48.05, 43.83 49.42, 12.25 49 M12.25 49 C3.12 48.67, 1.13 43.47, 0 36.75 M0 36.75 C0.91 29.39, 1.18 25.15, 0 12.25 M0 12.25 C-0.41 4.15, 5.29 1.2, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(1370.700594286782 764.2489338154951) rotate(0 29.162399291992188 7.397678214733517)"><text x="29.162399291992188" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">IDMapping</text></g><g stroke-linecap="round" transform="translate(724.885060953079 1011.3949780433005) rotate(0 67.83414177602211 24.499999999999993)"><path d="M12.25 0 C44.02 0.48, 75.96 -0.33, 123.42 0 C128.41 1.04, 134.26 7.26, 135.67 12.25 C134.92 19.63, 133.09 26.17, 135.67 36.75 C133.03 46.63, 133.99 48.31, 123.42 49 C104.12 45.55, 77.85 47.54, 12.25 49 C1.61 52.56, 2.83 42.04, 0 36.75 C-1.35 33.45, 1.84 22.72, 0 12.25 C0.03 6.3, 2.45 0.95, 12.25 0" stroke="none" stroke-width="0" fill="#ffc9c9"/><path d="M12.25 0 C37.38 -1.04, 57.59 1.61, 123.42 0 M123.42 0 C131.41 1.43, 136.54 3.85, 135.67 12.25 M135.67 12.25 C135.62 16.77, 136.06 24.66, 135.67 36.75 M135.67 36.75 C137.04 46.57, 130.04 49.15, 123.42 49 M123.42 49 C88.5 47.7, 49.41 48.29, 12.25 49 M12.25 49 C3.44 50.68, 1.12 44.91, 0 36.75 M0 36.75 C-0.99 27.95, -1.69 18.83, 0 12.25 M0 12.25 C0.51 5.37, 2.12 -0.77, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(762.8944270333003 1028.497299828567) rotate(0 29.82477569580078 7.397678214733524)"><text x="29.82477569580078" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ImageSpec</text></g><g stroke-linecap="round" transform="translate(436.760877946543 1316.2054355596408) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C52.34 -2.91, 97.27 -1.85, 123.42 0 C133.81 -3.6, 134.22 0.5, 135.67 12.25 C137.61 22.47, 136.37 27.35, 135.67 36.75 C133.18 45.92, 128.24 48.82, 123.42 49 C95.15 53.64, 70.19 50.6, 12.25 49 C5.08 46.45, 0.58 41.54, 0 36.75 C-1.09 28.63, -2.08 19.4, 0 12.25 C-0.3 5.27, 2.41 -2.42, 12.25 0" stroke="none" stroke-width="0" fill="#a5d8ff"/><path d="M12.25 0 C45.41 -1.03, 81.46 -1.32, 123.42 0 M12.25 0 C35.81 -1.73, 58.84 -0.83, 123.42 0 M123.42 0 C132.27 0.11, 135.94 5.9, 135.67 12.25 M123.42 0 C131.5 -1.48, 135.33 3.34, 135.67 12.25 M135.67 12.25 C135.35 18.53, 135.87 26.25, 135.67 36.75 M135.67 12.25 C136.15 18.7, 135.06 23.88, 135.67 36.75 M135.67 36.75 C136.44 43.56, 131.01 50.84, 123.42 49 M135.67 36.75 C133.49 43.04, 130.52 46.72, 123.42 49 M123.42 49 C83.63 46.75, 43.52 49.08, 12.25 49 M123.42 49 C84.48 49.09, 45.06 47.58, 12.25 49 M12.25 49 C5.55 48.52, -1.85 44.83, 0 36.75 M12.25 49 C4.43 48.33, 0.69 44.81, 0 36.75 M0 36.75 C-0.83 30.33, -0.35 26.99, 0 12.25 M0 36.75 C0.89 28.57, -0.45 20.31, 0 12.25 M0 12.25 C0.42 4.94, 2.93 -0.9, 12.25 0 M0 12.25 C1.98 2.77, 2.19 -1.31, 12.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(465.869532967194 1333.3077573449073) rotate(0 38.725486755371094 7.397678214733531)"><text x="38.725486755371094" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">RuntimeConfig</text></g><g stroke-linecap="round" transform="translate(724.8262374236672 1315.4538015727126) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C59.95 -0.18, 99.28 4.64, 123.42 0 C132.63 0.02, 138.52 5.03, 135.67 12.25 C136.46 14.94, 132.29 21.49, 135.67 36.75 C135.37 42.75, 133.17 49.62, 123.42 49 C99.07 51.26, 78.79 50.77, 12.25 49 C1.65 48.79, 2.52 42.37, 0 36.75 C1.4 31.03, 1.64 19.48, 0 12.25 C-2.78 5.83, 3.6 0.31, 12.25 0" stroke="none" stroke-width="0" fill="#a5d8ff"/><path d="M12.25 0 C46.58 0.26, 84.04 0.86, 123.42 0 M123.42 0 C130.33 0.09, 134.27 4.61, 135.67 12.25 M135.67 12.25 C134.12 18.63, 135.83 27.34, 135.67 36.75 M135.67 36.75 C137 45.32, 130.34 50.85, 123.42 49 M123.42 49 C91.87 50.35, 64.47 47.32, 12.25 49 M12.25 49 C3.28 48.47, -0.55 46.85, 0 36.75 M0 36.75 C1.3 28.04, 0.28 19.46, 0 12.25 M0 12.25 C1.92 2.61, 3.25 -0.41, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(733.7500856205877 1325.1584451432454) rotate(0 58.91029357910156 14.795356429467034)"><text x="58.91029357910156" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">LinuxRuntimeConfigur</text><text x="58.91029357910156" y="14.795356429467045" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ation</text></g><g stroke-linecap="round" transform="translate(1053.3098975543862 1316.7609911151962) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C40.85 -0.75, 70.99 -0.46, 123.42 0 C129.19 -0.04, 137.92 3.19, 135.67 12.25 C138.57 21.36, 132.2 25.76, 135.67 36.75 C132.58 43.45, 128.57 48.23, 123.42 49 C86.65 49.61, 51.53 48.02, 12.25 49 C2.14 48.59, -0.18 45.32, 0 36.75 C-3.19 32.33, -0.24 24.97, 0 12.25 C-1.91 2.64, 4.62 -1.62, 12.25 0" stroke="none" stroke-width="0" fill="#a5d8ff"/><path d="M12.25 0 C55.77 0.94, 96.51 -1.66, 123.42 0 M123.42 0 C131.92 0.35, 136.47 5.55, 135.67 12.25 M135.67 12.25 C136.02 20.98, 134.27 28.71, 135.67 36.75 M135.67 36.75 C134.13 43.61, 130.25 50.18, 123.42 49 M123.42 49 C87.94 46.96, 47.48 48.05, 12.25 49 M12.25 49 C3.38 50.59, -1.52 46.39, 0 36.75 M0 36.75 C-0.54 27.95, 1.2 19.57, 0 12.25 M0 12.25 C0.68 3.52, 5.45 -0.42, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(1084.9557000970099 1333.8633129004627) rotate(0 36.18833923339844 7.397678214733531)"><text x="36.18833923339844" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">CgroupDriver</text></g><g stroke-linecap="round" transform="translate(731.3844890863434 403.7021675857842) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C52.26 1.83, 92.48 -2.78, 123.42 0 C129.04 0.42, 133.83 5.84, 135.67 12.25 C132.67 20.6, 134.35 27.62, 135.67 36.75 C137.13 47.24, 132.64 47.3, 123.42 49 C93.26 47.31, 71.23 51.51, 12.25 49 C5.81 50.31, -0.79 47.47, 0 36.75 C-2.88 31.99, -0.43 24.72, 0 12.25 C-3.1 6.69, 2.85 -2.69, 12.25 0" stroke="none" stroke-width="0" fill="#ffc9c9"/><path d="M12.25 0 C51.86 2.69, 95.82 2.16, 123.42 0 M123.42 0 C132.3 -1.29, 135.2 2.36, 135.67 12.25 M135.67 12.25 C137.06 20.22, 137.41 30.75, 135.67 36.75 M135.67 36.75 C135.19 45.62, 131.19 50.69, 123.42 49 M123.42 49 C82.02 50.49, 39.83 51.58, 12.25 49 M12.25 49 C5.05 50.62, 0.05 45.54, 0 36.75 M0 36.75 C-0.87 30.51, -0.24 21.91, 0 12.25 M0 12.25 C-0.62 3.97, 3.68 -1.22, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(754.8156233550412 420.80448937105064) rotate(0 44.40300750732422 7.397678214733503)"><text x="44.40300750732422" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ContainerStats</text></g><g stroke-linecap="round" transform="translate(1038.3844890863434 403.7021675857842) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C47.76 2.13, 86.54 3.47, 123.42 0 C131.17 -2.58, 138.83 1.41, 135.67 12.25 C137 22.8, 136.6 27, 135.67 36.75 C133.75 42.58, 131.42 46.28, 123.42 49 C96.7 51.76, 65.76 53.01, 12.25 49 C1.88 45.96, -3.34 41.66, 0 36.75 C-3 31.26, -1.14 17.88, 0 12.25 C-2.95 5.25, 6.94 2.67, 12.25 0" stroke="none" stroke-width="0" fill="#a5d8ff"/><path d="M12.25 0 C57.28 -2.1, 100.41 -2.53, 123.42 0 M123.42 0 C132.63 1.29, 136.7 5.32, 135.67 12.25 M135.67 12.25 C135.38 19.5, 135.47 27.03, 135.67 36.75 M135.67 36.75 C137.15 46.45, 133.13 49.2, 123.42 49 M123.42 49 C87.54 51.57, 49.78 50.58, 12.25 49 M12.25 49 C4.62 47.26, 1.3 43.83, 0 36.75 M0 36.75 C-0.56 27.13, -0.35 16.51, 0 12.25 M0 12.25 C0.99 4.55, 3.78 0.65, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(1074.5841398955686 420.80448937105064) rotate(0 31.634490966796875 7.397678214733503)"><text x="31.634490966796875" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">SwapUsage</text></g><g stroke-linecap="round" transform="translate(432.2287700747164 369.47043193651996) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C39.47 2.25, 65.63 2.19, 123.42 0 M12.25 0 C41.72 -0.34, 71.47 0.15, 123.42 0 M123.42 0 C133.07 1.53, 137.21 4.28, 135.67 12.25 M123.42 0 C132.31 -0.45, 137.65 5.2, 135.67 12.25 M135.67 12.25 C136.21 19.84, 136.98 29.82, 135.67 36.75 M135.67 12.25 C135.3 21.71, 135.41 30.67, 135.67 36.75 M135.67 36.75 C136.66 45.39, 131.28 49.65, 123.42 49 M135.67 36.75 C136.64 46.55, 129.62 47.37, 123.42 49 M123.42 49 C88.54 48.11, 55.89 48.05, 12.25 49 M123.42 49 C86.49 50.03, 49.05 48.91, 12.25 49 M12.25 49 C3.02 47.7, -0.09 43.4, 0 36.75 M12.25 49 C3.07 50.72, 1.66 46.02, 0 36.75 M0 36.75 C-1.17 26.82, -1.8 18.46, 0 12.25 M0 36.75 C-0.81 29.38, -0.29 19.82, 0 12.25 M0 12.25 C-1.64 4.73, 5.67 1.48, 12.25 0 M0 12.25 C-0.84 3.96, 5.85 -0.74, 12.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(444.2693632667541 386.5727537217864) rotate(0 55.793548583984375 7.397678214733503)"><text x="55.793548583984375" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ListContainerStats</text></g><g stroke-linecap="round" transform="translate(434.7166786862059 444.0515502938721) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C52.12 0.9, 94.56 -1.16, 123.42 0 M12.25 0 C50.23 -0.17, 87.51 -0.84, 123.42 0 M123.42 0 C132.21 -0.39, 137.39 5.05, 135.67 12.25 M123.42 0 C133.44 0.06, 136.39 4.7, 135.67 12.25 M135.67 12.25 C137.59 17.02, 136.99 22.08, 135.67 36.75 M135.67 12.25 C136.11 19.7, 135.47 27, 135.67 36.75 M135.67 36.75 C136.52 46.33, 129.87 47.58, 123.42 49 M135.67 36.75 C135.94 43.74, 132.71 48.73, 123.42 49 M123.42 49 C98.74 47.61, 73.32 48.98, 12.25 49 M123.42 49 C82.35 47.83, 42.3 47.72, 12.25 49 M12.25 49 C3.2 50.5, 1.45 45.87, 0 36.75 M12.25 49 C4.92 48.5, 1.63 43.51, 0 36.75 M0 36.75 C0.83 31.02, 0.04 24.47, 0 12.25 M0 36.75 C-0.73 27.95, 0.88 19.25, 0 12.25 M0 12.25 C-0.73 3.98, 5.62 -0.65, 12.25 0 M0 12.25 C0 5.49, 3.49 0.09, 12.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(458.14781295490377 461.1538720791385) rotate(0 44.40300750732422 7.397678214733531)"><text x="44.40300750732422" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ContainerStats</text></g><g stroke-linecap="round"><g transform="translate(567.640771356552 395.03491325264895) rotate(0 79.99999999999997 16.428273446076275)"><path d="M0 0 C26.67 5.48, 133.33 27.38, 160 32.86 M0 0 C26.67 5.48, 133.33 27.38, 160 32.86" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(567.640771356552 395.03491325264895) rotate(0 79.99999999999997 16.428273446076275)"><path d="M135.27 36.51 C141.33 35.61, 147.39 34.72, 160 32.86 M135.27 36.51 C144.28 35.18, 153.29 33.85, 160 32.86" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(567.640771356552 395.03491325264895) rotate(0 79.99999999999997 16.428273446076275)"><path d="M138.71 19.76 C143.92 22.96, 149.14 26.17, 160 32.86 M138.71 19.76 C146.47 24.53, 154.23 29.3, 160 32.86" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(571.3849622382502 467.8862469019774) rotate(0 78.2229848072393 -16.820431437125194)"><path d="M0 0 C26.07 -5.61, 130.37 -28.03, 156.45 -33.64 M0 0 C26.07 -5.61, 130.37 -28.03, 156.45 -33.64" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(571.3849622382502 467.8862469019774) rotate(0 78.2229848072393 -16.820431437125194)"><path d="M135.28 -20.34 C143.48 -25.5, 151.69 -30.66, 156.45 -33.64 M135.28 -20.34 C141.93 -24.53, 148.59 -28.71, 156.45 -33.64" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(571.3849622382502 467.8862469019774) rotate(0 78.2229848072393 -16.820431437125194)"><path d="M131.68 -37.06 C141.28 -35.74, 150.89 -34.41, 156.45 -33.64 M131.68 -37.06 C139.47 -35.99, 147.26 -34.91, 156.45 -33.64" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(868.640771356552 427.0246012987799) rotate(0 83.5 0.6769032358964182)"><path d="M0 0 C27.83 0.23, 139.17 1.13, 167 1.35 M0 0 C27.83 0.23, 139.17 1.13, 167 1.35" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(868.640771356552 427.0246012987799) rotate(0 83.5 0.6769032358964182)"><path d="M143.44 9.71 C152.78 6.4, 162.12 3.09, 167 1.35 M143.44 9.71 C150.38 7.25, 157.33 4.79, 167 1.35" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(868.640771356552 427.0246012987799) rotate(0 83.5 0.6769032358964182)"><path d="M143.58 -7.39 C152.86 -3.92, 162.15 -0.46, 167 1.35 M143.58 -7.39 C150.48 -4.81, 157.38 -2.24, 167 1.35" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(1037.80662958053 270.7021675857842) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C48.8 -2.21, 87.24 -0.06, 123.42 0 M123.42 0 C132.53 -1.15, 134.69 2.8, 135.67 12.25 M135.67 12.25 C134.89 16.84, 135.91 23.74, 135.67 36.75 M135.67 36.75 C136.13 46.83, 132.71 49.42, 123.42 49 M123.42 49 C93.76 48.35, 64.94 50.91, 12.25 49 M12.25 49 C3.4 48.95, -1.46 45.53, 0 36.75 M0 36.75 C-0.96 28.74, 2.04 19.08, 0 12.25 M0 12.25 C-1.77 5.87, 5.27 1.44, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(1043.3121351871184 280.4068111563172) rotate(0 62.328636169433594 14.795356429467063)"><text x="62.328636169433594" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">LinuxPodSandboxStat</text><text x="62.328636169433594" y="14.795356429467045" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">s</text></g><g stroke-linecap="round"><g transform="translate(1108.322976155541 321.76442816925066) rotate(0 -119.17463784350002 44.79517094484771)"><path d="M0 0 C-39.72 14.93, -198.62 74.66, -238.35 89.59 M0 0 C-39.72 14.93, -198.62 74.66, -238.35 89.59" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1108.322976155541 321.76442816925066) rotate(0 -119.17463784350002 44.79517094484771)"><path d="M-219.37 73.32 C-224.52 77.74, -229.68 82.16, -238.35 89.59 M-219.37 73.32 C-224.93 78.08, -230.48 82.85, -238.35 89.59" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1108.322976155541 321.76442816925066) rotate(0 -119.17463784350002 44.79517094484771)"><path d="M-213.35 89.33 C-220.14 89.4, -226.93 89.47, -238.35 89.59 M-213.35 89.33 C-220.67 89.41, -227.99 89.48, -238.35 89.59" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(867.4749131325741 260.89746891444486) rotate(0 83.74939366798343 16.375585704465152)"><path d="M0 0 C27.92 5.46, 139.58 27.29, 167.5 32.75 M0 0 C27.92 5.46, 139.58 27.29, 167.5 32.75" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(867.4749131325741 260.89746891444486) rotate(0 83.74939366798343 16.375585704465152)"><path d="M142.8 36.63 C147.94 35.83, 153.08 35.02, 167.5 32.75 M142.8 36.63 C148.25 35.78, 153.69 34.92, 167.5 32.75" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(867.4749131325741 260.89746891444486) rotate(0 83.74939366798343 16.375585704465152)"><path d="M146.08 19.85 C150.54 22.53, 154.99 25.22, 167.5 32.75 M146.08 19.85 C150.8 22.7, 155.53 25.54, 167.5 32.75" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(436.83604134523586 1114.319814644608) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C37.48 0.59, 59.06 -1, 123.42 0 M12.25 0 C46.22 0.32, 80.52 0.37, 123.42 0 M123.42 0 C132.79 1.69, 137.64 2.99, 135.67 12.25 M123.42 0 C133.57 2.17, 137.15 5.37, 135.67 12.25 M135.67 12.25 C136.28 20.69, 135.68 26.29, 135.67 36.75 M135.67 12.25 C135.93 17, 135.67 23.28, 135.67 36.75 M135.67 36.75 C135.54 46.09, 132.02 50.95, 123.42 49 M135.67 36.75 C135.97 43.99, 130.58 47.75, 123.42 49 M123.42 49 C80.9 50.2, 42.23 51.63, 12.25 49 M123.42 49 C95 48.82, 65.85 49.56, 12.25 49 M12.25 49 C3.95 50.76, -1.13 45.98, 0 36.75 M12.25 49 C4.66 47.42, 1.86 45.85, 0 36.75 M0 36.75 C1.57 28.95, 1.32 23.15, 0 12.25 M0 36.75 C-0.86 28.79, 0.15 19.26, 0 12.25 M0 12.25 C-0.53 4.66, 3.6 -1.84, 12.25 0 M0 12.25 C-1.54 6.17, 3 1.27, 12.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(446.8303622594416 1131.4221364298744) rotate(0 57.839820861816406 7.397678214733517)"><text x="57.839820861816406" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">GetContainerEvents</text></g><g stroke-linecap="round" transform="translate(723.8360413452358 1117.319814644608) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C45.72 0.76, 82.23 -2.19, 123.42 0 C131.19 -3.52, 137.93 3.84, 135.67 12.25 C136.32 19.27, 138.34 32.72, 135.67 36.75 C136.11 41.88, 130.16 51.45, 123.42 49 C92.54 51.07, 61.34 47.71, 12.25 49 C5.92 49.64, -2 43.97, 0 36.75 C-2.31 31.96, -1.59 22.63, 0 12.25 C-2.54 4.65, 4.97 -0.48, 12.25 0" stroke="none" stroke-width="0" fill="#ffc9c9"/><path d="M12.25 0 C47.53 1.66, 81.62 -0.19, 123.42 0 M123.42 0 C131.32 -1.13, 135.51 5.28, 135.67 12.25 M135.67 12.25 C137.63 23.57, 137.2 32.22, 135.67 36.75 M135.67 36.75 C135.2 45.39, 132.59 49.55, 123.42 49 M123.42 49 C79.5 50.53, 37.49 49.48, 12.25 49 M12.25 49 C2.22 48.69, -0.67 44.79, 0 36.75 M0 36.75 C0.21 27.15, -0.93 18.09, 0 12.25 M0 12.25 C1.37 3.08, 5.67 -1.56, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(728.8743533483087 1127.0244582151408) rotate(0 62.79582977294922 14.795356429467049)"><text x="62.79582977294922" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ContainerEventRespon</text><text x="62.79582977294922" y="14.795356429467045" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">se</text></g><g stroke-linecap="round"><g transform="translate(573.440662366306 778.6894423781032) rotate(0 77.81122010135698 0.6744540001214432)"><path d="M0 0 C25.94 0.22, 129.69 1.12, 155.62 1.35 M0 0 C25.94 0.22, 129.69 1.12, 155.62 1.35" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(573.440662366306 778.6894423781032) rotate(0 77.81122010135698 0.6744540001214432)"><path d="M132.06 9.7 C141.28 6.43, 150.51 3.16, 155.62 1.35 M132.06 9.7 C137.49 7.77, 142.93 5.84, 155.62 1.35" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(573.440662366306 778.6894423781032) rotate(0 77.81122010135698 0.6744540001214432)"><path d="M132.21 -7.4 C141.37 -3.98, 150.54 -0.55, 155.62 1.35 M132.21 -7.4 C137.61 -5.39, 143.01 -3.37, 155.62 1.35" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(867.6414022336638 775.2942658770384) rotate(0 88.4795166942041 -21.827410047555276)"><path d="M0 0 C29.49 -7.28, 147.47 -36.38, 176.96 -43.65 M0 0 C29.49 -7.28, 147.47 -36.38, 176.96 -43.65" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(867.6414022336638 775.2942658770384) rotate(0 88.4795166942041 -21.827410047555276)"><path d="M156.2 -29.73 C161.92 -33.56, 167.64 -37.4, 176.96 -43.65 M156.2 -29.73 C161.73 -33.44, 167.26 -37.14, 176.96 -43.65" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(867.6414022336638 775.2942658770384) rotate(0 88.4795166942041 -21.827410047555276)"><path d="M152.1 -46.33 C158.95 -45.59, 165.8 -44.86, 176.96 -43.65 M152.1 -46.33 C158.72 -45.62, 165.34 -44.91, 176.96 -43.65" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(1183.363802021463 720.220169330809) rotate(0 72.79409471822296 -14.074342428347506)"><path d="M0 0 C24.26 -4.69, 121.32 -23.46, 145.59 -28.15 M0 0 C24.26 -4.69, 121.32 -23.46, 145.59 -28.15" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1183.363802021463 720.220169330809) rotate(0 72.79409471822296 -14.074342428347506)"><path d="M124.15 -15.29 C132.3 -20.18, 140.45 -25.07, 145.59 -28.15 M124.15 -15.29 C132.39 -20.24, 140.63 -25.18, 145.59 -28.15" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1183.363802021463 720.220169330809) rotate(0 72.79409471822296 -14.074342428347506)"><path d="M120.9 -32.08 C130.29 -30.59, 139.67 -29.09, 145.59 -28.15 M120.9 -32.08 C130.39 -30.57, 139.88 -29.06, 145.59 -28.15" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(1172.252690910352 740.6147763344474) rotate(0 76.96076138488957 17.556018587146355)"><path d="M0 0 C25.65 5.85, 128.27 29.26, 153.92 35.11 M0 0 C25.65 5.85, 128.27 29.26, 153.92 35.11" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1172.252690910352 740.6147763344474) rotate(0 76.96076138488957 17.556018587146355)"><path d="M129.12 38.22 C137.43 37.18, 145.74 36.14, 153.92 35.11 M129.12 38.22 C138.56 37.04, 148 35.85, 153.92 35.11" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1172.252690910352 740.6147763344474) rotate(0 76.96076138488957 17.556018587146355)"><path d="M132.92 21.55 C139.96 26.09, 146.99 30.64, 153.92 35.11 M132.92 21.55 C140.91 26.71, 148.91 31.88, 153.92 35.11" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(868.4357644662695 779.3095890115678) rotate(0 89.260562445838 14.464971459807117)"><path d="M0 0 C29.75 4.82, 148.77 24.11, 178.52 28.93 M0 0 C29.75 4.82, 148.77 24.11, 178.52 28.93" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(868.4357644662695 779.3095890115678) rotate(0 89.260562445838 14.464971459807117)"><path d="M153.96 33.61 C160.83 32.3, 167.69 30.99, 178.52 28.93 M153.96 33.61 C160.09 32.44, 166.22 31.28, 178.52 28.93" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(868.4357644662695 779.3095890115678) rotate(0 89.260562445838 14.464971459807117)"><path d="M156.7 16.73 C162.8 20.14, 168.9 23.55, 178.52 28.93 M156.7 16.73 C162.14 19.77, 167.59 22.82, 178.52 28.93" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(731.0288518027521 657.4617513809645) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C51.68 -0.68, 88.22 -0.61, 123.42 0 M123.42 0 C132.8 -0.85, 133.88 4.4, 135.67 12.25 M135.67 12.25 C134.54 23.57, 137.41 31.03, 135.67 36.75 M135.67 36.75 C133.83 44.61, 132.59 49.89, 123.42 49 M123.42 49 C86.05 49.07, 49.77 49.63, 12.25 49 M12.25 49 C2.96 49.39, 0.88 43.25, 0 36.75 M0 36.75 C-0.77 29.58, 1.14 22.97, 0 12.25 M0 12.25 C0.43 2.88, 2.32 -0.06, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(751.1007789181297 674.5640731662311) rotate(0 47.76221466064453 7.397678214733531)"><text x="47.76221466064453" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ContainerStatus</text></g><g stroke-linecap="round" transform="translate(435.47329624719646 656.3506402698536) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C45.15 -1.76, 76.38 0.27, 123.42 0 M12.25 0 C43.36 0.19, 76.8 -0.45, 123.42 0 M123.42 0 C131.52 1.33, 136.53 3.3, 135.67 12.25 M123.42 0 C130.73 -0.1, 133.75 5.34, 135.67 12.25 M135.67 12.25 C135.27 19.55, 137.05 29.03, 135.67 36.75 M135.67 12.25 C135.5 21.96, 135.74 30.72, 135.67 36.75 M135.67 36.75 C134.94 46.4, 132.19 50.55, 123.42 49 M135.67 36.75 C134.43 43.38, 130.19 48.05, 123.42 49 M123.42 49 C88.62 49.61, 56.71 48.25, 12.25 49 M123.42 49 C92.2 49.46, 62.75 49.41, 12.25 49 M12.25 49 C5.17 50.01, 0.48 44.48, 0 36.75 M12.25 49 C5.71 48.67, 0.69 43.38, 0 36.75 M0 36.75 C1.43 27.28, 0.34 20.67, 0 12.25 M0 36.75 C0.1 27.49, 0.2 18.86, 0 12.25 M0 12.25 C0.65 3.19, 3.94 0.14, 12.25 0 M0 12.25 C-1.01 3.71, 4.25 2.28, 12.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(455.545223362574 673.45296205512) rotate(0 47.76221466064453 7.397678214733531)"><text x="47.76221466064453" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ContainerStatus</text></g><g stroke-linecap="round"><g transform="translate(573.2739956996395 682.3984084919543) rotate(0 77.811220101357 0.6744540001214432)"><path d="M0 0 C25.94 0.22, 129.69 1.12, 155.62 1.35 M0 0 C25.94 0.22, 129.69 1.12, 155.62 1.35" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(573.2739956996395 682.3984084919543) rotate(0 77.811220101357 0.6744540001214432)"><path d="M132.06 9.7 C137.06 7.92, 142.06 6.15, 155.62 1.35 M132.06 9.7 C137.7 7.7, 143.34 5.7, 155.62 1.35" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(573.2739956996395 682.3984084919543) rotate(0 77.811220101357 0.6744540001214432)"><path d="M132.21 -7.4 C137.17 -5.55, 142.14 -3.69, 155.62 1.35 M132.21 -7.4 C137.81 -5.31, 143.42 -3.21, 155.62 1.35" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(866.6579866884917 688.3514980420296) rotate(0 88.90435329551116 15.523607547261435)"><path d="M0 0 C29.63 5.17, 148.17 25.87, 177.81 31.05 M0 0 C29.63 5.17, 148.17 25.87, 177.81 31.05" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(866.6579866884917 688.3514980420296) rotate(0 88.90435329551116 15.523607547261435)"><path d="M153.2 35.43 C162 33.86, 170.81 32.29, 177.81 31.05 M153.2 35.43 C162.43 33.79, 171.66 32.14, 177.81 31.05" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(866.6579866884917 688.3514980420296) rotate(0 88.90435329551116 15.523607547261435)"><path d="M156.14 18.58 C163.89 23.04, 171.65 27.5, 177.81 31.05 M156.14 18.58 C164.26 23.26, 172.39 27.93, 177.81 31.05" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(848.3020441859144 806.8132786968954) rotate(0 -0.10930845690953106 101.79084967320256)"><path d="M0 0 C-0.04 33.93, -0.18 169.65, -0.22 203.58 M0 0 C-0.04 33.93, -0.18 169.65, -0.22 203.58" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(848.3020441859144 806.8132786968954) rotate(0 -0.10930845690953106 101.79084967320256)"><path d="M-8.74 180.08 C-5.82 188.15, -2.89 196.23, -0.22 203.58 M-8.74 180.08 C-6.72 185.67, -4.69 191.26, -0.22 203.58" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(848.3020441859144 806.8132786968954) rotate(0 -0.10930845690953106 101.79084967320256)"><path d="M8.36 180.1 C5.41 188.17, 2.46 196.23, -0.22 203.58 M8.36 180.1 C6.32 185.68, 4.28 191.27, -0.22 203.58" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(577.235398607311 1339.3866633743935) rotate(0 71.72563418714951 1.8430938087596758)"><path d="M0 0 C23.91 0.61, 119.54 3.07, 143.45 3.69 M0 0 C23.91 0.61, 119.54 3.07, 143.45 3.69" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(577.235398607311 1339.3866633743935) rotate(0 71.72563418714951 1.8430938087596758)"><path d="M119.75 11.63 C127.54 9.02, 135.34 6.4, 143.45 3.69 M119.75 11.63 C128.94 8.55, 138.13 5.47, 143.45 3.69" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(577.235398607311 1339.3866633743935) rotate(0 71.72563418714951 1.8430938087596758)"><path d="M120.19 -5.46 C127.84 -2.45, 135.49 0.56, 143.45 3.69 M120.19 -5.46 C129.21 -1.92, 138.23 1.63, 143.45 3.69" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(1350.473296247196 193.73952915874236) rotate(0 67.83414177602208 24.5)"><path d="M12.25 0 C49.03 -0.76, 83.25 3.9, 123.42 0 C131.5 -3.18, 134.71 6.43, 135.67 12.25 C134.11 15.9, 136.41 23.77, 135.67 36.75 C137.89 47.69, 135.06 47.17, 123.42 49 C84.78 51, 50.01 53.22, 12.25 49 C4 52.44, 1.25 41.52, 0 36.75 C-0.38 30.68, 2.42 27.49, 0 12.25 C-1.56 4.36, 0.97 2.96, 12.25 0" stroke="none" stroke-width="0" fill="#b2f2bb"/><path d="M12.25 0 C56.06 -0.8, 95.14 0.03, 123.42 0 M123.42 0 C130.03 1.55, 136.85 5.76, 135.67 12.25 M135.67 12.25 C135.09 21.72, 135.07 28.79, 135.67 36.75 M135.67 36.75 C134.21 46.79, 131.38 49.85, 123.42 49 M123.42 49 C85.01 50.36, 46.16 50.91, 12.25 49 M12.25 49 C6.04 47.47, -0.34 45.82, 0 36.75 M0 36.75 C1.38 26.32, -0.22 19.13, 0 12.25 M0 12.25 C1.45 4.14, 3.22 1.64, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(1368.723682530054 210.8418509440088) rotate(0 49.58375549316406 7.397678214733503)"><text x="49.58375549316406" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">WindowsCpuUsage</text></g><g stroke-linecap="round"><g transform="translate(1173.2984425443387 218.158753443828) rotate(0 87.44874007321184 -0.11638676391532954)"><path d="M0 0 C29.15 -0.04, 145.75 -0.19, 174.9 -0.23 M0 0 C29.15 -0.04, 145.75 -0.19, 174.9 -0.23" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1173.2984425443387 218.158753443828) rotate(0 87.44874007321184 -0.11638676391532954)"><path d="M151.42 8.35 C158.72 5.68, 166.02 3.01, 174.9 -0.23 M151.42 8.35 C156.41 6.52, 161.4 4.7, 174.9 -0.23" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1173.2984425443387 218.158753443828) rotate(0 87.44874007321184 -0.11638676391532954)"><path d="M151.39 -8.75 C158.7 -6.1, 166.01 -3.45, 174.9 -0.23 M151.39 -8.75 C156.39 -6.94, 161.39 -5.13, 174.9 -0.23" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(578.3066754382014 1029.51607156414) rotate(0 71.7001089902458 0.6945006015921535)"><path d="M0 0 C23.9 0.23, 119.5 1.16, 143.4 1.39 M0 0 C23.9 0.23, 119.5 1.16, 143.4 1.39" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(578.3066754382014 1029.51607156414) rotate(0 71.7001089902458 0.6945006015921535)"><path d="M119.83 9.71 C129.02 6.47, 138.21 3.22, 143.4 1.39 M119.83 9.71 C127.71 6.93, 135.58 4.15, 143.4 1.39" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(578.3066754382014 1029.51607156414) rotate(0 71.7001089902458 0.6945006015921535)"><path d="M119.99 -7.39 C129.12 -3.97, 138.24 -0.55, 143.4 1.39 M119.99 -7.39 C127.82 -4.46, 135.64 -1.52, 143.4 1.39" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(1182.0842841485546 892.6341547496819) rotate(0 76.09553312681888 0.8931206790790114)"><path d="M0 0 C25.37 0.3, 126.83 1.49, 152.19 1.79 M0 0 C25.37 0.3, 126.83 1.49, 152.19 1.79" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1182.0842841485546 892.6341547496819) rotate(0 76.09553312681888 0.8931206790790114)"><path d="M128.6 10.06 C134.69 7.92, 140.78 5.79, 152.19 1.79 M128.6 10.06 C134.78 7.89, 140.97 5.72, 152.19 1.79" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1182.0842841485546 892.6341547496819) rotate(0 76.09553312681888 0.8931206790790114)"><path d="M128.8 -7.04 C134.84 -4.76, 140.88 -2.48, 152.19 1.79 M128.8 -7.04 C134.93 -4.73, 141.06 -2.41, 152.19 1.79" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(1469.974127618721 894.1113770890134) rotate(0 75.56330980443397 1.2560757579846253)"><path d="M0 0 C25.19 0.42, 125.94 2.09, 151.13 2.51 M0 0 C25.19 0.42, 125.94 2.09, 151.13 2.51" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1469.974127618721 894.1113770890134) rotate(0 75.56330980443397 1.2560757579846253)"><path d="M127.5 10.67 C134.49 8.26, 141.49 5.84, 151.13 2.51 M127.5 10.67 C134.34 8.31, 141.19 5.94, 151.13 2.51" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1469.974127618721 894.1113770890134) rotate(0 75.56330980443397 1.2560757579846253)"><path d="M127.78 -6.43 C134.69 -3.78, 141.61 -1.13, 151.13 2.51 M127.78 -6.43 C134.55 -3.84, 141.31 -1.25, 151.13 2.51" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(865.7911210827735 912.1703063344562) rotate(0 89.86925811691574 -6.296079407910341)"><path d="M0 0 C29.96 -2.1, 149.78 -10.49, 179.74 -12.59 M0 0 C29.96 -2.1, 149.78 -10.49, 179.74 -12.59" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(865.7911210827735 912.1703063344562) rotate(0 89.86925811691574 -6.296079407910341)"><path d="M156.9 -2.42 C161.79 -4.6, 166.68 -6.77, 179.74 -12.59 M156.9 -2.42 C164.17 -5.66, 171.44 -8.9, 179.74 -12.59" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(865.7911210827735 912.1703063344562) rotate(0 89.86925811691574 -6.296079407910341)"><path d="M155.71 -19.48 C160.85 -18.01, 165.99 -16.53, 179.74 -12.59 M155.71 -19.48 C163.36 -17.29, 171.01 -15.09, 179.74 -12.59" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(571.3466766383291 914.4405922685852) rotate(0 76.96076138488957 0.4950153215298272)"><path d="M0 0 C25.65 0.17, 128.27 0.83, 153.92 0.99 M0 0 C25.65 0.17, 128.27 0.83, 153.92 0.99" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(571.3466766383291 914.4405922685852) rotate(0 76.96076138488957 0.4950153215298272)"><path d="M130.37 9.39 C135.28 7.64, 140.18 5.89, 153.92 0.99 M130.37 9.39 C139.46 6.15, 148.54 2.91, 153.92 0.99" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(571.3466766383291 914.4405922685852) rotate(0 76.96076138488957 0.4950153215298272)"><path d="M130.48 -7.71 C135.36 -5.9, 140.24 -4.09, 153.92 0.99 M130.48 -7.71 C139.52 -4.36, 148.56 -1, 153.92 0.99" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(573.0360054748721 785.1626448203579) rotate(0 76.9731169500391 54.10842964922438)"><path d="M0 0 C25.66 18.04, 128.29 90.18, 153.95 108.22 M0 0 C25.66 18.04, 128.29 90.18, 153.95 108.22" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(573.0360054748721 785.1626448203579) rotate(0 76.9731169500391 54.10842964922438)"><path d="M129.81 101.7 C135.65 103.28, 141.48 104.85, 153.95 108.22 M129.81 101.7 C137.79 103.85, 145.76 106.01, 153.95 108.22" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(573.0360054748721 785.1626448203579) rotate(0 76.9731169500391 54.10842964922438)"><path d="M139.64 87.71 C143.1 92.67, 146.56 97.63, 153.95 108.22 M139.64 87.71 C144.37 94.49, 149.1 101.26, 153.95 108.22" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(577.9706459977479 1022.2789207882853) rotate(0 74.05305016671639 -42.834911163113695)"><path d="M0 0 C24.68 -14.28, 123.42 -71.39, 148.11 -85.67 M0 0 C24.68 -14.28, 123.42 -71.39, 148.11 -85.67" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(577.9706459977479 1022.2789207882853) rotate(0 74.05305016671639 -42.834911163113695)"><path d="M132.05 -66.51 C137.09 -72.52, 142.12 -78.52, 148.11 -85.67 M132.05 -66.51 C136.95 -72.35, 141.85 -78.2, 148.11 -85.67" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(577.9706459977479 1022.2789207882853) rotate(0 74.05305016671639 -42.834911163113695)"><path d="M123.49 -81.31 C131.21 -82.68, 138.93 -84.04, 148.11 -85.67 M123.49 -81.31 C131 -82.64, 138.51 -83.97, 148.11 -85.67" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(866.0749389098086 917.8072994872282) rotate(0 91.49848420990492 38.68877811444753)"><path d="M0 0 C30.5 12.9, 152.5 64.48, 183 77.38 M0 0 C30.5 12.9, 152.5 64.48, 183 77.38" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(866.0749389098086 917.8072994872282) rotate(0 91.49848420990492 38.68877811444753)"><path d="M158.03 76.1 C163.3 76.37, 168.58 76.64, 183 77.38 M158.03 76.1 C166.14 76.52, 174.25 76.93, 183 77.38" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(866.0749389098086 917.8072994872282) rotate(0 91.49848420990492 38.68877811444753)"><path d="M164.69 60.35 C168.56 63.95, 172.42 67.55, 183 77.38 M164.69 60.35 C170.64 65.88, 176.59 71.42, 183 77.38" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(1050.0719073296184 974.9440096251715) rotate(0 67.83414177602211 24.499999999999993)"><path d="M12.25 0 C52.77 -1.08, 91.85 1.27, 123.42 0 M123.42 0 C132.09 -0.38, 133.77 5.65, 135.67 12.25 M135.67 12.25 C137.63 21.84, 135 31.08, 135.67 36.75 M135.67 36.75 C135.84 45.58, 130.16 49.68, 123.42 49 M123.42 49 C99.05 50.22, 74.03 47.26, 12.25 49 M12.25 49 C5.9 50.32, 1.07 43.53, 0 36.75 M0 36.75 C-0.63 26.15, -0.29 18.86, 0 12.25 M0 12.25 C-1.74 3.06, 3.13 0.46, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(1055.7844060392342 984.6486531957045) rotate(0 62.12164306640625 14.795356429467045)"><text x="62.12164306640625" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">LinuxPodSandboxConfi</text><text x="62.12164306640625" y="14.795356429467045" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">g</text></g><g stroke-linecap="round"><g transform="translate(1186.7401908816626 996.4796617787745) rotate(0 73.91291060319878 1.8784479195910286)"><path d="M0 0 C24.64 0.63, 123.19 3.13, 147.83 3.76 M0 0 C24.64 0.63, 123.19 3.13, 147.83 3.76" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1186.7401908816626 996.4796617787745) rotate(0 73.91291060319878 1.8784479195910286)"><path d="M124.12 11.71 C130.29 9.64, 136.45 7.57, 147.83 3.76 M124.12 11.71 C133.15 8.68, 142.17 5.65, 147.83 3.76" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1186.7401908816626 996.4796617787745) rotate(0 73.91291060319878 1.8784479195910286)"><path d="M124.56 -5.39 C130.61 -3.01, 136.66 -0.63, 147.83 3.76 M124.56 -5.39 C133.42 -1.91, 142.28 1.58, 147.83 3.76" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(860.2980619773148 1338.306798169353) rotate(0 96.0059177885357 1.8349593476192751)"><path d="M0 0 C32 0.61, 160.01 3.06, 192.01 3.67 M0 0 C32 0.61, 160.01 3.06, 192.01 3.67" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(860.2980619773148 1338.306798169353) rotate(0 96.0059177885357 1.8349593476192751)"><path d="M168.36 11.77 C173.89 9.88, 179.42 7.98, 192.01 3.67 M168.36 11.77 C177.23 8.73, 186.09 5.7, 192.01 3.67" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(860.2980619773148 1338.306798169353) rotate(0 96.0059177885357 1.8349593476192751)"><path d="M168.69 -5.33 C174.14 -3.22, 179.59 -1.12, 192.01 3.67 M168.69 -5.33 C177.43 -1.96, 186.17 1.42, 192.01 3.67" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(573.5043248972801 1136.3988325425858) rotate(0 73.34503424636205 0.6933682538536488)"><path d="M0 0 C24.45 0.23, 122.24 1.16, 146.69 1.39 M0 0 C24.45 0.23, 122.24 1.16, 146.69 1.39" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(573.5043248972801 1136.3988325425858) rotate(0 73.34503424636205 0.6933682538536488)"><path d="M123.12 9.71 C132.43 6.42, 141.75 3.13, 146.69 1.39 M123.12 9.71 C127.99 7.99, 132.86 6.27, 146.69 1.39" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(573.5043248972801 1136.3988325425858) rotate(0 73.34503424636205 0.6933682538536488)"><path d="M123.28 -7.39 C132.53 -3.92, 141.78 -0.45, 146.69 1.39 M123.28 -7.39 C128.12 -5.57, 132.95 -3.76, 146.69 1.39" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(190.8954367413835 717.6676998825245) rotate(0 28.459141776022108 14.5)"><path d="M7.25 0 C22.47 0.96, 35.44 1.05, 49.67 0 M7.25 0 C19.57 -0.58, 29.82 -0.66, 49.67 0 M49.67 0 C56.11 1.71, 56.24 2.04, 56.92 7.25 M49.67 0 C56.6 -0.4, 58.55 3.5, 56.92 7.25 M56.92 7.25 C57.84 11.89, 56.37 17.08, 56.92 21.75 M56.92 7.25 C57.19 12.2, 56.26 18.78, 56.92 21.75 M56.92 21.75 C55.53 25.19, 56.5 30.77, 49.67 29 M56.92 21.75 C58.21 28.15, 53.48 27.59, 49.67 29 M49.67 29 C34.46 28.12, 21.15 27.91, 7.25 29 M49.67 29 C38.26 29.2, 25.66 29.51, 7.25 29 M7.25 29 C1.54 27.68, 0.89 26.76, 0 21.75 M7.25 29 C2.68 31.2, -0.19 26.24, 0 21.75 M0 21.75 C-0.23 16.83, 1.21 13.58, 0 7.25 M0 21.75 C-0.23 17.25, 0.04 12.34, 0 7.25 M0 7.25 C1.85 1.35, 3.18 0.6, 7.25 0 M0 7.25 C0.96 2.27, 1.5 1.34, 7.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g stroke-linecap="round" transform="translate(11.770436741383492 717.6676998825245) rotate(0 28.45914177602208 14.5)"><path d="M7.25 0 C16.26 -1.51, 26.66 3.03, 49.67 0 C54.02 0.86, 57.57 -1.18, 56.92 7.25 C56.58 13.55, 56.08 20.6, 56.92 21.75 C58.6 25.45, 54.66 30.06, 49.67 29 C36.06 31.69, 13.94 26.06, 7.25 29 C-0.53 28.97, -1.13 24.58, 0 21.75 C1.95 17.99, 1 13.32, 0 7.25 C1.74 3.98, 2.82 -0.37, 7.25 0" stroke="none" stroke-width="0" fill="#a5d8ff"/><path d="M7.25 0 C17.98 -1.84, 29.71 0.81, 49.67 0 M7.25 0 C18.89 0.19, 30.25 -0.86, 49.67 0 M49.67 0 C53.49 0.99, 57.95 3.46, 56.92 7.25 M49.67 0 C52.83 -2.26, 54.84 0.38, 56.92 7.25 M56.92 7.25 C56.18 9.02, 56 12.67, 56.92 21.75 M56.92 7.25 C56.36 12.38, 57.11 17.53, 56.92 21.75 M56.92 21.75 C54.94 25.29, 56.27 27.43, 49.67 29 M56.92 21.75 C55.57 28.54, 53.16 28.8, 49.67 29 M49.67 29 C36.16 29.79, 26.82 26.72, 7.25 29 M49.67 29 C37.75 27.83, 25.56 29.45, 7.25 29 M7.25 29 C4.15 27.12, -0.22 25.03, 0 21.75 M7.25 29 C2.28 28.26, 1.7 25.88, 0 21.75 M0 21.75 C0.16 17.22, 0.34 14.92, 0 7.25 M0 21.75 C-0.47 16.95, 0.62 12.83, 0 7.25 M0 7.25 C0.87 2.52, 0.94 -0.97, 7.25 0 M0 7.25 C-1.34 2.49, 2.24 1.35, 7.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(99.22957851740568 667.1676998825245) rotate(0 24.000045776367188 10)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="16px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">无变更</text></g><g transform="translate(100.22953274103838 723.4176998825246) rotate(0 16.000030517578125 10)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="16px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">新增</text></g><g stroke-linecap="round" transform="translate(191.8329367413835 800.1676998825246) rotate(0 28.459141776022108 14.5)"><path d="M7.25 0 C20.73 0.66, 35.61 1.39, 49.67 0 M49.67 0 C55.84 0.42, 58.86 3.94, 56.92 7.25 M56.92 7.25 C56.67 12.23, 56.52 18.54, 56.92 21.75 M56.92 21.75 C57.51 25.79, 56.33 28.62, 49.67 29 M49.67 29 C31.86 30.65, 17.6 30.27, 7.25 29 M7.25 29 C1.29 28.27, 0.65 28.07, 0 21.75 M0 21.75 C1.32 18.48, -0.62 11.43, 0 7.25 M0 7.25 C0.02 0.68, 0.81 -0.83, 7.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g stroke-linecap="round" transform="translate(10.207936741383492 797.6676998825246) rotate(0 28.45914177602208 14.5)"><path d="M7.25 0 C22.43 0.79, 34.43 1.63, 49.67 0 C53.54 -1.26, 59.87 4.57, 56.92 7.25 C59.25 8.58, 54.91 14.64, 56.92 21.75 C57.53 27.64, 53.58 27.03, 49.67 29 C35.65 28.18, 20.44 31.03, 7.25 29 C-0.55 26.97, 2.02 28.52, 0 21.75 C-1.45 16.59, 2.11 9.43, 0 7.25 C0.36 3.17, 4.4 -0.65, 7.25 0" stroke="none" stroke-width="0" fill="#ffc9c9"/><path d="M7.25 0 C22.57 1.51, 40.7 0.21, 49.67 0 M7.25 0 C24.54 -0.61, 40.7 -0.64, 49.67 0 M49.67 0 C53.86 0.58, 57.51 1.63, 56.92 7.25 M49.67 0 C56.61 -0.44, 58.63 2.04, 56.92 7.25 M56.92 7.25 C56.57 13.42, 55.96 17.7, 56.92 21.75 M56.92 7.25 C57.29 12.13, 57.71 17.52, 56.92 21.75 M56.92 21.75 C56.13 24.79, 54.52 27.26, 49.67 29 M56.92 21.75 C55.07 25.62, 53.19 26.87, 49.67 29 M49.67 29 C41.95 29.45, 33.19 29.64, 7.25 29 M49.67 29 C38.13 27.91, 25.01 28.1, 7.25 29 M7.25 29 C0.44 28.4, 0.39 26.31, 0 21.75 M7.25 29 C0.13 27.85, 1.98 27.31, 0 21.75 M0 21.75 C0.38 15.34, -0.46 12.82, 0 7.25 M0 21.75 C-0.68 18.48, -0.45 14.78, 0 7.25 M0 7.25 C-0.73 0.91, 1.24 0.02, 7.25 0 M0 7.25 C-0.56 0.78, 3.14 0.64, 7.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(101.16703274103838 803.4176998825246) rotate(0 16.000030517578125 10)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="16px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">修改</text></g><g transform="translate(279.76079799982733 724.1676998825246) rotate(0 16.000030517578125 10)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="16px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">接口</text></g><g transform="translate(280.69829799982733 804.1676998825246) rotate(0 32.00006103515625 10)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="16px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">内部数据</text></g><g stroke-linecap="round" transform="translate(10 877.6676998825245) rotate(0 28.45914177602208 14.499999999999993)"><path d="M7.25 0 C21.86 3.4, 35.33 0.84, 49.67 0 C55.09 2.05, 58.81 1.02, 56.92 7.25 C55.97 10.96, 58.66 13.43, 56.92 21.75 C59.77 25.14, 53.54 29.59, 49.67 29 C34.86 28.94, 23.74 31.73, 7.25 29 C2.96 30.48, -0.19 28.52, 0 21.75 C-2.47 18.76, -2.33 12.65, 0 7.25 C-2.87 0.71, 2.53 -2.13, 7.25 0" stroke="none" stroke-width="0" fill="#b2f2bb"/><path d="M7.25 0 C20.74 1.66, 33.56 1.55, 49.67 0 M7.25 0 C19.85 -0.96, 32.47 -0.34, 49.67 0 M49.67 0 C53.66 -0.08, 58.57 4.17, 56.92 7.25 M49.67 0 C54.85 -0.47, 57.22 1.28, 56.92 7.25 M56.92 7.25 C56.09 10.24, 56.51 16.32, 56.92 21.75 M56.92 7.25 C57.61 10.52, 57.76 14.74, 56.92 21.75 M56.92 21.75 C55.18 27.23, 55.15 27.26, 49.67 29 M56.92 21.75 C58.97 24.66, 52.3 29.2, 49.67 29 M49.67 29 C35.87 27.16, 18.47 29.82, 7.25 29 M49.67 29 C36.38 28.29, 22.32 28.57, 7.25 29 M7.25 29 C0.85 30.57, 0.64 24.68, 0 21.75 M7.25 29 C3.01 28.45, -1.62 25.55, 0 21.75 M0 21.75 C-0.57 16.84, -0.86 13.58, 0 7.25 M0 21.75 C0.21 17.93, -0.56 12.79, 0 7.25 M0 7.25 C0.1 2.16, 3.01 -1.45, 7.25 0 M0 7.25 C1.09 3.3, 3.16 -0.28, 7.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(99.709095999655 887.1676998825246) rotate(0 24.000045776367188 10)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="16px" fill="#1e1e1e" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">不支持</text></g><g stroke-linecap="round" transform="translate(1350.3559291693493 279.16666666666663) rotate(0 67.83414177602208 24.5)"><path d="M12.25 0 C49.07 -1.55, 87.59 1.21, 123.42 0 C130.37 1.51, 132.64 7.05, 135.67 12.25 C134.46 20.39, 138.44 27.33, 135.67 36.75 C139.18 41.35, 128.27 47.21, 123.42 49 C100.8 49.03, 78.45 51.16, 12.25 49 C2.6 49.08, -0.37 42.43, 0 36.75 C0.66 29.82, 0.68 21.34, 0 12.25 C0.38 6.37, 5.67 3.2, 12.25 0" stroke="none" stroke-width="0" fill="#b2f2bb"/><path d="M12.25 0 C50.34 1.17, 90.66 -0.92, 123.42 0 M123.42 0 C130.03 -0.31, 133.78 3.78, 135.67 12.25 M135.67 12.25 C134.71 18.27, 135.12 22.95, 135.67 36.75 M135.67 36.75 C134.91 46.9, 129.6 47.86, 123.42 49 M123.42 49 C85.68 50.83, 51.22 50.1, 12.25 49 M12.25 49 C2.69 49.48, -1.08 43.23, 0 36.75 M0 36.75 C-1 30.36, -0.55 24.42, 0 12.25 M0 12.25 C-1.34 4.84, 4.8 -0.96, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(1358.5878446880472 296.26898845193307) rotate(0 59.60222625732422 7.397678214733503)"><text x="59.60222625732422" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">WindowsMemoryUsage</text></g><g stroke-linecap="round" transform="translate(1348.6892625026826 10) rotate(0 67.83414177602208 24.5)"><path d="M12.25 0 C52.58 -2.73, 88.64 -1.95, 123.42 0 C130.83 1.3, 137.55 0.94, 135.67 12.25 C137.62 15.39, 138.42 20.76, 135.67 36.75 C136.4 44.4, 131.82 46.16, 123.42 49 C92.93 52.93, 61.51 50.61, 12.25 49 C3.7 52.28, 1.89 47.32, 0 36.75 C3.63 30.99, 1.23 23.74, 0 12.25 C0.78 1.05, 1.56 -0.56, 12.25 0" stroke="none" stroke-width="0" fill="#b2f2bb"/><path d="M12.25 0 C56.26 2.93, 98.7 2.33, 123.42 0 M123.42 0 C129.92 1.45, 134.51 5.39, 135.67 12.25 M135.67 12.25 C135.23 18.06, 135.67 26.42, 135.67 36.75 M135.67 36.75 C135.54 43.7, 132.86 48.35, 123.42 49 M123.42 49 C97.39 48.03, 68.81 47.25, 12.25 49 M12.25 49 C4.85 50.96, 1.46 46.03, 0 36.75 M0 36.75 C0.91 30.66, 0.89 28.44, 0 12.25 M0 12.25 C-0.76 5, 6.02 1.3, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(1354.3367206239195 27.10232178526644) rotate(0 62.186683654785156 7.397678214733503)"><text x="62.186683654785156" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">WindowsNetworkUsage</text></g><g stroke-linecap="round"><g transform="translate(1175.74133087216 233.50763722152283) rotate(0 84.94874007321187 25.656125263757474)"><path d="M0 0 C28.32 8.55, 141.58 42.76, 169.9 51.31 M0 0 C28.32 8.55, 141.58 42.76, 169.9 51.31" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1175.74133087216 233.50763722152283) rotate(0 84.94874007321187 25.656125263757474)"><path d="M144.94 52.71 C152.66 52.27, 160.39 51.84, 169.9 51.31 M144.94 52.71 C153.88 52.21, 162.82 51.71, 169.9 51.31" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1175.74133087216 233.50763722152283) rotate(0 84.94874007321187 25.656125263757474)"><path d="M149.88 36.33 C156.08 40.97, 162.27 45.6, 169.9 51.31 M149.88 36.33 C157.05 41.7, 164.22 47.06, 169.9 51.31" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(1167.4127278042233 196.13354013480375) rotate(0 88.0332065536096 -78.16868985747266)"><path d="M0 0 C29.34 -26.06, 146.72 -130.28, 176.07 -156.34 M0 0 C29.34 -26.06, 146.72 -130.28, 176.07 -156.34" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1167.4127278042233 196.13354013480375) rotate(0 88.0332065536096 -78.16868985747266)"><path d="M164.18 -134.35 C167.45 -140.39, 170.71 -146.44, 176.07 -156.34 M164.18 -134.35 C167.21 -139.96, 170.25 -145.58, 176.07 -156.34" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1167.4127278042233 196.13354013480375) rotate(0 88.0332065536096 -78.16868985747266)"><path d="M152.82 -147.13 C159.21 -149.66, 165.6 -152.19, 176.07 -156.34 M152.82 -147.13 C158.76 -149.48, 164.7 -151.83, 176.07 -156.34" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(436.60592916935013 1422.4166666666665) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C48.45 1.28, 80.77 -0.21, 123.42 0 C128.48 0.93, 135.92 4.74, 135.67 12.25 C137.59 19.6, 136.64 24.74, 135.67 36.75 C132.25 44.76, 128.43 49.36, 123.42 49 C81.34 49.42, 37.87 51.33, 12.25 49 C3.42 50.09, 3.39 41.87, 0 36.75 C1.28 26.6, -0.61 26.59, 0 12.25 C0.56 7.26, 4.43 -0.17, 12.25 0" stroke="none" stroke-width="0" fill="#a5d8ff"/><path d="M12.25 0 C51.82 1.06, 89.46 -0.57, 123.42 0 M12.25 0 C44.49 -0.12, 77.15 -0.72, 123.42 0 M123.42 0 C132.11 0.69, 137.06 3.92, 135.67 12.25 M123.42 0 C133.35 -0.55, 135.65 1.89, 135.67 12.25 M135.67 12.25 C134.84 18.5, 136.24 26.75, 135.67 36.75 M135.67 12.25 C135.47 17.44, 136.18 23.43, 135.67 36.75 M135.67 36.75 C136.39 43, 131.95 50.55, 123.42 49 M135.67 36.75 C136.87 45.45, 130.67 49.53, 123.42 49 M123.42 49 C85.71 46.45, 47.75 46.26, 12.25 49 M123.42 49 C96.31 47.97, 69.47 48.42, 12.25 49 M12.25 49 C5.26 47.48, -1.9 44.52, 0 36.75 M12.25 49 C5.28 47.59, 0.64 44.03, 0 36.75 M0 36.75 C-1.77 28.99, 0.71 18.58, 0 12.25 M0 36.75 C0.67 29.22, -1.02 22.8, 0 12.25 M0 12.25 C0.35 4.63, 4.47 -1.67, 12.25 0 M0 12.25 C0.22 3.94, 3.12 -1.95, 12.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(441.9813459697863 1439.5189884519332) rotate(0 62.45872497558594 7.397678214733531)"><text x="62.45872497558594" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ListMetricDescriptors</text></g><g stroke-linecap="round" transform="translate(435.35592916935013 1521.1666666666665) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C39.3 1.39, 73.02 0.27, 123.42 0 C132.51 2.22, 135.09 2.32, 135.67 12.25 C136.1 20.66, 132.04 24.96, 135.67 36.75 C138.17 47.81, 129.04 50.8, 123.42 49 C90.31 51.24, 55.29 44.48, 12.25 49 C4.09 52.59, 1.79 48.39, 0 36.75 C1.33 31.28, 3.28 26.44, 0 12.25 C-1.61 1.7, 6.9 2.67, 12.25 0" stroke="none" stroke-width="0" fill="#a5d8ff"/><path d="M12.25 0 C36.02 0.87, 58.63 -0.53, 123.42 0 M12.25 0 C35.02 -0.03, 59.39 0.22, 123.42 0 M123.42 0 C130.15 -1.23, 135.32 4.61, 135.67 12.25 M123.42 0 C132.77 0.32, 136.32 2.66, 135.67 12.25 M135.67 12.25 C135.51 16.55, 134.84 21.34, 135.67 36.75 M135.67 12.25 C134.84 20.99, 134.87 29.96, 135.67 36.75 M135.67 36.75 C135.29 46.12, 133.2 48.62, 123.42 49 M135.67 36.75 C136.63 46.96, 129.61 46.74, 123.42 49 M123.42 49 C90.67 47.9, 56.03 49.94, 12.25 49 M123.42 49 C79.7 49.23, 36.85 49.39, 12.25 49 M12.25 49 C3.95 48.18, -0.69 45.93, 0 36.75 M12.25 49 C3.14 48.55, 2.28 46.81, 0 36.75 M0 36.75 C-1.79 28.28, 0.08 22.19, 0 12.25 M0 36.75 C-0.54 30.79, -0.62 26.06, 0 12.25 M0 12.25 C1.23 3.42, 5.22 1.34, 12.25 0 M0 12.25 C-1.79 4.07, 5.72 -1.19, 12.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(443.0319135967394 1530.8713102371996) rotate(0 60.15815734863281 14.795356429467063)"><text x="60.15815734863281" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ListPodSandboxMetri</text><text x="60.15815734863281" y="14.795356429467045" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">cs</text></g><g stroke-linecap="round" transform="translate(725.2170402804612 1421.1666666666665) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C52.96 -1.69, 91.23 1.5, 123.42 0 C130.99 1.87, 134 1.55, 135.67 12.25 C139.02 22.06, 136.43 25.95, 135.67 36.75 C135.63 45.49, 130.52 47.77, 123.42 49 C85.72 48.33, 45.16 53.14, 12.25 49 C1.22 51.26, -0.58 46.21, 0 36.75 C0.1 33.82, 0.03 26.93, 0 12.25 C-3.33 4.04, 0.54 -1.24, 12.25 0" stroke="none" stroke-width="0" fill="#a5d8ff"/><path d="M12.25 0 C50.21 -1.22, 91.42 1.01, 123.42 0 M123.42 0 C130.33 0.5, 135.82 3.14, 135.67 12.25 M135.67 12.25 C136.79 19.95, 137.67 25.48, 135.67 36.75 M135.67 36.75 C135.62 46.71, 132.37 47.35, 123.42 49 M123.42 49 C91.48 48.92, 63.23 47.96, 12.25 49 M12.25 49 C3.18 50.04, -1.07 43.68, 0 36.75 M0 36.75 C1.3 26.51, -0.48 18.23, 0 12.25 M0 12.25 C-0.97 5.13, 3.93 0.38, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(745.1943476448622 1438.2689884519332) rotate(0 47.856834411621094 7.397678214733531)"><text x="47.856834411621094" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">MetricDescriptor</text></g><g stroke-linecap="round" transform="translate(720.3559291693501 1519.9166666666665) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C38.37 1.72, 64.8 1.97, 123.42 0 C128.92 3.18, 132.92 2.65, 135.67 12.25 C138.65 24.06, 137.98 30.93, 135.67 36.75 C134.98 46.03, 132.77 51.53, 123.42 49 C86.25 50.02, 43.61 51.91, 12.25 49 C7.33 45.87, 3.1 46.74, 0 36.75 C0.49 31.23, 0.1 24.42, 0 12.25 C-1.71 2.25, 7.53 2.06, 12.25 0" stroke="none" stroke-width="0" fill="#a5d8ff"/><path d="M12.25 0 C47.91 1.39, 85.96 -1.86, 123.42 0 M123.42 0 C129.63 0.28, 135.64 2.12, 135.67 12.25 M135.67 12.25 C134.41 17.63, 135.46 22.35, 135.67 36.75 M135.67 36.75 C136.57 46.54, 131.48 49.88, 123.42 49 M123.42 49 C93.12 46.77, 62.09 49.31, 12.25 49 M12.25 49 C4.04 47.93, 0.77 45.09, 0 36.75 M0 36.75 C1.83 28.13, -0.56 21.81, 0 12.25 M0 12.25 C-1.82 3.9, 2.53 -0.38, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(733.2422331157824 1537.0189884519332) rotate(0 54.947837829589844 7.397678214733503)"><text x="54.947837829589844" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">PodSandboxMetrics</text></g><g stroke-linecap="round" transform="translate(1056.60592916935 1521.1666666666665) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C44.28 2.46, 83.69 -1.02, 123.42 0 C130.72 2.07, 136.75 3.19, 135.67 12.25 C137.29 17.19, 139.32 26.89, 135.67 36.75 C133.62 43.48, 129.78 51.34, 123.42 49 C84.19 48.23, 47.47 46.99, 12.25 49 C2.09 46.58, 3.19 44.1, 0 36.75 C-1.84 27.04, 1.66 24.72, 0 12.25 C1.38 0.84, 0.81 1.13, 12.25 0" stroke="none" stroke-width="0" fill="#a5d8ff"/><path d="M12.25 0 C55.05 -0.05, 99.88 1.78, 123.42 0 M123.42 0 C132.98 0.55, 137.57 5.33, 135.67 12.25 M135.67 12.25 C134.15 18.4, 136.32 22.28, 135.67 36.75 M135.67 36.75 C133.75 43.03, 129.72 50.2, 123.42 49 M123.42 49 C83.01 48.52, 42.2 50.79, 12.25 49 M12.25 49 C3.08 50.61, 1.39 44.54, 0 36.75 M0 36.75 C-0.87 29.65, 0.96 23.81, 0 12.25 M0 12.25 C-0.99 5.74, 5.1 0.3, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(1076.3466718975205 1538.2689884519332) rotate(0 48.09339904785156 7.397678214733503)"><text x="48.09339904785156" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ContainerMetrics</text></g><g stroke-linecap="round" transform="translate(1060.7725958360168 1629.0833333333335) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C54.22 3.64, 97.62 3.06, 123.42 0 C132.23 1.87, 139.03 6.32, 135.67 12.25 C138.19 22.99, 138.65 33.49, 135.67 36.75 C138.32 43.9, 129.13 45.69, 123.42 49 C81.31 46.58, 40.27 46.52, 12.25 49 C3.37 46.24, -0.2 44.45, 0 36.75 C-1.2 31.24, -0.07 20.32, 0 12.25 C1.27 1.36, 7.56 -1.4, 12.25 0" stroke="none" stroke-width="0" fill="#a5d8ff"/><path d="M12.25 0 C45.82 -0.82, 81.42 -0.94, 123.42 0 M123.42 0 C132.79 1.36, 136.24 2.24, 135.67 12.25 M135.67 12.25 C137.51 17.88, 134.11 24.02, 135.67 36.75 M135.67 36.75 C134.73 46.57, 133.33 49.16, 123.42 49 M123.42 49 C90.92 48.73, 57.84 48.85, 12.25 49 M12.25 49 C6.04 47.79, 0.81 44.61, 0 36.75 M0 36.75 C-1.56 28.58, 1.78 22.48, 0 12.25 M0 12.25 C0.55 4.07, 3.08 1.84, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(1110.692949770242 1646.1856551185997) rotate(0 17.913787841796875 7.397678214733503)"><text x="17.913787841796875" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">Metric</text></g><g stroke-linecap="round" transform="translate(1347.8559291693507 1632.8333333333335) rotate(0 67.83414177602208 24.5)"><path d="M12.25 0 C56.64 -0.51, 97.73 -0.75, 123.42 0 C128.22 1.67, 132.31 3.28, 135.67 12.25 C138.47 17.86, 137.89 25.63, 135.67 36.75 C139.11 45.35, 129.82 49.3, 123.42 49 C85.42 53.41, 48.76 51.67, 12.25 49 C4.23 48.59, -0.29 42.04, 0 36.75 C-0.78 31.47, 0.45 21.94, 0 12.25 C0.65 3.58, 1.53 -2.7, 12.25 0" stroke="none" stroke-width="0" fill="#a5d8ff"/><path d="M12.25 0 C54.31 -1.14, 98.75 0.54, 123.42 0 M123.42 0 C129.69 -0.56, 136.96 4.8, 135.67 12.25 M135.67 12.25 C136.25 20.09, 134.36 28.29, 135.67 36.75 M135.67 36.75 C134.13 43.41, 131.24 49.72, 123.42 49 M123.42 49 C97.52 51.15, 73.77 48.07, 12.25 49 M12.25 49 C5.09 50.97, 0.67 45.52, 0 36.75 M0 36.75 C-0.63 27.04, 0.09 19.28, 0 12.25 M0 12.25 C0.21 5.07, 3.25 -0.95, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(1384.096984702697 1649.9356551185997) rotate(0 31.59308624267578 7.397678214733503)"><text x="31.59308624267578" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">MetricType</text></g><g stroke-linecap="round"><g transform="translate(576.4644367582226 1450.0736772219464) rotate(0 73.87630176111928 -0.03190371532741665)"><path d="M0 0 C24.63 -0.01, 123.13 -0.05, 147.75 -0.06 M0 0 C24.63 -0.01, 123.13 -0.05, 147.75 -0.06" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(576.4644367582226 1450.0736772219464) rotate(0 73.87630176111928 -0.03190371532741665)"><path d="M124.26 8.5 C129.2 6.7, 134.14 4.9, 147.75 -0.06 M124.26 8.5 C132.2 5.6, 140.14 2.71, 147.75 -0.06" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(576.4644367582226 1450.0736772219464) rotate(0 73.87630176111928 -0.03190371532741665)"><path d="M124.26 -8.6 C129.2 -6.81, 134.14 -5.01, 147.75 -0.06 M124.26 -8.6 C132.2 -5.72, 140.14 -2.83, 147.75 -0.06" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(572.7144367582227 1543.823572857907) rotate(0 74.22563418714951 -0.6569061912402958)"><path d="M0 0 C24.74 -0.22, 123.71 -1.09, 148.45 -1.31 M0 0 C24.74 -0.22, 123.71 -1.09, 148.45 -1.31" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(572.7144367582227 1543.823572857907) rotate(0 74.22563418714951 -0.6569061912402958)"><path d="M125.04 7.44 C133.38 4.32, 141.73 1.2, 148.45 -1.31 M125.04 7.44 C132.82 4.53, 140.61 1.62, 148.45 -1.31" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(572.7144367582227 1543.823572857907) rotate(0 74.22563418714951 -0.6569061912402958)"><path d="M124.88 -9.66 C133.28 -6.68, 141.69 -3.71, 148.45 -1.31 M124.88 -9.66 C132.72 -6.88, 140.56 -4.11, 148.45 -1.31" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(857.0242127213943 1541.3233917199345) rotate(0 98.32074620556371 -0.031815622254214304)"><path d="M0 0 C32.77 -0.01, 163.87 -0.05, 196.64 -0.06 M0 0 C32.77 -0.01, 163.87 -0.05, 196.64 -0.06" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(857.0242127213943 1541.3233917199345) rotate(0 98.32074620556371 -0.031815622254214304)"><path d="M173.15 8.49 C180.75 5.73, 188.34 2.96, 196.64 -0.06 M173.15 8.49 C179.21 6.29, 185.27 4.08, 196.64 -0.06" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(857.0242127213943 1541.3233917199345) rotate(0 98.32074620556371 -0.031815622254214304)"><path d="M173.15 -8.61 C180.74 -5.84, 188.34 -3.08, 196.64 -0.06 M173.15 -8.61 C179.21 -6.4, 185.27 -4.2, 196.64 -0.06" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(858.3394367582227 1566.9204600077703) rotate(0 98.80896752048272 41.30451624073362)"><path d="M0 0 C32.94 13.77, 164.68 68.84, 197.62 82.61 M0 0 C32.94 13.77, 164.68 68.84, 197.62 82.61" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(858.3394367582227 1566.9204600077703) rotate(0 98.80896752048272 41.30451624073362)"><path d="M172.65 81.44 C178.08 81.69, 183.51 81.95, 197.62 82.61 M172.65 81.44 C180.79 81.82, 188.93 82.2, 197.62 82.61" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(858.3394367582227 1566.9204600077703) rotate(0 98.80896752048272 41.30451624073362)"><path d="M179.24 65.66 C183.24 69.35, 187.24 73.03, 197.62 82.61 M179.24 65.66 C185.23 71.19, 191.22 76.71, 197.62 82.61" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(1125.0061034248895 1571.948572857907) rotate(0 -0.35214880024327044 28.067380237713223)"><path d="M0 0 C-0.12 9.36, -0.59 46.78, -0.7 56.13 M0 0 C-0.12 9.36, -0.59 46.78, -0.7 56.13" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1125.0061034248895 1571.948572857907) rotate(0 -0.35214880024327044 28.067380237713223)"><path d="M-8.96 32.54 C-7.29 37.32, -5.61 42.1, -0.7 56.13 M-8.96 32.54 C-5.95 41.14, -2.94 49.75, -0.7 56.13" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1125.0061034248895 1571.948572857907) rotate(0 -0.35214880024327044 28.067380237713223)"><path d="M8.14 32.75 C6.35 37.49, 4.55 42.23, -0.7 56.13 M8.14 32.75 C4.92 41.28, 1.69 49.8, -0.7 56.13" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(1197.5061034248893 1651.911206424821) rotate(0 72.76730085381632 0.6117770253025583)"><path d="M0 0 C24.26 0.2, 121.28 1.02, 145.53 1.22 M0 0 C24.26 0.2, 121.28 1.02, 145.53 1.22" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1197.5061034248893 1651.911206424821) rotate(0 72.76730085381632 0.6117770253025583)"><path d="M121.97 9.58 C128.08 7.41, 134.2 5.24, 145.53 1.22 M121.97 9.58 C129.47 6.92, 136.96 4.26, 145.53 1.22" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1197.5061034248893 1651.911206424821) rotate(0 72.76730085381632 0.6117770253025583)"><path d="M122.12 -7.52 C128.19 -5.26, 134.26 -2.99, 145.53 1.22 M122.12 -7.52 C129.56 -4.74, 137.01 -1.96, 145.53 1.22" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(435.60592916935013 1209.9166666666665) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C56.89 -0.88, 99.73 -0.58, 123.42 0 M12.25 0 C45.67 -0.49, 79.17 1.13, 123.42 0 M123.42 0 C129.74 0.73, 135.9 5.73, 135.67 12.25 M123.42 0 C129.62 -1.07, 134.86 3.94, 135.67 12.25 M135.67 12.25 C137.14 23.14, 134.85 30.58, 135.67 36.75 M135.67 12.25 C136.22 19.81, 135.66 28.3, 135.67 36.75 M135.67 36.75 C135.8 43.49, 130.73 47.5, 123.42 49 M135.67 36.75 C135.11 43.33, 132.78 49.88, 123.42 49 M123.42 49 C92.64 50.2, 59.92 51.05, 12.25 49 M123.42 49 C78.64 49.09, 35.49 49.42, 12.25 49 M12.25 49 C4.66 47.02, 0.78 44.34, 0 36.75 M12.25 49 C2.43 49.54, -1.26 43.28, 0 36.75 M0 36.75 C0.68 26.17, 1.38 18.04, 0 12.25 M0 36.75 C0.59 31.26, -0.89 25.8, 0 12.25 M0 12.25 C-0.6 4.95, 3.83 -0.94, 12.25 0 M0 12.25 C-2.06 5.95, 3.2 0.32, 12.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(467.58883650933706 1227.0189884519332) rotate(0 35.851234436035156 7.397678214733517)"><text x="35.851234436035156" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ImageFsInfo</text></g><g stroke-linecap="round" transform="translate(722.6059291693501 1212.9166666666665) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C53.5 -1.13, 88.04 -5.17, 123.42 0 C131.85 -1.15, 137.12 5.13, 135.67 12.25 C133.26 18.01, 133.86 19.54, 135.67 36.75 C139.11 47.4, 134.62 50.54, 123.42 49 C92.75 48.34, 64.21 51.47, 12.25 49 C5.24 45.48, 2.4 43.84, 0 36.75 C-3.17 31.26, -1.33 20.43, 0 12.25 C-1.19 2.33, 4.04 -3.08, 12.25 0" stroke="none" stroke-width="0" fill="#ffc9c9"/><path d="M12.25 0 C44.95 0.45, 76.71 -0.39, 123.42 0 M123.42 0 C129.64 -0.41, 136.98 2.24, 135.67 12.25 M135.67 12.25 C133.99 19.57, 134.99 28.62, 135.67 36.75 M135.67 36.75 C137.16 43.91, 132.79 50.6, 123.42 49 M123.42 49 C80.89 48.52, 39.43 50.21, 12.25 49 M12.25 49 C3.65 49.09, 0.98 45.05, 0 36.75 M0 36.75 C-0.38 29.77, 1.14 26.31, 0 12.25 M0 12.25 C-0.26 4.94, 4.78 1.14, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(728.7324293438097 1230.0189884519332) rotate(0 61.7076416015625 7.397678214733517)"><text x="61.7076416015625" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ImageFsInfoResponse</text></g><g stroke-linecap="round"><g transform="translate(572.2742127213944 1231.9956845646443) rotate(0 73.34503424636208 0.6933682538536488)"><path d="M0 0 C24.45 0.23, 122.24 1.16, 146.69 1.39 M0 0 C24.45 0.23, 122.24 1.16, 146.69 1.39" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(572.2742127213944 1231.9956845646443) rotate(0 73.34503424636208 0.6933682538536488)"><path d="M123.12 9.71 C130.84 6.99, 138.56 4.26, 146.69 1.39 M123.12 9.71 C131.05 6.91, 138.99 4.11, 146.69 1.39" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(572.2742127213944 1231.9956845646443) rotate(0 73.34503424636208 0.6933682538536488)"><path d="M123.28 -7.39 C130.95 -4.51, 138.61 -1.64, 146.69 1.39 M123.28 -7.39 C131.16 -4.43, 139.04 -1.48, 146.69 1.39" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(1351.4279645846748 369.85674374603775) rotate(0 67.83414177602208 24.5)"><path d="M12.25 0 C53.57 1.89, 96.82 0.1, 123.42 0 C131.79 -3.41, 132.72 7.6, 135.67 12.25 C135.3 17.33, 134.87 23.94, 135.67 36.75 C133.91 48.47, 133.21 52.3, 123.42 49 C82.98 49.16, 45.28 50.87, 12.25 49 C1.26 50.35, 1.26 47.16, 0 36.75 C2.81 30.26, 3.04 24.25, 0 12.25 C2.18 6.87, 2.27 -2, 12.25 0" stroke="none" stroke-width="0" fill="#b2f2bb"/><path d="M12.25 0 C48.8 0.32, 85.19 -0.81, 123.42 0 M123.42 0 C132.25 1.6, 137.53 2.81, 135.67 12.25 M135.67 12.25 C135.41 20.97, 137.76 29.9, 135.67 36.75 M135.67 36.75 C136.48 44.66, 129.99 49.98, 123.42 49 M123.42 49 C88.63 48.59, 51.71 50.21, 12.25 49 M12.25 49 C2.97 49.56, 0 44.47, 0 36.75 M0 36.75 C-1.7 29.3, -1.83 21.27, 0 12.25 M0 12.25 C-0.15 4.97, 4.75 0.39, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(1359.4883713143101 379.56138731657074) rotate(0 59.77373504638672 14.795356429467063)"><text x="59.77373504638672" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">WindowsContainerSta</text><text x="59.77373504638672" y="14.795356429467045" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ts</text></g><g stroke-linecap="round"><g transform="translate(1169.8458929275653 237.78533834648454) rotate(0 89.56283278521221 74.92714362439386)"><path d="M0 0 C29.85 24.98, 149.27 124.88, 179.13 149.85 M0 0 C29.85 24.98, 149.27 124.88, 179.13 149.85" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1169.8458929275653 237.78533834648454) rotate(0 89.56283278521221 74.92714362439386)"><path d="M155.62 141.34 C160.64 143.16, 165.66 144.97, 179.13 149.85 M155.62 141.34 C162.61 143.87, 169.61 146.41, 179.13 149.85" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1169.8458929275653 237.78533834648454) rotate(0 89.56283278521221 74.92714362439386)"><path d="M166.59 128.22 C169.27 132.84, 171.94 137.46, 179.13 149.85 M166.59 128.22 C170.32 134.66, 174.05 141.09, 179.13 149.85" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(1349.7612979180085 94.85674374603798) rotate(0 67.83414177602208 24.5)"><path d="M12.25 0 C43.61 -1.19, 71.62 -0.62, 123.42 0 C134.99 1.09, 133.83 6.46, 135.67 12.25 C133.89 16.24, 137.38 23.06, 135.67 36.75 C138.6 42.94, 134.59 48.7, 123.42 49 C94.03 48.54, 71.57 46.13, 12.25 49 C1.69 50.06, -3.06 47.37, 0 36.75 C-1.74 28.22, -1.14 24.13, 0 12.25 C-0.76 6.4, 1.71 -0.67, 12.25 0" stroke="none" stroke-width="0" fill="#b2f2bb"/><path d="M12.25 0 C56.94 -1.4, 101.72 -0.92, 123.42 0 M123.42 0 C132.75 1.71, 135.26 5.69, 135.67 12.25 M135.67 12.25 C136.02 17.02, 133.67 25.43, 135.67 36.75 M135.67 36.75 C135.6 45, 129.74 50.11, 123.42 49 M123.42 49 C85.27 49.79, 45.74 50.79, 12.25 49 M12.25 49 C3.16 50.8, 1.88 45.51, 0 36.75 M0 36.75 C-0.91 28.84, -1.73 22.47, 0 12.25 M0 12.25 C-0.13 5.13, 2.8 1.45, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(1355.7399556852415 111.95906553130442) rotate(0 61.85548400878906 7.397678214733503)"><text x="61.85548400878906" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">WindowsProcessUsage</text></g><g stroke-linecap="round" transform="translate(1626.4279645846748 14.85674374603775) rotate(0 67.83414177602208 24.5)"><path d="M12.25 0 C38.64 1.88, 55.07 2.61, 123.42 0 C130.15 -1.07, 133.95 6.98, 135.67 12.25 C134.56 20.78, 136.13 23.97, 135.67 36.75 C134.88 43.95, 134.52 48.21, 123.42 49 C96.6 52.12, 72.65 53.58, 12.25 49 C2.79 52.05, 1.33 48.44, 0 36.75 C1.03 29.58, -3.35 23.17, 0 12.25 C-2.34 7.32, 7.19 -2.66, 12.25 0" stroke="none" stroke-width="0" fill="#b2f2bb"/><path d="M12.25 0 C46.81 -1.06, 84.38 2.03, 123.42 0 M123.42 0 C131.01 -0.19, 136.82 3.35, 135.67 12.25 M135.67 12.25 C135.89 17.52, 134.52 21.67, 135.67 36.75 M135.67 36.75 C136.7 44.85, 130.42 48.48, 123.42 49 M123.42 49 C90.52 48.7, 56.34 46.26, 12.25 49 M12.25 49 C5.9 48.9, -1.01 46.85, 0 36.75 M0 36.75 C-1.21 27.17, -2 19.52, 0 12.25 M0 12.25 C-1.51 5.81, 5.94 0.23, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(1631.6732620613805 24.561387316570745) rotate(0 62.588844299316406 14.795356429467063)"><text x="62.588844299316406" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">WindowsNetworkInterf</text><text x="62.588844299316406" y="14.795356429467045" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">aceUsage</text></g><g stroke-linecap="round"><g transform="translate(1489.5260432416285 39.18331563937147) rotate(0 64.73606311906804 0.037485750479049784)"><path d="M0 0 C21.58 0.01, 107.89 0.06, 129.47 0.07 M0 0 C21.58 0.01, 107.89 0.06, 129.47 0.07" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1489.5260432416285 39.18331563937147) rotate(0 64.73606311906804 0.037485750479049784)"><path d="M105.97 8.61 C113.66 5.82, 121.35 3.03, 129.47 0.07 M105.97 8.61 C110.74 6.88, 115.51 5.15, 129.47 0.07" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1489.5260432416285 39.18331563937147) rotate(0 64.73606311906804 0.037485750479049784)"><path d="M105.98 -8.49 C113.67 -5.69, 121.35 -2.89, 129.47 0.07 M105.98 -8.49 C110.75 -6.75, 115.51 -5.02, 129.47 0.07" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(1173.2984425443387 199.57766598372075) rotate(0 86.22187670114835 -36.751902043458585)"><path d="M0 0 C28.74 -12.25, 143.7 -61.25, 172.44 -73.5 M0 0 C28.74 -12.25, 143.7 -61.25, 172.44 -73.5" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1173.2984425443387 199.57766598372075) rotate(0 86.22187670114835 -36.751902043458585)"><path d="M154.19 -56.43 C160.68 -62.5, 167.17 -68.58, 172.44 -73.5 M154.19 -56.43 C158.39 -60.36, 162.59 -64.29, 172.44 -73.5" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1173.2984425443387 199.57766598372075) rotate(0 86.22187670114835 -36.751902043458585)"><path d="M147.48 -72.16 C156.36 -72.64, 165.24 -73.12, 172.44 -73.5 M147.48 -72.16 C153.23 -72.47, 158.97 -72.78, 172.44 -73.5" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(1629.761297918008 374.856743746038) rotate(0 67.83414177602208 24.5)"><path d="M12.25 0 C39.69 2.41, 62.56 -2.14, 123.42 0 C128.28 -1.4, 139.16 5.4, 135.67 12.25 C133.87 17.71, 132.8 25.01, 135.67 36.75 C132.52 46.87, 130.79 52.02, 123.42 49 C87.73 44.8, 53.99 50.33, 12.25 49 C7.04 52.57, 2.97 45.85, 0 36.75 C-0.23 29.16, -1.99 23.1, 0 12.25 C3.6 4.31, 3.27 -2.58, 12.25 0" stroke="none" stroke-width="0" fill="#b2f2bb"/><path d="M12.25 0 C48.52 -1.46, 85.62 -0.8, 123.42 0 M123.42 0 C133.58 -0.52, 135.41 5.11, 135.67 12.25 M135.67 12.25 C135.84 19.61, 135.18 27.45, 135.67 36.75 M135.67 36.75 C137.55 46.56, 130.52 50.72, 123.42 49 M123.42 49 C81.18 49.39, 35.13 48.01, 12.25 49 M12.25 49 C3.34 47.69, 0.52 43.7, 0 36.75 M0 36.75 C-1.58 28.15, 1.95 20.38, 0 12.25 M0 12.25 C-0.13 2.93, 4.73 -1.31, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(1635.7931707120965 384.561387316571) rotate(0 61.802268981933594 14.795356429467063)"><text x="61.802268981933594" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">WindowsFilesystemSta</text><text x="61.802268981933594" y="14.795356429467045" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ts</text></g><g stroke-linecap="round"><g transform="translate(1490.018178207848 397.09481711058345) rotate(0 69.24392815284932 0.6017535978714932)"><path d="M0 0 C23.08 0.2, 115.41 1, 138.49 1.2 M0 0 C23.08 0.2, 115.41 1, 138.49 1.2" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1490.018178207848 397.09481711058345) rotate(0 69.24392815284932 0.6017535978714932)"><path d="M114.92 9.55 C121.13 7.35, 127.34 5.15, 138.49 1.2 M114.92 9.55 C124.14 6.29, 133.35 3.02, 138.49 1.2" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1490.018178207848 397.09481711058345) rotate(0 69.24392815284932 0.6017535978714932)"><path d="M115.07 -7.55 C121.24 -5.24, 127.42 -2.94, 138.49 1.2 M115.07 -7.55 C124.23 -4.13, 133.39 -0.7, 138.49 1.2" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(1383.0449367984043 368.85674374603775) rotate(0 0.39000117375269383 -18.119761521136354)"><path d="M0 0 C0.13 -6.04, 0.65 -30.2, 0.78 -36.24 M0 0 C0.13 -6.04, 0.65 -30.2, 0.78 -36.24" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1383.0449367984043 368.85674374603775) rotate(0 0.39000117375269383 -18.119761521136354)"><path d="M6.61 -19.08 C4.79 -24.43, 2.98 -29.77, 0.78 -36.24 M6.61 -19.08 C4.6 -25, 2.59 -30.91, 0.78 -36.24" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1383.0449367984043 368.85674374603775) rotate(0 0.39000117375269383 -18.119761521136354)"><path d="M-5.78 -19.35 C-3.74 -24.61, -1.69 -29.87, 0.78 -36.24 M-5.78 -19.35 C-3.52 -25.17, -1.26 -31, 0.78 -36.24" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round"><g transform="translate(1447.1992735754848 367.76293345497754) rotate(0 -0.43716721478767795 -58.40618970893951)"><path d="M0 0 C-0.15 -19.47, -0.73 -97.34, -0.87 -116.81 M0 0 C-0.15 -19.47, -0.73 -97.34, -0.87 -116.81" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1447.1992735754848 367.76293345497754) rotate(0 -0.43716721478767795 -58.40618970893951)"><path d="M7.85 -93.38 C4.88 -101.36, 1.91 -109.33, -0.87 -116.81 M7.85 -93.38 C5.39 -99.99, 2.93 -106.6, -0.87 -116.81" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(1447.1992735754848 367.76293345497754) rotate(0 -0.43716721478767795 -58.40618970893951)"><path d="M-9.25 -93.26 C-6.4 -101.27, -3.55 -109.29, -0.87 -116.81 M-9.25 -93.26 C-6.89 -99.9, -4.53 -106.54, -0.87 -116.81" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/><g stroke-linecap="round" transform="translate(435.198830777186 547.902462121212) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C35.82 0.12, 58.15 0.34, 123.42 0 M12.25 0 C35.2 0.86, 56.43 -0.45, 123.42 0 M123.42 0 C132.26 -0.89, 137.13 5.77, 135.67 12.25 M123.42 0 C129.73 1.99, 136.85 2.81, 135.67 12.25 M135.67 12.25 C134.36 19.82, 136.22 25.97, 135.67 36.75 M135.67 12.25 C135.99 17.52, 134.73 24.5, 135.67 36.75 M135.67 36.75 C134.76 46.88, 132.26 50.9, 123.42 49 M135.67 36.75 C133.67 43.39, 133.03 49.56, 123.42 49 M123.42 49 C86.55 51.44, 45.61 49.45, 12.25 49 M123.42 49 C85.21 49.54, 46.53 49.91, 12.25 49 M12.25 49 C4.33 50.77, -1.63 44.49, 0 36.75 M12.25 49 C3.28 47.59, -2.25 43.02, 0 36.75 M0 36.75 C1.45 27.23, -1.55 19.37, 0 12.25 M0 36.75 C-0.51 30.61, -0.48 23.56, 0 12.25 M0 12.25 C0.01 4.35, 3.13 1.18, 12.25 0 M0 12.25 C2.19 6.35, 2.82 -1.55, 12.25 0" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(448.41631911082527 565.0047839064785) rotate(0 54.61665344238281 7.397678214733531)"><text x="54.61665344238281" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">PodSandboxStatus</text></g><g stroke-linecap="round" transform="translate(722.198830777186 550.902462121212) rotate(0 67.83414177602211 24.5)"><path d="M12.25 0 C55.94 -0.75, 98.68 2.38, 123.42 0 C130.69 2.21, 136.11 4.54, 135.67 12.25 C134.05 19.56, 131.78 28.31, 135.67 36.75 C136.7 43.82, 130.95 51.48, 123.42 49 C97.08 47.29, 73.73 45.77, 12.25 49 C5.59 47.17, -0.3 44.93, 0 36.75 C3.63 32.61, -1.76 18.96, 0 12.25 C1.13 4.3, 7.07 0.68, 12.25 0" stroke="none" stroke-width="0" fill="#ffc9c9"/><path d="M12.25 0 C39.59 0.7, 70.57 1.6, 123.42 0 M123.42 0 C132.91 -0.77, 134.38 4.76, 135.67 12.25 M135.67 12.25 C134.24 20.23, 136.87 23.65, 135.67 36.75 M135.67 36.75 C135.11 46.36, 131.54 47.52, 123.42 49 M123.42 49 C83.64 49.25, 45.13 49.87, 12.25 49 M12.25 49 C3.26 47.12, 1.79 44, 0 36.75 M0 36.75 C-1.82 25.67, 1.18 17.73, 0 12.25 M0 12.25 C0.82 2.97, 5.69 1.05, 12.25 0" stroke="#1e1e1e" stroke-width="1" fill="none" stroke-dasharray="8 8.5"/></g><g transform="translate(728.1715528754737 560.6071056917449) rotate(0 61.861419677734375 14.795356429467034)"><text x="61.861419677734375" y="0" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">PodSandboxStatusRe</text><text x="61.861419677734375" y="14.795356429467045" font-family="Virgil, Segoe UI Emoji" font-size="11.836285143573637px" fill="#1e1e1e" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">sponse</text></g><g stroke-linecap="round"><g transform="translate(571.8671143292302 569.9814800191898) rotate(0 73.34503424636208 0.6933682538536345)"><path d="M0 0 C24.45 0.23, 122.24 1.16, 146.69 1.39 M0 0 C24.45 0.23, 122.24 1.16, 146.69 1.39" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(571.8671143292302 569.9814800191898) rotate(0 73.34503424636208 0.6933682538536345)"><path d="M123.12 9.71 C130.69 7.04, 138.27 4.36, 146.69 1.39 M123.12 9.71 C129.14 7.59, 135.15 5.46, 146.69 1.39" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g><g transform="translate(571.8671143292302 569.9814800191898) rotate(0 73.34503424636208 0.6933682538536345)"><path d="M123.28 -7.39 C130.8 -4.57, 138.33 -1.75, 146.69 1.39 M123.28 -7.39 C129.26 -5.15, 135.23 -2.91, 146.69 1.39" stroke="#1e1e1e" stroke-width="0.5" fill="none"/></g></g><mask/></svg> +\ No newline at end of file +-- +2.25.1 + diff --git a/0069-add-exec-workdir-support-for-oci-runtime.patch b/0069-add-exec-workdir-support-for-oci-runtime.patch new file mode 100644 index 0000000..e57cee6 --- /dev/null +++ b/0069-add-exec-workdir-support-for-oci-runtime.patch @@ -0,0 +1,57 @@ +From 921ee84f80adda64fb0a7125f9f709bff416945c Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Tue, 26 Dec 2023 10:17:19 +0800 +Subject: [PATCH 69/71] add exec workdir support for oci runtime + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/cmd/isula/stream/exec.h | 2 +- + src/cmd/isulad-shim/process.c | 4 ++++ + src/daemon/modules/runtime/isula/isula_rt_ops.c | 3 +++ + 3 files changed, 8 insertions(+), 1 deletion(-) + +diff --git a/src/cmd/isula/stream/exec.h b/src/cmd/isula/stream/exec.h +index 83a4af06..8d8e4347 100644 +--- a/src/cmd/isula/stream/exec.h ++++ b/src/cmd/isula/stream/exec.h +@@ -48,7 +48,7 @@ extern "C" { + "Username or UID (format: <name|uid>[:<group|gid>])", \ + NULL }, \ + { CMD_OPT_TYPE_STRING_DUP, false, "workdir", 0, &(cmdargs).custom_conf.workdir, \ +- "Working directory inside the container, supported only when runtime is lcr", NULL } ++ "Working directory inside the container", NULL } + + extern const char g_cmd_exec_desc[]; + extern const char g_cmd_exec_usage[]; +diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c +index 97524f1a..06726a40 100644 +--- a/src/cmd/isulad-shim/process.c ++++ b/src/cmd/isulad-shim/process.c +@@ -1274,6 +1274,10 @@ static void get_runtime_cmd(process_t *p, const char *log_path, const char *pid_ + #endif + params[i++] = "--process"; + params[i++] = process_desc; ++ if (p->state->cwd != NULL) { ++ params[i++] = "--cwd"; ++ params[i++] = p->state->cwd; ++ } + } else { + params[i++] = "create"; + params[i++] = "--bundle"; +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index fbb779f7..c754fc54 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -1345,6 +1345,9 @@ static int preparation_exec(const char *id, const char *runtime, const char *wor + p.runtime_args = (char **)runtime_args; + p.runtime_args_len = runtime_args_len; + copy_process(&p, process); ++ if (params->workdir != NULL) { ++ p.cwd = (char *)params->workdir; ++ } + + ret = create_process_json_file(workdir, &p); + if (ret != 0) { +-- +2.25.1 + diff --git a/0070-add-testcases-for-exec-workdir.patch b/0070-add-testcases-for-exec-workdir.patch new file mode 100644 index 0000000..38687aa --- /dev/null +++ b/0070-add-testcases-for-exec-workdir.patch @@ -0,0 +1,27 @@ +From cf7effbfaf81d1982d81bbc6ca1c5eafbc07d07b Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Tue, 26 Dec 2023 10:24:52 +0800 +Subject: [PATCH 70/71] add testcases for exec --workdir + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + CI/test_cases/container_cases/runc_exec.sh | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/CI/test_cases/container_cases/runc_exec.sh b/CI/test_cases/container_cases/runc_exec.sh +index f963724e..aa7020ee 100755 +--- a/CI/test_cases/container_cases/runc_exec.sh ++++ b/CI/test_cases/container_cases/runc_exec.sh +@@ -53,6 +53,9 @@ function exec_runc_test() + isula exec -tid $container_name /bin/sh -c 'exit 2' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - exit code should be 0" && ((ret++)) + ++ isula exec -ti --workdir /tmp $container_name pwd | grep "/tmp" ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - workdir is not /workdir failed" && ((ret++)) ++ + ls /var/run/isulad/runc/${ID}/exec/ + ls /var/run/isulad/runc/${ID}/exec/ | wc -l | grep 0 + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - residual dir after success exec" && ((ret++)) +-- +2.25.1 + diff --git a/0071-iSulad-restart-isuald-when-upgrade-active-isulad.patch b/0071-iSulad-restart-isuald-when-upgrade-active-isulad.patch new file mode 100644 index 0000000..033dd14 --- /dev/null +++ b/0071-iSulad-restart-isuald-when-upgrade-active-isulad.patch @@ -0,0 +1,34 @@ +From cffeca42fa88c749f122b904ecc2e634d4911576 Mon Sep 17 00:00:00 2001 +From: yangjiaqi <yangjiaqi16@huawei.com> +Date: Thu, 28 Dec 2023 15:56:47 +0800 +Subject: [PATCH 71/71] iSulad: restart isuald when upgrade active isulad + +Signed-off-by: yangjiaqi <yangjiaqi16@huawei.com> +--- + iSulad.spec | 10 ++++++++++ + 1 file changed, 10 insertions(+) + +diff --git a/iSulad.spec b/iSulad.spec +index 0efbf043..71b61c8b 100644 +--- a/iSulad.spec ++++ b/iSulad.spec +@@ -196,6 +196,16 @@ if [ -e %{_unitdir}/lcrd.service.rpmsave ]; then + mv %{_unitdir}/lcrd.service.rpmsave %{_unitdir}/isulad.service + sed -i 's/lcrd/isulad/g' %{_unitdir}/isulad.service + fi ++# During the isulad upgrade process, the isulad service may still be running, but the service may be unavailable ++# due to configuration updates and other reasons. ++# it may fail if the X package is upgraded synchronously with isulad and depends on the isulad command, ++# For example syscontianer-tools and lxcfs-tools. ++# Therefore, after upgrading isulad, if the original status of isulad is running, ++# we need to restart isulad to ensure that the service is available during the upgrade process. ++systemctl status isulad | grep 'Active:' | grep 'running' ++if [ $? -eq 0 ]; then ++ systemctl restart isulad ++fi + %else + /sbin/service isulad status | grep 'Active:' | grep 'running' + if [ $? -eq 0 ]; then +-- +2.25.1 + diff --git a/iSulad.spec b/iSulad.spec new file mode 100644 index 0000000..61cb1cb --- /dev/null +++ b/iSulad.spec @@ -0,0 +1,1000 @@ +%global _version 2.1.4 +%global _release 5 +%global is_systemd 1 +%global enable_criv1 1 +%global enable_shimv2 1 +%global is_embedded 1 +%global cpp_std 17 + +Name: iSulad +Version: %{_version} +Release: %{_release} +Summary: Lightweight Container Runtime Daemon +License: Mulan PSL v2 +URL: https://gitee.com/openeuler/iSulad +Source: https://gitee.com/openeuler/iSulad/repository/archive/v%{version}.tar.gz +BuildRoot: {_tmppath}/iSulad-%{version} + +Patch0001: 0001-sandbox-del-m_containers-and-m_containersMutex.patch +Patch0002: 0002-sandbox-adapt-UT-when-del-m_containers-and-m_contain.patch +Patch0003: 0003-Add-Readonly-Masked-Path-and-RunAsGroup-support-for-.patch +Patch0004: 0004-network-support-version-opt.patch +Patch0005: 0005-doc-support-version-opt.patch +Patch0006: 0006-2242-disable-grpc-remote-connect-by-default.patch +Patch0007: 0007-2244-Save-task-address-of-shim-v2.patch +Patch0008: 0008-2233-add-runc-append-function-design-doc.patch +Patch0009: 0009-2243-Refactor-capbilities-specs.patch +Patch0010: 0010-2245-fix-utils_verify_ut-failure-when-remote-grpc-fu.patch +Patch0011: 0011-add-runc-attach-implement.patch +Patch0012: 0012-add-runc-attach-implement-unit-test-and-ci-test.patch +Patch0013: 0013-support-gcov-of-CI.patch +Patch0014: 0014-compatibility-for-manage-pods-which-created-by-old-i.patch +Patch0015: 0015-2250-Remove-PERFMON-BPF-CHECKPOINT_RESTORE.patch +Patch0016: 0016-improve-event-logs.patch +Patch0017: 0017-2251-open-ENABLE_GRPC_REMOTE_CONNECT-in-CI.patch +Patch0018: 0018-Add-compatibility-between-iSulad-and-k8s.patch +Patch0019: 0019-2254-lcr-container-with-a-damaged-config-file-will-r.patch +Patch0020: 0020-2253-bugfix-for-runc-container-exec.patch +Patch0021: 0021-bugfix-of-update-restart-policy-for-auto-remove-cont.patch +Patch0022: 0022-add-update-restart-policy-test.patch +Patch0023: 0023-2260-bugfix-for-rebuild-config.patch +Patch0024: 0024-2170-isula-image-pull.patch +Patch0025: 0025-2084-image-pull.patch +Patch0026: 0026-CI-add-ncurse-for-ubuntu-and-centos.patch +Patch0027: 0027-improve-code-of-pull-progress.patch +Patch0028: 0028-2230-format-code.patch +Patch0029: 0029-2255-Fix-cpusets-offline-issue.patch +Patch0030: 0030-modify-daemon-json-default-runtime-to-runc.patch +Patch0031: 0031-modify-CI-for-default-runtime-to-runc.patch +Patch0032: 0032-add-ut-for-devicemapper.patch +Patch0033: 0033-2275-bugfix-for-rt_lcr_rebuild_config.patch +Patch0034: 0034-2277-remove-shim-v2-format-error-log.patch +Patch0035: 0035-2276-bugfix-for-integration_check.sh.patch +Patch0036: 0036-modify-create_network.sh-for-default-runtime-changed.patch +Patch0037: 0037-modify-the-container-runtime-when-running-embedded.s.patch +Patch0038: 0038-save-sandbox-to-disk-after-network-ready.patch +Patch0039: 0039-fix-the-problem-of-abnormal-branches-not-waiting-for.patch +Patch0040: 0040-remove-embedded-image-support-in-readme.patch +Patch0041: 0041-Acquire-system-info-in-on-demand.patch +Patch0042: 0042-2268-bugfix-for-the-bliko-zero-value-exception-when-.patch +Patch0043: 0043-move-variable-declaration-out-of-loop.patch +Patch0044: 0044-2289-check-protobuf-and-grpc-version-in-cmake-for-cr.patch +Patch0045: 0045-improve-ut-for-devicemapper.patch +Patch0046: 0046-2292-bugfix-for-run.sh-and-add-build-notify-msg-for-.patch +Patch0047: 0047-2295-keep-the-service-status-unchanged-after-iSulad-.patch +Patch0048: 0048-modify-attach-socket-name.patch +Patch0049: 0049-2298-bugfix-for-hook_ignore_poststart_error-run-in-o.patch +Patch0050: 0050-2304-remove-build-and-test-in-coverage.patch +Patch0051: 0051-2303-use-a-timeout-epoll-loop-to-ensure-complete-dat.patch +Patch0052: 0052-modify-the-default-value-of-ISULAD_TMPDIR-to-var-lib.patch +Patch0053: 0053-prevent-the-parent-dir-from-being-bind-mounted-to-th.patch +Patch0054: 0054-2308-Remove-unused-header-file.patch +Patch0055: 0055-verify-the-mount-dir-first-and-then-create-tmpdir.patch +Patch0056: 0056-2300-Maintaining-a-uniform-code-style.patch +Patch0057: 0057-2312-Add-Huawei-Cloud-CodeArts-compilation-script.patch +Patch0058: 0058-bugfix-del-redundant-code.patch +Patch0059: 0059-improve-code-of-pull.patch +Patch0060: 0060-remove-var-in-coverage-and-fix-build-test-remove.patch +Patch0061: 0061-2320-improve-CI-test.patch +Patch0062: 0062-verify-name-and-digest-consistency.patch +Patch0063: 0063-code-improve-for-oci_rmi.patch +Patch0064: 0064-bugfix-for-resort_image_names.patch +Patch0065: 0065-fix-stopp-removes-cont-error-remove-inspect-error-lo.patch +Patch0066: 0066-2313-use-lxc-5.X-in-CI-testcase.patch +Patch0067: 0067-2329-modify-mount-dev-directory-for-lxc-5.X.patch +Patch0068: 0068-add-cri-1.29-api-change-docs.patch +Patch0069: 0069-add-exec-workdir-support-for-oci-runtime.patch +Patch0070: 0070-add-testcases-for-exec-workdir.patch +Patch0071: 0071-iSulad-restart-isuald-when-upgrade-active-isulad.patch + +%ifarch x86_64 aarch64 +Provides: libhttpclient.so()(64bit) +Provides: libisula_client.so()(64bit) +Provides: libisulad_tools.so()(64bit) +%endif + +%if 0%{?is_systemd} +# Systemd 230 and up no longer have libsystemd-journal +BuildRequires: pkgconfig(systemd) +Requires: systemd-units +%else +Requires(post): chkconfig +Requires(preun): chkconfig +# This is for /sbin/service +Requires(preun): initscripts +%endif + +%if 0%{?is_embedded} +BuildRequires: sqlite-devel +Requires: sqlite +%endif + +%if %{defined openeuler} +BuildRequires: gtest-devel gmock-devel +%endif + +%define lcrver_lower 2.1.3-0 +%define lcrver_upper 2.1.4-0 + +BuildRequires: libisula-devel > %{lcrver_lower} libisula-devel < %{lcrver_upper} +BuildRequires: cmake gcc-c++ yajl-devel +BuildRequires: grpc grpc-plugins grpc-devel protobuf-devel ncurses-devel +BuildRequires: libcurl libcurl-devel libarchive-devel device-mapper-devel +BuildRequires: http-parser-devel +BuildRequires: libseccomp-devel libcap-devel libselinux-devel libwebsockets libwebsockets-devel +BuildRequires: systemd-devel git +BuildRequires: libevhtp-devel libevent-devel +%if 0%{?enable_shimv2} +BuildRequires: lib-shim-v2 lib-shim-v2-devel +%endif + + +Requires: libisula > %{lcrver_lower} libisula < %{lcrver_upper} +Requires: grpc protobuf +Requires: libcurl +Requires: http-parser libseccomp +Requires: libcap libselinux libwebsockets libarchive device-mapper +Requires: systemd +Requires: (docker-runc or runc) +BuildRequires: libevhtp libevent +%if 0%{?enable_shimv2} +Requires: lib-shim-v2 +%endif + +%description +This is a umbrella project for gRPC-services based Lightweight Container +Runtime Daemon, written by C. + +%prep +%autosetup -n iSulad-v%{_version} -Sgit -p1 + +%build +mkdir -p build +cd build +%cmake \ + -DDEBUG=ON \ + -DCMAKE_SKIP_RPATH=TRUE \ + -DLIB_INSTALL_DIR=%{_libdir} \ + -DCMAKE_INSTALL_PREFIX=/usr \ +%if 0%{?enable_criv1} + -DENABLE_CRI_API_V1=ON \ + -DENABLE_SANDBOXER=ON \ +%endif +%if 0%{?enable_shimv2} + -DENABLE_SHIM_V2=ON \ +%endif +%if %{defined openeuler} + -DENABLE_UT=OFF \ +%endif + -DENABLE_GRPC_REMOTE_CONNECT=OFF \ + -DENABLE_GRPC=ON \ + -DCMAKE_CXX_STANDARD=%{cpp_std} \ + ../ + +sed -i "10 a\# undef linux" grpc/src/api/services/cri/v1alpha/api.pb.h +%if 0%{?enable_criv1} +sed -i "10 a\# undef linux" grpc/src/api/services/cri/v1/api_v1.pb.h +%endif + +%make_build + +%check +%if %{defined openeuler} +cd build +# registry_images_ut and volume_ut must run with root user +ctest -E "registry_images_ut|volume_ut" +%endif + +%install +rm -rf %{buildroot} +cd build +install -d $RPM_BUILD_ROOT/%{_libdir} +install -m 0755 ./src/libisula_client.so %{buildroot}/%{_libdir}/libisula_client.so +install -m 0755 ./src/utils/http/libhttpclient.so %{buildroot}/%{_libdir}/libhttpclient.so +install -m 0755 ./src/libisulad_tools.so %{buildroot}/%{_libdir}/libisulad_tools.so + +install -d $RPM_BUILD_ROOT/%{_libdir}/pkgconfig +install -m 0640 ./conf/isulad.pc %{buildroot}/%{_libdir}/pkgconfig/isulad.pc + +install -d $RPM_BUILD_ROOT/%{_bindir} + +install -m 0755 ./src/isula %{buildroot}/%{_bindir}/isula +install -m 0755 ./src/isulad-shim %{buildroot}/%{_bindir}/isulad-shim + +install -m 0755 ./src/isulad %{buildroot}/%{_bindir}/isulad + +install -d $RPM_BUILD_ROOT/%{_includedir}/isulad + +install -d $RPM_BUILD_ROOT/%{_sysconfdir}/isulad +install -m 0640 ../src/contrib/config/daemon.json %{buildroot}/%{_sysconfdir}/isulad/daemon.json +install -m 0440 ../src/contrib/config/daemon_constants.json %{buildroot}/%{_sysconfdir}/isulad/daemon_constants.json +install -m 0640 ../src/contrib/config/seccomp_default.json %{buildroot}/%{_sysconfdir}/isulad/seccomp_default.json + +install -d $RPM_BUILD_ROOT/%{_sysconfdir}/default/isulad +install -m 0640 ../src/contrib/config/config.json %{buildroot}/%{_sysconfdir}/default/isulad/config.json +install -m 0640 ../src/contrib/config/systemcontainer_config.json %{buildroot}/%{_sysconfdir}/default/isulad/systemcontainer_config.json +install -m 0550 ../src/contrib/sysmonitor/isulad-check.sh %{buildroot}/%{_sysconfdir}/default/isulad/isulad-check.sh + +mkdir -p $RPM_BUILD_ROOT/%{_sysconfdir}/sysmonitor/process +cp ../src/contrib/sysmonitor/isulad-monit $RPM_BUILD_ROOT/etc/sysmonitor/process + +install -d $RPM_BUILD_ROOT/%{_sysconfdir}/default/isulad/hooks +install -m 0640 ../src/contrib/config/hooks/default.json %{buildroot}/%{_sysconfdir}/default/isulad/hooks/default.json + +install -d $RPM_BUILD_ROOT/%{_sysconfdir}/sysconfig +install -p -m 0640 ../src/contrib/config/iSulad.sysconfig $RPM_BUILD_ROOT/%{_sysconfdir}/sysconfig/iSulad + +%if 0%{?is_systemd} +install -d $RPM_BUILD_ROOT/%{_unitdir} +install -p -m 0640 ../src/contrib/init/isulad.service $RPM_BUILD_ROOT/%{_unitdir}/isulad.service +%else +install -d $RPM_BUILD_ROOT/%{_initddir} +install -p -m 0640 ../src/contrib/init/isulad.init $RPM_BUILD_ROOT/%{_initddir}/isulad.init +%endif + +%clean +rm -rf %{buildroot} + +%pre +# support update from lcrd to isulad, will remove in next version +if [ "$1" = "2" ]; then +%if 0%{?is_systemd} +systemctl stop lcrd &>/dev/null +systemctl disable lcrd &>/dev/null +if [ -e %{_sysconfdir}/isulad/daemon.json ];then + sed -i 's#/etc/default/lcrd/hooks#/etc/default/isulad/hooks#g' %{_sysconfdir}/isulad/daemon.json +fi +%else +/sbin/chkconfig --del lcrd &>/dev/null +%endif +fi + +%post +if ! getent group isula > /dev/null; then + groupadd --system isula +fi + +if [ "$1" = "1" ]; then +%if 0%{?is_systemd} +systemctl enable isulad +systemctl start isulad +%else +/sbin/chkconfig --add isulad +%endif +elif [ "$1" = "2" ]; then +%if 0%{?is_systemd} +# support update from lcrd to isulad, will remove in next version +if [ -e %{_unitdir}/lcrd.service.rpmsave ]; then + mv %{_unitdir}/lcrd.service.rpmsave %{_unitdir}/isulad.service + sed -i 's/lcrd/isulad/g' %{_unitdir}/isulad.service +fi +# During the isulad upgrade process, the isulad service may still be running, but the service may be unavailable +# due to configuration updates and other reasons. +# it may fail if the X package is upgraded synchronously with isulad and depends on the isulad command, +# For example syscontianer-tools and lxcfs-tools. +# Therefore, after upgrading isulad, if the original status of isulad is running, +# we need to restart isulad to ensure that the service is available during the upgrade process. +systemctl status isulad | grep 'Active:' | grep 'running' +if [ $? -eq 0 ]; then + systemctl restart isulad +fi +%else +/sbin/service isulad status | grep 'Active:' | grep 'running' +if [ $? -eq 0 ]; then + /sbin/service isulad restart +fi +%endif +fi + +if ! getent group isula > /dev/null; then + groupadd --system isula +fi + +%preun +%if 0%{?is_systemd} +%systemd_preun isulad +%else +if [ $1 -eq 0 ] ; then + /sbin/service isulad stop >/dev/null 2>&1 + /sbin/chkconfig --del isulad +fi +%endif + +%postun +%if 0%{?is_systemd} +%systemd_postun_with_restart isulad +%else +if [ "$1" -ge "1" ] ; then + /sbin/service isulad condrestart >/dev/null 2>&1 || : +fi +%endif + +%files +%attr(0600,root,root) %{_sysconfdir}/sysmonitor/process/isulad-monit +%attr(0550,root,root) %{_sysconfdir}/default/isulad/isulad-check.sh +%defattr(0640,root,root,0750) +%{_sysconfdir}/isulad +%{_sysconfdir}/isulad/* +%{_sysconfdir}/default/* +%defattr(-,root,root,-) +%if 0%{?is_systemd} +%{_unitdir}/isulad.service +%attr(0640,root,root) %{_unitdir}/isulad.service +%else +%{_initddir}/isulad.init +%attr(0640,root,root) %{_initddir}/isulad.init +%endif +%attr(0755,root,root) %{_libdir}/pkgconfig +%attr(0640,root,root) %{_libdir}/pkgconfig/isulad.pc +%defattr(0755,root,root,0755) +%{_bindir}/* +%{_libdir}/* +%attr(0640,root,root) %{_sysconfdir}/sysconfig/iSulad +%attr(0640,root,root) %{_sysconfdir}/isulad/daemon.json + +%config(noreplace,missingok) %{_sysconfdir}/sysconfig/iSulad +%config(noreplace,missingok) %{_sysconfdir}/isulad/daemon.json +%if 0%{?is_systemd} +%config(noreplace,missingok) %{_unitdir}/isulad.service +%else +%config(noreplace,missingok) %{_initddir}/isulad.init +%endif + +%changelog +* Fri Dec 29 2023 yangjiaqi <yangjiaqi16@huawei.com> - 2.1.4-5 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: keep the service status unchanged after iSulad service upgrade + +* Thu Dec 28 2023 leizhongkai <leizhongkai@huawei.com> - 2.1.4-4 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: upgrade from upstream + +* Wed Dec 20 2023 zhongtao <zhongtao17@huawei.com> - 2.1.4-3 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: upgrade from upstream + +* Wed Nov 15 2023 zhongtao <zhongtao17@huawei.com> - 2.1.4-2 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: delete libisulad_img.so and compilation dependency: lxc, and add running dependency:runc + +* Tue Nov 14 2023 zhongtao <zhongtao17@huawei.com> - 2.1.4-1 +- Type: update +- ID: NA +- SUG: NA +- DESC: update to v2.1.4 + +* Wed Sep 13 2023 xuxuepeng <xuxuepeng1@huawei.com> - 2.1.3-2 +- Type: update +- ID: NA +- SUG: NA +- DESC: Add vsock support for exec + +* Tue Aug 29 2023 xuxuepeng <xuxuepeng1@huawei.com> - 2.1.3-1 +- Type: update +- ID: NA +- SUG: NA +- DESC: update to v2.1.3 + +* Mon Aug 28 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.2-8 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: modify lxc dependence to docker-runc dependence + +* Mon Aug 21 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.2-7 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: enable grpc and fix compile failed + +* Wed Aug 09 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.2-6 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: modify daemon json default runtime to runc + +* Thu Jul 20 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.2-5 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: compile using c++17 + +* Mon May 29 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.2-4 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: bugfix for memleak and malloc + +* Thu May 25 2023 zhongtao <zhongtao17@huawei.com> - 2.1.2-3 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: upgrade from upstream + +* Fri May 12 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.2-2 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix remote grpc macro + +* Thu May 11 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.2-1 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update from upstream to update cri + +* Fri Mar 24 2023 wangrunze <wangrunze13@huawei.com> - 2.1.1-6 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update from upstream to include remote feature + +* Thu Mar 16 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.1-5 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: use CURLOPT_XFERINFOFUNCTION instead of deprecated CURLOPT_PROGRESSFUNCTION since curl 7.32.0 + +* Wed Feb 22 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.1-4 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: upgrade from upstream + +* Thu Feb 16 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.1-3 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: add check + +* Mon Feb 06 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.1-2 +- Type: update +- ID: NA +- SUG: NA +- DESC: modify dependence from lcr to libisula + +* Mon Feb 06 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.1-1 +- Type: update +- ID: NA +- SUG: NA +- DESC: update to v2.1.1 + +* Tue Jan 03 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.18-1 +- Type: update +- ID: NA +- SUG: NA +- DESC: update to v2.0.18 + +* Thu Dec 22 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-14 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: remove clean_module_fill_ctx for libisulad_img.so + +* Mon Dec 19 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-13 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update lcr dependence version + +* Fri Dec 16 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-12 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update from upstream + +* Tue Dec 06 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-11 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update from upstream + +* Mon Nov 28 2022 yangjiaqi <yangjiaqi16@huawei.com> - 2.0.17-10 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: remove chmod 751 permission for dirs by engine when user-remap enabled + +* Fri Nov 25 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-9 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: add primary group to additional groups + +* Mon Nov 21 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-8 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix cannot install isulad and unknown option + +* Wed Nov 16 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-7 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update tar package from source + +* Wed Nov 02 2022 wangrunze <wangrunze13@huawei.com> - 2.0.17-6 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix cleanup module memory leak + +* Tue Nov 01 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: add ut and bugfix for device mapper and websocket + +* Mon Oct 31 2022 wujing <wujing50@huawei.com> - 2.0.17-4 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from openEuler + +* Wed Oct 19 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: add required package lcr clibcni lower and upper version + +* Mon Oct 10 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-2 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: use epoll instead of select for wait_exit_fifo + +* Sun Oct 09 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-1 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: update iSulad version to 2.0.17-1 + +* Thu Sep 29 2022 haozi007 <liuhao27@huawei.com> - 2.0.16-8 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from openEuler + +* Tue Sep 20 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.16-7 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: change libisulad_tools.so mode + +* Thu Sep 15 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.16-6 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: shield upgrade error if lcrd not exist + +* Tue Sep 13 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.16-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: calculate timezone by tm_gmtoff + +* Thu Sep 08 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.16-4 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: invalid free default-runtime and cri-runtime after free json-confs + +* Wed Sep 07 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.16-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sycn patches from openeuler/iSulad + +* Tue Aug 30 2022 leizhongkai <leizhongkai@huawei.com> - 2.0.16-2 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream openeuler/iSulad + +* Tue Aug 23 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.16-1 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: update iSulad version to 2.0.16-1 + +* Mon Aug 22 2022 zhongtao <zhongtao17@huawei.com> - 2.0.15-6 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: remove rpath by cmake + +* Wed Aug 17 2022 haozi007 <liuhao27@huawei.com> - 2.0.15-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sycn patches from openeuler + +* Mon Aug 15 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.15-4 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: change default umask to 0022 + +* Tue Aug 9 2022 haozi007 <liuhao27@huawei.com> - 2.0.15-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sycn patches from openeuler + +* Mon Aug 1 2022 chengzeruizhi <chengzeruizhi@huawei.com> - 2.0.15-2 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sycn patches from openeuler branch + +* Fri Jul 8 2022 haozi007 <liuhao27@huawei.com> - 2.0.15-1 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: update version to v2.0.15 + +* Fri Jul 8 2022 haozi007 <liuhao27@huawei.com> - 2.0.14-11 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: add limit to lcr version + +* Wed Jun 22 2022 yangjiaqi <yangjiaqi16@huawei.com> - 2.0.14-10 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream openeuler/iSulad + +* Tue Jun 21 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.14-9 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream openeuler/iSulad + +* Wed Jun 15 2022 chengzeruizhi <chengzeruizhi@huawei.com> - 2.0.14-8 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream openeuler/iSulad + +* Tue May 31 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.14-7 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: fix type convert, add null pointer check, remove unuse macro + +* Tue May 31 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.14-6 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: fix different type convert and add check to arguments + +* Mon May 30 2022 chengzrz <chengzeruizhi@huawei.com> - 2.0.14-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: seccomp optimization + +* Fri May 27 2022 haozi007 <liuhao27@huawei.com> - 2.0.14-4 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: improve fuzz test for pw and gr parser + +* Tue May 24 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.14-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: fix install error when android + +* Tue May 24 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.14-2 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: do not mkdir of isulad if no controller found + +* Mon May 23 2022 haozi007 <liuhao27@huawei.com> - 2.0.14-1 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: update version to v2.0.14 + +* Mon May 16 2022 haozi007<liuhao27@huawei.com> - 2.0.13-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream openEuler/iSulad + +* Tue May 10 2022 hejunjie<hejunjie10@huawei.com> - 2.0.13-4 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: bionic adaptation, increase lcov coverage + +* Thu May 5 2022 hejunjie<hejunjie10@huawei.com> - 2.0.13-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: bionic adaptation for pwgr obj parser + +* Mon Apr 25 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.13-2 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: sync from upstream + +* Mon Apr 18 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.13-1 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update version to v2.0.13 + +* Fri Mar 25 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.12-1 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update version to v2.0.12 + +* Thu Mar 17 2022 haozi007 <liuhao27@huawei.com> - 2.0.11-6 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: remove unnecessary error message + +* Thu Mar 17 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.11-5 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix uid/gid error when load image + +* Wed Mar 09 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.11-4 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: add provides of libisulad_tools.so + +* Thu Mar 03 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.11-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: Add the function of isolating the user namespaces + +* Thu Mar 03 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.11-2 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: add libisulad_tools.so + +* Thu Feb 24 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.11-1 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: update version to v2.0.11 + +* Wed Jan 12 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.10-15 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix compile error of isula-transform + +* Wed Jan 12 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.10-14 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix compile error with grpc 1.41.x + +* Tue Jan 4 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.10-13 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix compile error when building embedded image + +* Mon Dec 27 2021 wangfengtu <wangfengtu@huawei.com> - 2.0.10-12 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: sync patches from upstream + +* Thu Dec 09 2021 chengzeruizhi <chengzeruizhi@huawei.com> - 2.0.10-11 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fixed a bug that occurs when starting a container in host mode + +* Thu Dec 09 2021 wangfengtu <wagnfengtu@huawei.com> - 2.0.10-10 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: remove dependance of sqlite + +* Mon Dec 06 2021 gaohuatao <gaohuatao@huawei.com> - 2.0.10-9 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: specify version + +* Fri Dec 03 2021 wangfengtu <wangfengtu@huawei.com> - 2.0.10-8 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix undefined reference to `service_arguments_free' in libisulad_img.so + +* Thu Dec 02 2021 wangfengtu <wangfengtu@huawei.com> - 2.0.10-7 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: sync patches from upstream + +* Tue Nov 23 2021 chengzeruizhi <chengzeruizhi@huawei.com> - 2.0.10-6 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: modify the procedure of running a pod + +* Fri Nov 19 2021 gaohuatao <gaohuatao@huawei.com> - 2.0.10-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream + +* Fri Nov 19 2021 wangfengtu <wangfengtu@huawei.com> - 2.0.10-4 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix memleak when use multiple --volumes-from + +* Tue Nov 16 2021 wujing <wujing50@huawei.com> - 2.0.10-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: add shimv2 build switch + +* Tue Nov 16 2021 wujing <wujing50@huawei.com> - 2.0.10-2 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: remove build platform restrictions + +* Tue Nov 09 2021 gaohuatao <gaohuatao@huawei.com> - 2.0.10-1 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update from openeuler + +* Tue Oct 19 2021 wangfengtu <wangfengtu@huawei.com> - 2.0.9-20211019.121837.gitf067b3ce +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: strip sha256 prefix when decrease hold references + +* Fri Jun 25 2021 wujing <wujing50@huawei.com> - 2.0.9-20210625.165022.git5a088d9c +- Type: update to v2.0.9 +- ID: NA +- SUG: NA +- DESC: update from master + +* Tue May 18 2021 wangfengtu <wangfengtu@huawei.com> - 2.0.8-20210518.144540.git5288ed92 +- Type: sync from upstream +- ID: NA +- SUG: NA +- DESC: update from master + +* Fri Mar 26 2021 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.8-20210326.094027.gitac974aa6 +- Type: sync from upstream +- ID: NA +- SUG: NA +- DESC: update from master + +* Tue Mar 23 2021 haozi007 <liuhao27@huawei.com> - 20210323.094917.git7e6aa593 +- Type: sync from upstream +- ID: NA +- SUG: NA +- DESC: update from master + +* Tue Feb 2 2021 lifeng <lifeng68@huawei.com> - 2.0.8-20210202.153251.gite082dcf3 +- Type: sync from upstream +- ID: NA +- SUG: NA +- DESC: update from master + +* Mon Jan 18 2021 lifeng <lifeng68@huawei.com> - 2.0.8-20210118.195254.git077e10f2 +- Type: sync from upstream +- ID: NA +- SUG: NA +- DESC: update from master + +* Wed Dec 30 2020 lifeng <lifeng68@huawei.com> - 2.0.8-20201230.155843.git6557a6eb +- Type: update to v2.0.8 +- ID: NA +- SUG: NA +- DESC: update from master + +* Mon Dec 7 2020 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.7-20201207.151847.gita1fce123 +- Type: update +- ID: NA +- SUG: NA +- DESC: update from master + +* Sat Dec 5 2020 lifeng <lifeng68@huawei.com> - 2.0.7-20201205.145752.gita461cc51 +- Type: bugfix +- ID:NA +- SUG:NA +- DESC: ignore list containers errors + +* Thu Dec 3 2020 haozi007 <liuhao27@huawei.com> - 2.0.7-20201203.190902.git48f598fd +- Type:update from master +- ID:NA +- SUG:NA +- DESC: update from master + +* Sat Nov 28 2020 lifeng<lifeng68@huawei.com> - 2.0.7-20201128.095506.git1e1623a5 +- Type: bugfix +- ID:NA +- SUG:NA +- DESC: Mounts: only qsort the configed mounts and make possible to bind mount /proc and /sys/fs. +- related lxc PR fixed: +- 1.add check whether have /proc mounts entry, if has, skip the auto +- 2.mount cgroup before do mount entrys +- 3.pass if the mount on top of /proc and the source of the mount is a proc filesystem + +* Wed Nov 25 2020 wangfengtu<wangfengtu@huawei.com> - 2.0.7-20201125.165149.git7d150c3c +- Type: bugfix +- ID:NA +- SUG:NA +- DESC: update from openeuler + +* Wed Nov 25 2020 wangfengtu<wangfengtu@huawei.com> - 2.0.6-20201125.160534.git9fb5e75d +- Type: bugfix +- ID:NA +- SUG:NA +- DESC: fix rpath not work + +* Thu Nov 12 2020 gaohuatao<gaohuatao@huawei.com> - 2.0.6-20201112.193005.git8a6b73c8 +- Type: update from openeuler +- ID:NA +- SUG:NA +- DESC: update from openeuler + +* Wed Oct 14 2020 lifeng68<lifeng68@huawei.com> - 2.0.6-20201014.152749.gitc8a43925 +- Type: upgrade to v2.0.6 +- ID:NA +- SUG:NA +- DESC: upgrade to v2.0.6 + +* Fri Sep 18 2020 <lifeng68@huawei.com> - 2.0.5-20200918.112827.git9aea9b75 +- Type:bugfix +- ID:NA +- SUG:NA +- DESC: modify log level to warn + +* Mon Sep 14 2020 <lifeng68@huawei.com> - 2.0.5-20200914.172527.gitae86920a +- Type:bugfix +- ID:NA +- SUG:NA +- DESC: remove unused config + +* Thu Sep 10 2020 <yangjiaqi11@huawei.com> - 2.0.5-20200910.144345.git71b1055b +- Type:enhancement +- ID:NA +- SUG:NA +- DESC: add chrpath + +* Fri Sep 04 2020 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.5-20200904.114315.gitff1761c3 +- Type:enhancement +- ID:NA +- SUG:NA +- DESC: upgrade from v2.0.3 to v2.0.5 + +* Wed Sep 02 2020 YoungJQ <yangjiaqi11@huawei.com> - 2.0.3-20200902.114727.git6d945f26 +- Type:enhancement +- ID:NA +- SUG:NA +- DESC: modify source0 address @@ -0,0 +1 @@ +bc668d146a03204fdc5db3cae5c3dedf v2.1.4.tar.gz |