From d21dc88cb33049ec5d9ce8e16eb1a2f9c070219b Mon Sep 17 00:00:00 2001 From: zhongtao Date: Tue, 30 Jul 2024 20:17:08 +0800 Subject: [PATCH 117/121] [NRI] add nri head file and common func Signed-off-by: zhongtao --- cmake/checker.cmake | 8 ++ cmake/options.cmake | 13 ++- src/CMakeLists.txt | 10 +++ src/daemon/CMakeLists.txt | 9 ++ src/daemon/common/CMakeLists.txt | 5 ++ src/daemon/common/nri/CMakeLists.txt | 13 +++ src/daemon/common/nri/nri_convert.h | 36 ++++++++ src/daemon/common/nri/nri_spec.h | 24 ++++++ src/daemon/common/nri/nri_utils.h | 71 ++++++++++++++++ src/daemon/config/isulad_config.h | 16 ++++ src/daemon/modules/api/specs_api.h | 10 ++- src/daemon/modules/spec/specs.c | 67 ++++++++++++++- src/daemon/nri/CMakeLists.txt | 6 ++ src/daemon/nri/nri_adaption.h | 123 +++++++++++++++++++++++++++ src/daemon/nri/nri_helpers.h | 63 ++++++++++++++ src/daemon/nri/nri_plugin_ops.h | 38 +++++++++ src/daemon/nri/nri_result.h | 114 +++++++++++++++++++++++++ src/daemon/nri/plugin.h | 95 +++++++++++++++++++++ src/utils/cpputils/transform.cc | 16 ++++ src/utils/cpputils/transform.h | 2 + src/utils/cutils/utils.c | 7 ++ src/utils/cutils/utils.h | 2 + src/utils/tar/util_archive.c | 7 -- 23 files changed, 744 insertions(+), 11 deletions(-) create mode 100644 src/daemon/common/nri/CMakeLists.txt create mode 100644 src/daemon/common/nri/nri_convert.h create mode 100644 src/daemon/common/nri/nri_spec.h create mode 100644 src/daemon/common/nri/nri_utils.h create mode 100644 src/daemon/nri/CMakeLists.txt create mode 100644 src/daemon/nri/nri_adaption.h create mode 100644 src/daemon/nri/nri_helpers.h create mode 100644 src/daemon/nri/nri_plugin_ops.h create mode 100644 src/daemon/nri/nri_result.h create mode 100644 src/daemon/nri/plugin.h diff --git a/cmake/checker.cmake b/cmake/checker.cmake index 13f0fd62..b877c1c8 100644 --- a/cmake/checker.cmake +++ b/cmake/checker.cmake @@ -113,6 +113,14 @@ if (ENABLE_SHIM_V2) _CHECK(LIBSHIM_V2_LIBRARY "LIBSHIM_V2_LIBRARY-NOTFOUND" "libshim_v2.so") endif() +if (ENABLE_NRI) + find_path(NRI_INCLUDE_DIR nri_plugin.h) + _CHECK(NRI_INCLUDE_DIR "NRI_INCLUDE_DIR-NOTFOUND" "nri_plugin.h") + + find_library(LIBISULA_NRI_LIBRARY isula_nri) + _CHECK(LIBISULA_NRI_LIBRARY "LIBISULA_NRI_LIBRARY-NOTFOUND" "libisula_nri.so") +endif() + if (OPENSSL_VERIFY) find_path(OPENSSL_INCLUDE_DIR openssl/x509.h) _CHECK(OPENSSL_INCLUDE_DIR "OPENSSL_INCLUDE_DIR-NOTFOUND" "openssl/x509.h") diff --git a/cmake/options.cmake b/cmake/options.cmake index 5b17f631..41177fe0 100644 --- a/cmake/options.cmake +++ b/cmake/options.cmake @@ -59,12 +59,23 @@ if (ENABLE_SANDBOXER STREQUAL "ON") endif() option(ENABLE_OOM_MONITOR "Enable oom monitor" ON) -IF (ENABLE_OOM_MONITOR STREQUAL "ON") +if (ENABLE_OOM_MONITOR STREQUAL "ON") add_definitions(-DENABLE_OOM_MONITOR) set(ENABLE_OOM_MONITOR 1) message("${Green}-- Enable oom monitor${ColourReset}") endif() +option(ENABLE_NRI "Enable NRI API" OFF) +if (ENABLE_NRI STREQUAL "ON") + if (ENABLE_CRI_API_V1) + add_definitions(-DENABLE_NRI) + set(ENABLE_NRI 1) + message("${Green}-- Enable NRI API${ColourReset}") + else() + message("${Yellow}-- Can not enable NRI, NRI need enable CRI API V1 first ${ColourReset}") + endif() +endif() + option(ENABLE_SYSTEMD_NOTIFY "Enable systemd notify" ON) if (ENABLE_SYSTEMD_NOTIFY STREQUAL "ON") add_definitions(-DSYSTEMD_NOTIFY) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 48c1bad0..70a8ac91 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -15,6 +15,12 @@ if (ENABLE_SHIM_V2) ) endif() +if (ENABLE_NRI) + list(APPEND CHECKED_INCLUDE_DIRS + ${NRI_INCLUDE_DIR} + ) +endif() + if (GRPC_CONNECTOR) list(APPEND CHECKED_INCLUDE_DIRS ${GRPC_INCLUDE_DIR} @@ -168,6 +174,10 @@ if (ENABLE_SHIM_V2) target_link_libraries(isulad ${LIBSHIM_V2_LIBRARY}) endif() +if (ENABLE_NRI) + target_link_libraries(isulad ${LIBISULA_NRI_LIBRARY}) +endif() + if (ENABLE_EMBEDDED_IMAGE) target_link_libraries(isulad ${SQLITE3_LIBRARY}) endif() diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index 29af3dca..801f5797 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -36,6 +36,15 @@ if (ENABLE_CRI_API_V1) list (APPEND local_daemon_incs ${SANDBOX_INCS} ) + if (ENABLE_NRI) + add_subdirectory(nri) + list (APPEND local_daemon_srcs + ${NRI_SRCS} + ) + list (APPEND local_daemon_incs + ${NRI_INCS} + ) + endif() endif() set(DAEMON_SRCS diff --git a/src/daemon/common/CMakeLists.txt b/src/daemon/common/CMakeLists.txt index e88578dd..ffeed4b3 100644 --- a/src/daemon/common/CMakeLists.txt +++ b/src/daemon/common/CMakeLists.txt @@ -10,12 +10,16 @@ if (GRPC_CONNECTOR) endif() add_subdirectory(cgroup) +if (ENABLE_NRI) + add_subdirectory(nri) +endif() set(local_daemon_common_srcs ${daemon_common_top_srcs}) set(DAEMON_COMMON_SRCS ${COMMON_CRI_SRCS} ${COMMON_CGROUP_SRCS} + ${COMMON_NRI_SRCS} ${local_daemon_common_srcs} PARENT_SCOPE ) @@ -23,6 +27,7 @@ set(DAEMON_COMMON_SRCS set(DAEMON_COMMON_INCS ${COMMON_CRI_INCS} ${COMMON_CGROUP_INCS} + ${COMMON_NRI_INCS} ${CMAKE_CURRENT_SOURCE_DIR} PARENT_SCOPE ) diff --git a/src/daemon/common/nri/CMakeLists.txt b/src/daemon/common/nri/CMakeLists.txt new file mode 100644 index 00000000..512ba694 --- /dev/null +++ b/src/daemon/common/nri/CMakeLists.txt @@ -0,0 +1,13 @@ +# get current directory sources files +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_common_nri_srcs) +set(local_common_nri_incs ${CMAKE_CURRENT_SOURCE_DIR}) + +set(COMMON_NRI_SRCS + ${local_common_nri_srcs} + PARENT_SCOPE +) + +set(COMMON_NRI_INCS + ${local_common_nri_incs} + PARENT_SCOPE +) diff --git a/src/daemon/common/nri/nri_convert.h b/src/daemon/common/nri/nri_convert.h new file mode 100644 index 00000000..883f7c41 --- /dev/null +++ b/src/daemon/common/nri/nri_convert.h @@ -0,0 +1,36 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * iSulad licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: zhongtao + * Create: 2024-03-16 + * Description: provide nri convert functions + *********************************************************************************/ +#ifndef DAEMON_COMMON_NRI_NRI_CONVERT_H +#define DAEMON_COMMON_NRI_NRI_CONVERT_H +#include +#include +#include +#include + +#include "isula_libutils/nri_pod_sandbox.h" +#include "isula_libutils/nri_container.h" +#include "isula_libutils/nri_mount.h" +#include "isula_libutils/nri_linux_resources.h" + +#include "sandbox.h" +#include "api_v1.pb.h" + +auto PodSandboxToNRI(const std::shared_ptr &sandbox, nri_pod_sandbox *pod) -> bool; +auto ContainerToNRIByConConfig(const runtime::v1::ContainerConfig &containerConfig, nri_container *con) -> bool; +auto ContainerToNRIByID(const std::string &id, nri_container *con) -> bool; +auto PodSandboxesToNRI(const std::vector> &arrs, nri_pod_sandbox **pod, int pod_len) -> bool; + +auto LinuxResourcesFromNRI(const nri_linux_resources *src, runtime::v1::LinuxContainerResources &resources) -> bool; +#endif // DAEMON_COMMON_NRI_NRI_CONVERT_H diff --git a/src/daemon/common/nri/nri_spec.h b/src/daemon/common/nri/nri_spec.h new file mode 100644 index 00000000..e7c5035d --- /dev/null +++ b/src/daemon/common/nri/nri_spec.h @@ -0,0 +1,24 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * iSulad licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: zhongtao + * Create: 2024-07-17 + * Description: provide nri oci functions + *********************************************************************************/ + +#ifndef DAEMON_COMMON_NRI_NRI_SPEC_H +#define DAEMON_COMMON_NRI_NRI_SPEC_H + +#include +#include + +int nri_adjust_oci_spec(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec); + +#endif // DAEMON_COMMON_NRI_NRI_SPEC_H \ No newline at end of file diff --git a/src/daemon/common/nri/nri_utils.h b/src/daemon/common/nri/nri_utils.h new file mode 100644 index 00000000..3aa50ae4 --- /dev/null +++ b/src/daemon/common/nri/nri_utils.h @@ -0,0 +1,71 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * iSulad licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: zhongtao + * Create: 2024-07-17 + * Description: provide nri utils functions + *********************************************************************************/ +#ifndef DAEMON_COMMON_NRI_NRI_UTILS_H +#define DAEMON_COMMON_NRI_NRI_UTILS_H + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum { + UNKNOWN = 0, + RUN_POD_SANDBOX = 1, + STOP_POD_SANDBOX = 2, + REMOVE_POD_SANDBOX = 3, + CREATE_CONTAINER = 4, + POST_CREATE_CONTAINER = 5, + START_CONTAINER = 6, + POST_START_CONTAINER = 7, + UPDATE_CONTAINER = 8, + POST_UPDATE_CONTAINER = 9, + STOP_CONTAINER = 10, + REMOVE_CONTAINER = 11, + LAST = 12, +} NRI_Event; + +bool copy_nri_mount(const nri_mount *src, nri_mount **dest); +bool copy_nri_key_value(const nri_key_value *src, nri_key_value **dest); +bool copy_nri_hook(const nri_hook *src, nri_hook **dest); +bool copy_nri_posix_rlimit(const nri_posix_rlimit *src, nri_posix_rlimit **dest); +bool copy_nri_linux_resources(const nri_linux_resources *src, nri_linux_resources **dest); +bool copy_nri_linux_cpu(const nri_linux_cpu *src, nri_linux_cpu **dest); + +bool is_marked_for_removal(const char* key, char **out); + +bool realloc_and_copy_nri_hooks(nri_hook **targetHooks, size_t targetSize, const nri_hook **sourceHooks, size_t sourceLen); + +bool init_nri_container_adjust(nri_container_adjustment **adjust); +bool init_nri_container_update(nri_container_update **update, const char *id, uint8_t ignore_failure); +bool init_nri_linux_resources(nri_linux_resources **resources); + +#ifdef __cplusplus +} +#endif + +#endif // DAEMON_COMMON_NRI_NRI_UTILS_H \ No newline at end of file diff --git a/src/daemon/config/isulad_config.h b/src/daemon/config/isulad_config.h index f29cd564..c57ca8d6 100644 --- a/src/daemon/config/isulad_config.h +++ b/src/daemon/config/isulad_config.h @@ -40,6 +40,22 @@ struct isulad_conf { #define DEFAULT_SANDBOXER_NAME "shim" char *conf_get_sandbox_rootpath(void); char *conf_get_sandbox_statepath(void); + +#ifdef ENABLE_NRI +#define DEFAULT_SOCKET_PATH "/var/run/nri/nri.sock" +#define DEFAULT_MUX_SOCKET_PATH "/var/run/nri/nri_mux.sock" +#define DEFAULT_PLUGIN_PATH "/opt/nri/plugins" +#define DEFAULT_PLUGIN_CONFIG_PATH "/etc/nri/conf.d" +#define DEFAULT_PLUGIN_REGISTRY_TIMEOUT 5 +#define DEFAULT_PLUGIN_REQUST_TIMEOUT 2 +bool conf_get_nri_support(void); +bool conf_get_nri_external_support(void); +char *conf_get_nri_plugin_config_path(void); +char *conf_get_nri_plugin_path(void); +char *conf_get_socket_path(void); +uint64_t conf_get_nri_plugin_registration_timeout(void); +uint64_t conf_get_nri_plugin_requst_timeout(void); +#endif #endif char *conf_get_isulad_pidfile(void); diff --git a/src/daemon/modules/api/specs_api.h b/src/daemon/modules/api/specs_api.h index 0999836b..6a1cd776 100644 --- a/src/daemon/modules/api/specs_api.h +++ b/src/daemon/modules/api/specs_api.h @@ -58,6 +58,9 @@ int spec_module_init(void); #ifdef ENABLE_CDI int defs_process_add_multiple_env(defs_process *dp, const char **envs, size_t env_len); int spec_add_multiple_process_env(oci_runtime_spec *oci_spec, const char **envs, size_t env_len); +#endif/* ENABLE_CDI */ + +#if defined (ENABLE_CDI) || defined(ENABLE_NRI) int spec_add_device(oci_runtime_spec *oci_spec, defs_device *device); int spec_add_linux_resources_device(oci_runtime_spec *oci_spec, bool allow, const char *dev_type, int64_t major, int64_t minor, const char *access); @@ -66,7 +69,12 @@ int spec_add_mount(oci_runtime_spec *oci_spec, defs_mount *mnt); int spec_add_prestart_hook(oci_runtime_spec *oci_spec, defs_hook *prestart_hook); int spec_add_poststart_hook(oci_runtime_spec *oci_spec, defs_hook *poststart_hook); int spec_add_poststop_hook(oci_runtime_spec *oci_spec, defs_hook *poststop_hook); -#endif /* ENABLE_CDI */ +#endif /* ENABLE_CDI || ENABLE_NRI */ + +#ifdef ENABLE_NRI +int spec_add_linux_resources_hugepage_limit(oci_runtime_spec *oci_spec, const char *page_size, uint64_t limit); +int spec_add_linux_resources_rlimit(oci_runtime_spec *oci_spec, const char *type, uint64_t hard, uint64_t soft); +#endif /* ENABLE_NRI */ #ifdef __cplusplus } diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c index f0538e26..1fd9e5a8 100644 --- a/src/daemon/modules/spec/specs.c +++ b/src/daemon/modules/spec/specs.c @@ -2608,7 +2608,7 @@ int spec_module_init(void) static int add_env(defs_process *dp, const char *env, const char *key) { size_t i; - + for (i = 0; i < dp->env_len; i++) { __isula_auto_free char *oci_key = NULL; if (util_valid_split_env(dp->env[i], &oci_key, NULL) < 0) { @@ -2684,7 +2684,9 @@ int spec_add_multiple_process_env(oci_runtime_spec *oci_spec, const char **envs, return ret; } +#endif /* ENABLE_CDI */ +#if defined (ENABLE_CDI) || defined(ENABLE_NRI) int spec_add_device(oci_runtime_spec *oci_spec, defs_device *device) { int ret = 0; @@ -2817,4 +2819,65 @@ SPEC_ADD_HOOKS_ITEM_DEF(prestart) SPEC_ADD_HOOKS_ITEM_DEF(poststart) SPEC_ADD_HOOKS_ITEM_DEF(poststop) -#endif /* ENABLE_CDI */ \ No newline at end of file +#endif /* ENABLE_CDI || ENABLE_NRI */ + +#ifdef ENABLE_NRI +int spec_add_linux_resources_hugepage_limit(oci_runtime_spec *oci_spec, const char *page_size, uint64_t limit) +{ + int ret = 0; + defs_resources_hugepage_limits_element *hugepage_limit = NULL; + + ret = make_sure_oci_spec_linux_resources(oci_spec); + if (ret < 0) { + return -1; + } + + hugepage_limit = util_common_calloc_s(sizeof(*hugepage_limit)); + if (hugepage_limit == NULL) { + ERROR("Out of memory"); + return -1; + } + hugepage_limit->page_size = util_strdup_s(page_size); + hugepage_limit->limit = limit; + + if (util_mem_realloc((void **)&oci_spec->linux->resources->hugepage_limits, + (oci_spec->linux->resources->hugepage_limits_len + 1) * sizeof(defs_resources_hugepage_limits_element *), + (void *)oci_spec->linux->resources->hugepage_limits, + oci_spec->linux->resources->hugepage_limits_len * sizeof(defs_resources_hugepage_limits_element *)) != 0) { + ERROR("Out of memory"); + free_defs_resources_hugepage_limits_element(hugepage_limit); + return -1; + } + oci_spec->linux->resources->hugepage_limits[oci_spec->linux->resources->hugepage_limits_len] = hugepage_limit; + oci_spec->linux->resources->hugepage_limits_len++; + + return 0; +} + +int spec_add_linux_resources_rlimit(oci_runtime_spec *oci_spec, const char *type, uint64_t hard, uint64_t soft) +{ + int ret = 0; + defs_process_rlimits_element *rlimit = NULL; + + ret = make_sure_oci_spec_linux_resources(oci_spec); + if (ret < 0) { + return -1; + } + + rlimit = util_common_calloc_s(sizeof(*rlimit)); + if (rlimit == NULL) { + ERROR("Out of memory"); + return -1; + } + rlimit->type = util_strdup_s(type); + rlimit->hard = hard; + rlimit->soft = soft; + + if (do_merge_one_ulimit_override(oci_spec, rlimit) != 0) { + ERROR("Failed to merge one nri ulimit to oci spec"); + return -1; + } + + return 0; +} +#endif /* ENABLE_NRI */ \ No newline at end of file diff --git a/src/daemon/nri/CMakeLists.txt b/src/daemon/nri/CMakeLists.txt new file mode 100644 index 00000000..361d79f8 --- /dev/null +++ b/src/daemon/nri/CMakeLists.txt @@ -0,0 +1,6 @@ +# get current directory sources files +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} nri_top_srcs) + +set(NRI_INCS ${CMAKE_CURRENT_SOURCE_DIR} PARENT_SCOPE) + +set(NRI_SRCS ${nri_top_srcs} PARENT_SCOPE) diff --git a/src/daemon/nri/nri_adaption.h b/src/daemon/nri/nri_adaption.h new file mode 100644 index 00000000..7f0640df --- /dev/null +++ b/src/daemon/nri/nri_adaption.h @@ -0,0 +1,123 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * iSulad licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: zhongtao + * Create: 2024-03-15 + * Description: provide plugin manager(NRI adaption) class definition + *********************************************************************************/ + +#ifndef DAEMON_NRI_PLUGIN_NRI_ADAPTION_H +#define DAEMON_NRI_PLUGIN_NRI_ADAPTION_H + +// #include "read_write_lock.h" +#include +#include + +#include "plugin.h" +#include "sandbox.h" +#include "v1_cri_container_manager_service.h" + +const std::string PluginNameEnv = "NRI_PLUGIN_NAME"; +const std::string PluginIdxEnv = "NRI_PLUGIN_IDX"; +const std::string PluginSocketEnv = "NRI_PLUGIN_SOCKET"; + +struct nri_plugin_exec_args_t { + const char *workdir; + const char *cmd; + const char *name; + const char *index; + const uint32_t sockFd; +}; + +class NRIAdaptation { +public: + // Singleton + static NRIAdaptation *GetInstance() noexcept; + + // initialize value + auto Init(Errors &error) -> bool; + + auto GetSockpath(std::vector &paths) -> bool; + + // Stop plugins. + auto StopPlugins() -> bool; + + void RemoveClosedPlugins(); + + auto GetPluginByIndex(const std::string &index) -> std::shared_ptr; + void AddPluginByIndex(const std::string &index, std::shared_ptr plugin); + void RemovePluginByIndex(const std::string &index); + + auto RunPodSandbox(std::shared_ptr sandbox, Errors &error) ->bool; + auto StopPodSandbox(std::shared_ptr sandbox, Errors &error) ->bool; + auto RemovePodSandbox(std::shared_ptr sandbox, Errors &error) ->bool; + auto CreateContainer(std::shared_ptr sandbox, const std::string &conId, const runtime::v1::ContainerConfig &containerConfig, nri_container_adjustment **adjust, Errors &error) -> bool; + auto PostCreateContainer(const std::string &conId, Errors &error) ->bool; + auto UndoCreateContainer(std::shared_ptr sandbox, const std::string &conId, Errors &error) -> bool; + auto StartContainer(const std::string &conId, Errors &error) ->bool; + auto PostStartContainer(const std::string &conId, Errors &error) ->bool; + auto UpdateContainer(const std::string &conId, Errors &error) ->bool; + auto PostUpdateContainer(const std::string &conId, Errors &error) ->bool; + auto StopContainer(const std::string &conId, Errors &error) ->bool; + auto RemoveContainer(const std::string &conId, Errors &error) ->bool; + auto StateChange(nri_state_change_event *evt, Errors &error) ->bool; + auto updateContainers(const nri_update_containers_request *req, nri_update_containers_response **resp) ->bool; + + auto NewExternalPlugin(int fd) -> bool; +private: + NRIAdaptation() = default; + NRIAdaptation(const NRIAdaptation &other) = delete; + NRIAdaptation &operator=(const NRIAdaptation &) = delete; + virtual ~NRIAdaptation(); + + auto StartPlugin() -> bool; + auto NewLaunchedPlugin(const std::shared_ptr &) -> bool; + auto DiscoverPlugins(std::map> &map) -> bool; + // Synchronizing NRI (plugin) with current runtime state + auto SyncPlugin() -> bool; + + auto SortPlugins() -> bool; + void GetClosedPlugins(std::vector &closedPlugin); + + auto IsSupport() -> bool; + + auto ApplyUpdates(const std::vector &update, std::vector &failed, bool getFailed, + Errors &error) -> bool; + + auto NRIPodSandbox(const std::shared_ptr &sandbox, Errors& error) -> std::unique_ptr>; + auto NRIContainerByConConfig(const std::shared_ptr &sandbox, const runtime::v1::ContainerConfig &containerConfig, Errors& error) -> std::unique_ptr>; + auto NRIContainerByID(const std::string &id, Errors& error) -> std::unique_ptr>; + + auto GetNRIPluginConfigPath(void) -> std::string; + auto GetNRIPluginPath(void) -> std::string; + auto GetNRISockPath(void) -> std::string; +private: + RWMutex m_mutex; + static std::atomic m_instance; + bool m_support; + bool m_external_support; + std::string m_version; + std::string m_sock_path; + std::string m_pluginConfigPath; + std::string m_pluginPath; + std::vector m_socketPathArr; + std::string m_disableConnections; + // id --> NRIPlugin map + std::map> m_storeMap; + // TODO:plugin monitor thread id?? + // shutdown() to clean resource + // init to create thread + // todo: if Singleton? + std::unique_ptr m_containerManager; + int64_t m_plugin_registration_timeout; + int64_t m_plugin_requst_timeout; +}; + +#endif // DAEMON_NRI_PLUGIN_NRI_ADAPTION_H \ No newline at end of file diff --git a/src/daemon/nri/nri_helpers.h b/src/daemon/nri/nri_helpers.h new file mode 100644 index 00000000..06ee8419 --- /dev/null +++ b/src/daemon/nri/nri_helpers.h @@ -0,0 +1,63 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2017-2019. All rights reserved. + * iSulad licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: zhongtao + * Create: 2024-07-13 + * Description: provide nri helpers functions + *********************************************************************************/ +#ifndef DAEMON_NRI_PLUGIN_NRI_HELPERS_H +#define DAEMON_NRI_PLUGIN_NRI_HELPERS_H + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "errors.h" + +namespace NRIHelpers { +std::string MarkForRemoval(const std::string &key); + +auto GetPluginConfig(std::string &idx, std::string &name, std::string &config) -> bool; + +std::string GenerateRandomExternalName(void); + +bool CheckPluginIndex(const std::string &idx); + +template +void freeArray(T ** &arr, int size) +{ + if (arr == NULL) { + return; + } + + for (int i = 0; i < size; i++) { + if (arr[i] == NULL) { + return; + } + free(arr[i]); + } + + free(arr); + arr = NULL; +} +}; // namespace NRIHelpers + +#endif // DAEMON_NRI_PLUGIN_NRI_HELPERS_H diff --git a/src/daemon/nri/nri_plugin_ops.h b/src/daemon/nri/nri_plugin_ops.h new file mode 100644 index 00000000..37d437d2 --- /dev/null +++ b/src/daemon/nri/nri_plugin_ops.h @@ -0,0 +1,38 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * iSulad licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: zhongtaoo + * Create: 2024-03-26 + * Description: provide nri plugin api definition + ******************************************************************************/ + +#ifndef DAEMON_NRI_PLUGIN_OPS_H +#define DAEMON_NRI_PLUGIN_OPS_H + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +bool nri_adaption_init(void); + +#ifdef __cplusplus +} +#endif + +int nri_update_containers(const nri_update_containers_request *request, nri_update_containers_response **response); +int nri_registry_containers(const nri_register_plugin_request *request); + +int nri_external_plugin_connect(int fd); + +#endif // DAEMON_NRI_PLUGIN_OPS_H diff --git a/src/daemon/nri/nri_result.h b/src/daemon/nri/nri_result.h new file mode 100644 index 00000000..f2896ea0 --- /dev/null +++ b/src/daemon/nri/nri_result.h @@ -0,0 +1,114 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * iSulad licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: zhongtao + * Create: 2024-06-29 + * Description: provide nri result definition + *********************************************************************************/ + +#ifndef DAEMON_NRI_PLUGIN_NRI_RESULT_H +#define DAEMON_NRI_PLUGIN_NRI_RESULT_H + +#include + +#include +#include +#include +#include +#include +#include + +#include "api_v1.pb.h" +#include "nri_helpers.h" +#include "nri_utils.h" + +using EventMask = std::int32_t; + +const EventMask ValidEvents = (1 << (LAST - 1)) - 1; + +struct owners { + std::map annotations; + std::map mounts; + std::map devices; + std::map env; + std::string memLimit; + std::string memReservation; + std::string memSwapLimit; + std::string memKernelLimit; + std::string memTCPLimit; + std::string memSwappiness; + std::string memDisableOomKiller; + std::string memUseHierarchy; + std::string cpuShares; + std::string cpuQuota; + std::string cpuPeriod; + std::string cpuRealtimeRuntime; + std::string cpuRealtimePeriod; + std::string cpusetCpus; + std::string cpusetMems; + std::map hugepageLimits; + std::string blockioClass; + std::string rdtClass; + std::map unified; + std::string cgroupsPath; + std::map rlimits; +}; + +struct resultReply { + nri_container_adjustment* adjust; + std::vector update; +}; + +using resultOwners = std::map; + +class pluginResult { +public: + pluginResult() = default; + + ~pluginResult() = default; + + auto InitByConId(std::string conId) -> bool; + auto InitByUpdateReq(nri_update_container_request *req) -> bool; + + auto GetReplyUpdate() -> std::vector; + auto GetReplyAdjust() -> nri_container_adjustment *; + + auto Apply(int32_t event, nri_container_adjustment *adjust, nri_container_update **update, size_t update_len, + const std::string &plugin) -> bool; + auto Update(nri_container_update **updates, size_t update_len, const std::string &plugin) -> bool; + +private: + auto GetContainerUpdate(nri_container_update *update, const std::string &plugin, nri_container_update **out) -> bool; + auto UpdateResources(nri_container_update *reply, nri_container_update *u, const std::string &plugin) -> bool; + + auto InitReply(void) -> bool; + + auto Adjust(nri_container_adjustment *adjust, const std::string &plugin) -> bool; + + auto AdjustAnnotations(json_map_string_string *annos, const std::string &plugin) -> bool; + auto AdjustMounts(nri_mount **mounts, size_t mounts_size, const std::string &plugin) -> bool; + auto AdjustEnv(nri_key_value **envs, size_t envs_size, const std::string &plugin) -> bool; + auto AdjustHooks(nri_hooks *hooks, const std::string &plugin) -> bool; + auto AdjustDevices(nri_linux_device **devices, size_t devices_size, const std::string &plugin) -> bool; + auto AdjustResources(nri_linux_resources *resources, const std::string &plugin) -> bool; + bool ClaimAndCopyResources(nri_linux_resources *src, std::string &id, const std::string &plugin, + nri_linux_resources *dest); + auto AdjustCgroupsPath(char *path, const std::string &plugin) -> bool; + auto AdjustRlimits(nri_posix_rlimit **rlimits, size_t rlimits_len, const std::string &plugin) -> bool; + +private: + std::string m_conId; + nri_linux_resources *m_update_req; + resultReply m_reply; + std::map m_updates; + resultOwners m_owners; +}; + +#endif \ No newline at end of file diff --git a/src/daemon/nri/plugin.h b/src/daemon/nri/plugin.h new file mode 100644 index 00000000..f60a9b3d --- /dev/null +++ b/src/daemon/nri/plugin.h @@ -0,0 +1,95 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * iSulad licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: zhongtao + * Create: 2024-03-15 + * Description: provide plugin class definition + *********************************************************************************/ + +#ifndef DAEMON_NRI_PLUGIN_PLUGIN_H +#define DAEMON_NRI_PLUGIN_PLUGIN_H + +#include + +#include +#include +#include + +#include "errors.h" +#include "read_write_lock.h" +#include "nri_result.h" + +const std::string NRIRruntime = "v2"; +const std::string NRIVersion = "2.0.0-beta.2+unknown"; + +class NRIPlugin { +public: + // init client conn + NRIPlugin(std::string &idx, std::string &name, std::string &config); + NRIPlugin(int fd, std::string &name); + // todo: close client conn ?? or single close func? + virtual ~NRIPlugin() = default; + // wait for plugin to register, then configure it. + auto Start(int64_t registry_timeout, int64_t request_timeout) -> bool; + // close a plugin shutting down its multiplexed ttrpc connections. + auto Close(void) -> bool; + // stop a plugin (if it was launched by us) + auto Stop(void) -> bool; + + // Name returns a string indentication for the plugin. + auto GetName(void) -> const std::string &; + auto GetIndex(void) -> const std::string &; + auto GetPeerSockFd(void) -> uint32_t; + auto GetQualifiedName(void) -> std::string; + + void SetReady(void); + void SetPid(int pid); + + auto IsClose(void) -> bool; + + auto CreateSocketPair(void) -> bool; + + auto Configure(Errors &error) -> bool; + // Only called in external plugin scenario + auto Synchronize(std::vector> pods, + std::vector> &containers, nri_container_update ***update, size_t update_len, + Errors &error) -> bool; + auto CreateContainer(nri_create_container_request *req, nri_create_container_response **resp, Errors &error) -> bool; + auto UpdateContainer(nri_update_container_request *req, nri_update_container_response **resp, Errors &error) -> bool; + auto StopContainer(nri_stop_container_request *req, nri_stop_container_response **resp, Errors &error) -> bool; + auto StateChange(nri_state_change_event *evt, Errors &error) -> bool; + +private: + auto Connect(int64_t timeout) -> bool; + + auto WaitForReady(int64_t timeout) -> bool; + auto IsSetEvent(EventMask e) -> bool; + +private: + RWMutex m_mutex; + bool m_external; + std::string m_idx; + std::string m_name; + std::string m_config; + int m_pid; + std::string m_cmd; + std::vector m_sockFds; + std::string m_localFileName; + std::string m_peerFileName; + // TODO:zhontao monitor? + bool m_closed; + std::mutex m_readyMutex; + bool m_ready; + std::condition_variable m_condition; + EventMask m_events; +}; + + +#endif // DAEMON_NRI_PLUGIN_PLUGIN_H \ No newline at end of file diff --git a/src/utils/cpputils/transform.cc b/src/utils/cpputils/transform.cc index 74c178a9..51c154fb 100644 --- a/src/utils/cpputils/transform.cc +++ b/src/utils/cpputils/transform.cc @@ -89,4 +89,20 @@ void CharArrayToStringVector(const char **src, size_t len, std::vector &ptrs) -> char ** +{ + size_t len = ptrs.size(); + char **result = (char **)util_smart_calloc_s(sizeof(char *), (len + 1)); + if (result == nullptr) { + return nullptr; + } + size_t i {}; + for (const auto &it : ptrs) { + result[i++] = util_strdup_s(it.c_str()); + } + + return result; +} + } // namespace Transform diff --git a/src/utils/cpputils/transform.h b/src/utils/cpputils/transform.h index 476f39a6..57c58d9e 100644 --- a/src/utils/cpputils/transform.h +++ b/src/utils/cpputils/transform.h @@ -35,6 +35,8 @@ auto StringVectorToCharArray(std::vector &strVec) -> char **; void CharArrayToStringVector(const char **src, size_t len, std::vector &dest); +auto RepeatedPtrFieldToCharArray(const google::protobuf::RepeatedPtrField &ptrs) -> char **; + }; // namespace Transform #endif // UTILS_CPPUTILS_TRANSFORM_H \ No newline at end of file diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c index 9a33f935..f81a9141 100644 --- a/src/utils/cutils/utils.c +++ b/src/utils/cutils/utils.c @@ -1699,3 +1699,10 @@ int util_chown_for_shm(const char *shm_path, const char *user_remap) return 0; } + +void set_child_process_pdeathsig(void) +{ + if (prctl(PR_SET_PDEATHSIG, SIGKILL) < 0) { + SYSERROR("Failed to set child process pdeathsig"); + } +} \ No newline at end of file diff --git a/src/utils/cutils/utils.h b/src/utils/cutils/utils.h index ce0ca703..76684ed3 100644 --- a/src/utils/cutils/utils.h +++ b/src/utils/cutils/utils.h @@ -409,6 +409,8 @@ int util_create_shm_path(const char *spath, const int64_t shm_size); int util_chown_for_shm(const char *shm_path, const char *user_remap); +void set_child_process_pdeathsig(void); + /** * retry_cnt: max count of call cb; * interval_us: how many us to sleep, after call cb; diff --git a/src/utils/tar/util_archive.c b/src/utils/tar/util_archive.c index 204dab83..985e0f16 100644 --- a/src/utils/tar/util_archive.c +++ b/src/utils/tar/util_archive.c @@ -812,13 +812,6 @@ static void close_archive_pipes_fd(int *pipes, size_t pipe_size) } } -static void set_child_process_pdeathsig(void) -{ - if (prctl(PR_SET_PDEATHSIG, SIGKILL) < 0) { - SYSERROR("Failed to set child process pdeathsig"); - } -} - int archive_unpack(const struct io_read_wrapper *content, const char *dstdir, const struct archive_options *options, const char *root_dir, char **errmsg) { -- 2.25.1