From 343be112a3d5cfd3857f166fb5dc2473534e64d7 Mon Sep 17 00:00:00 2001 From: zhongtao Date: Thu, 15 Aug 2024 15:19:25 +0800 Subject: [PATCH 121/121] [nri] impl for nri plugin and adaption Signed-off-by: zhongtao --- src/cmd/isulad/main.c | 15 + src/daemon/common/nri/nri_convert.cc | 148 ++- src/daemon/common/nri/nri_convert.h | 7 +- src/daemon/common/nri/nri_spec.c | 223 +++- src/daemon/common/nri/nri_spec.h | 6 + src/daemon/common/nri/nri_utils.c | 476 +++---- src/daemon/common/nri/nri_utils.h | 14 +- src/daemon/config/isulad_config.c | 2 +- .../v1/v1_cri_container_manager_service.cc | 104 +- .../v1/v1_cri_pod_sandbox_manager_service.cc | 37 +- .../executor/container_cb/execution_create.c | 21 + .../modules/service/service_container.c | 12 + src/daemon/modules/spec/specs.c | 7 +- src/daemon/nri/nri_adaption.cc | 1165 +++++++++++++++++ src/daemon/nri/nri_adaption.h | 13 +- src/daemon/nri/nri_helpers.cc | 21 + src/daemon/nri/nri_helpers.h | 4 + src/daemon/nri/nri_plugin_ops.cc | 123 ++ src/daemon/nri/nri_plugin_ops.h | 6 +- src/daemon/nri/nri_result.cc | 977 ++++++++++++++ src/daemon/nri/nri_result.h | 17 +- src/daemon/nri/plugin.cc | 417 ++++++ src/daemon/nri/plugin.h | 20 +- src/utils/cutils/utils.c | 2 +- 24 files changed, 3527 insertions(+), 310 deletions(-) create mode 100644 src/daemon/nri/nri_adaption.cc create mode 100644 src/daemon/nri/nri_plugin_ops.cc create mode 100644 src/daemon/nri/nri_result.cc create mode 100644 src/daemon/nri/plugin.cc diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c index 52ac3172..0228caa8 100644 --- a/src/cmd/isulad/main.c +++ b/src/cmd/isulad/main.c @@ -86,6 +86,9 @@ #ifdef ENABLE_CDI #include "cdi_operate_api.h" #endif /* ENABLE_CDI */ +#ifdef ENABLE_NRI +#include "nri_plugin_ops.h" +#endif sem_t g_daemon_shutdown_sem; sem_t g_daemon_wait_shutdown_sem; @@ -289,6 +292,11 @@ static void daemon_shutdown() EVENT("Network module exit completed"); #endif +#ifdef ENABLE_NRI + nri_adaption_shutdown(); + EVENT("nri module exit completed"); +#endif + clean_residual_files(); EVENT("Clean residual files completed"); @@ -1834,6 +1842,13 @@ int main(int argc, char **argv) } #endif +#ifdef ENABLE_NRI + if (!nri_adaption_init()) { + ERROR("Failed to init nri adaption"); + goto failure; + } +#endif + clock_gettime(CLOCK_MONOTONIC, &t_end); use_time = (double)(t_end.tv_sec - t_start.tv_sec) * (double)1000000000 + (double)(t_end.tv_nsec - t_start.tv_nsec); use_time /= 1000000000; diff --git a/src/daemon/common/nri/nri_convert.cc b/src/daemon/common/nri/nri_convert.cc index 7cce64ec..30caf1dd 100644 --- a/src/daemon/common/nri/nri_convert.cc +++ b/src/daemon/common/nri/nri_convert.cc @@ -20,6 +20,7 @@ #include "path.h" #include "transform.h" #include "nri_utils.h" +#include "cstruct_wrapper.h" static int64_t DefaultOOMScoreAdj = 0; @@ -142,22 +143,28 @@ static auto NRILinuxResourcesFromCRI(const runtime::v1::LinuxContainerResources static auto NRILinuxFromCRI(const runtime::v1::LinuxPodSandboxConfig &config, nri_linux_pod_sandbox &linux) -> bool { - if (!init_nri_linux_resources(&linux.pod_overhead)) { - ERROR("Failed to init nri linux overhead resources for pod"); - return false; - } - if (!init_nri_linux_resources(&linux.pod_resources)) { - ERROR("Failed to init nri linux resources resources for pod"); - return false; - } - if (config.has_overhead() && !NRILinuxResourcesFromCRI(config.overhead(), *linux.pod_overhead)) { - ERROR("Failed to transform overhead to nri for pod"); - return false; + if (config.has_overhead()) { + linux.pod_overhead = init_nri_linux_resources(); + if (linux.pod_overhead == nullptr) { + ERROR("Failed to init nri linux overhead resources for pod"); + return false; + } + if (!NRILinuxResourcesFromCRI(config.overhead(), *linux.pod_overhead)) { + ERROR("Failed to transform overhead to nri for pod"); + return false; + } } - if (config.has_resources() && !NRILinuxResourcesFromCRI(config.resources(), *linux.pod_resources)) { - ERROR("Failed to transform resources to nri for pod"); - return false; + if (config.has_resources()) { + linux.pod_resources = init_nri_linux_resources(); + if (linux.pod_resources == nullptr) { + ERROR("Failed to init nri linux resources resources for pod"); + return false; + } + if (!NRILinuxResourcesFromCRI(config.resources(), *linux.pod_resources)) { + ERROR("Failed to transform resources to nri for pod"); + return false; + } } linux.cgroup_parent = util_strdup_s(config.cgroup_parent().c_str()); @@ -507,13 +514,13 @@ auto LinuxResourcesFromNRI(const nri_linux_resources *src, runtime::v1::LinuxCon } if (src->cpu != nullptr) { - if (src->cpu->shares != NULL) { + if (src->cpu->shares != nullptr) { resources.set_cpu_shares(*src->cpu->shares); } - if (src->cpu->quota != NULL) { + if (src->cpu->quota != nullptr) { resources.set_cpu_quota(*src->cpu->quota); } - if (src->cpu->period != NULL) { + if (src->cpu->period != nullptr) { resources.set_cpu_period(*src->cpu->period); } @@ -535,5 +542,112 @@ auto LinuxResourcesFromNRI(const nri_linux_resources *src, runtime::v1::LinuxCon Transform::JsonMapToProtobufMapForString(src->unified, *resources.mutable_unified()); } + return true; +} + +auto LinuxResourcesToNRI(const runtime::v1::LinuxContainerResources &src) -> nri_linux_resources * +{ + nri_linux_resources *resources = nullptr; + + resources = init_nri_linux_resources(); + if (resources == nullptr) { + ERROR("Failed to init nri linux resources"); + return nullptr; + } + + resources->cpu->shares = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); + if (resources->cpu->shares == nullptr) { + ERROR("Out of memory"); + goto error_out; + } + *(resources->cpu->shares) = src.cpu_shares(); + + resources->cpu->quota = (int64_t *)util_common_calloc_s(sizeof(int64_t)); + if (resources->cpu->quota == nullptr) { + ERROR("Out of memory"); + goto error_out; + } + *(resources->cpu->quota) = src.cpu_quota(); + + resources->cpu->period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); + if (resources->cpu->period == nullptr) { + ERROR("Out of memory"); + goto error_out; + } + *(resources->cpu->period) = src.cpu_period(); + + resources->cpu->cpus = util_strdup_s(src.cpuset_cpus().c_str()); + resources->cpu->mems = util_strdup_s(src.cpuset_mems().c_str()); + + resources->memory->limit = (int64_t *)util_common_calloc_s(sizeof(int64_t)); + if (resources->memory->limit == nullptr) { + ERROR("Out of memory"); + goto error_out; + } + *(resources->memory->limit) = src.memory_limit_in_bytes(); + + resources->hugepage_limits = (nri_hugepage_limit **)util_smart_calloc_s(sizeof(nri_hugepage_limit *), + src.hugepage_limits_size()); + if (resources->hugepage_limits == nullptr) { + ERROR("Out of memory"); + goto error_out; + } + + for (int i = 0; i < src.hugepage_limits_size(); i++) { + resources->hugepage_limits[i] = (nri_hugepage_limit *)util_common_calloc_s(sizeof(nri_hugepage_limit)); + if (resources->hugepage_limits[i] == nullptr) { + ERROR("Out of memory"); + goto error_out; + } + resources->hugepage_limits[i]->page_size = util_strdup_s(src.hugepage_limits(i).page_size().c_str()); + resources->hugepage_limits[i]->limit = src.hugepage_limits(i).limit(); + resources->hugepage_limits_len++; + } + + return resources; + +error_out: + free_nri_linux_resources(resources); + resources = nullptr; + return resources; +} + +auto PodSandboxesToNRI(const std::vector> &arrs, + std::vector &pods) -> bool +{ + size_t i = 0; + for (i = 0; i < arrs.size(); i++) { + nri_pod_sandbox *pod = (nri_pod_sandbox *)util_common_calloc_s(sizeof(nri_pod_sandbox)); + if (pod == nullptr) { + ERROR("Out of memory"); + return false; + } + if (!PodSandboxToNRI(arrs[i], *pod)) { + ERROR("Failed to transform pod to nri for pod : %s", arrs[i]->GetName().c_str()); + return false; + } + pods.push_back(pod); + } + + return true; +} + +auto ContainersToNRI(std::vector> &containers, + std::vector &cons) -> bool +{ + size_t i = 0; + for (i = 0; i < containers.size(); i++) { + nri_container *con = (nri_container *)util_common_calloc_s(sizeof(nri_container)); + if (con == nullptr) { + ERROR("Out of memory"); + return false; + } + if (!ContainerToNRIByID(containers[i].get()->id(), *con)) { + ERROR("Failed to transform container to nri for container : %s", containers[i]->metadata().name().c_str()); + return false; + } + cons.push_back(con); + } + return true; } \ No newline at end of file diff --git a/src/daemon/common/nri/nri_convert.h b/src/daemon/common/nri/nri_convert.h index c04b14e4..a0caf6ce 100644 --- a/src/daemon/common/nri/nri_convert.h +++ b/src/daemon/common/nri/nri_convert.h @@ -30,8 +30,11 @@ auto PodSandboxToNRI(const std::shared_ptr &sandbox, nri_pod_sandbox &pod) -> bool; auto ContainerToNRIByConConfig(const runtime::v1::ContainerConfig &containerConfig, nri_container &con) -> bool; auto ContainerToNRIByID(const std::string &id, nri_container &con) -> bool; -auto PodSandboxesToNRI(const std::vector> &arrs, nri_pod_sandbox **pod, - int pod_len) -> bool; +auto PodSandboxesToNRI(const std::vector> &arrs, + std::vector &pods) -> bool; +auto ContainersToNRI(std::vector> &containers, + std::vector &cons) -> bool; auto LinuxResourcesFromNRI(const nri_linux_resources *src, runtime::v1::LinuxContainerResources &resources) -> bool; +auto LinuxResourcesToNRI(const runtime::v1::LinuxContainerResources &src) -> nri_linux_resources *; #endif // DAEMON_COMMON_NRI_NRI_CONVERT_H diff --git a/src/daemon/common/nri/nri_spec.c b/src/daemon/common/nri/nri_spec.c index 855fe3b3..327d31e1 100644 --- a/src/daemon/common/nri/nri_spec.c +++ b/src/daemon/common/nri/nri_spec.c @@ -176,6 +176,7 @@ static int nri_adjust_annotation(const nri_container_adjustment *adjust, oci_run free_json_map_string_string(oci_spec->annotations); oci_spec->annotations = cleard; + cleard = NULL; ret = 0; free_out: @@ -330,7 +331,34 @@ static int nri_adjust_hooks(const nri_container_adjustment *adjust, oci_runtime_ return ret; } -static int nri_adjust_devices(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) +static int host_spec_add_device(host_config *spec, defs_device *device) +{ + size_t i; + + if (device == NULL) { + return -1; + } + + for (i = 0; i < spec->nri_devices_len; i++) { + if (strcmp(spec->nri_devices[i]->path, device->path) == 0) { + free_defs_device(spec->nri_devices[i]); + spec->nri_devices[i] = device; + return 0; + } + } + + if (util_mem_realloc((void **)&spec->nri_devices, (spec->nri_devices_len + 1) * sizeof(defs_device *), + (void *)spec->nri_devices, spec->nri_devices_len * sizeof(defs_device *)) != 0) { + ERROR("Out of memory"); + return -1; + } + spec->nri_devices[spec->nri_devices_len] = device; + spec->nri_devices_len++; + + return 0; +} + +static int nri_adjust_oci_spec_devices(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) { if (adjust->linux == NULL || adjust->linux->devices == NULL || adjust->linux->devices_len == 0) { return 0; @@ -349,6 +377,25 @@ static int nri_adjust_devices(const nri_container_adjustment *adjust, oci_runtim return 0; } +static int nri_adjust_host_spec_devices(const nri_container_adjustment *adjust, host_config *spec) +{ + if (adjust->linux == NULL || adjust->linux->devices == NULL || adjust->linux->devices_len == 0) { + return 0; + } + + size_t i; + + for (i = 0; i < adjust->linux->devices_len; i++) { + nri_linux_device *dev = adjust->linux->devices[i]; + if (host_spec_add_device(spec, nri_device_to_oci(dev)) != 0) { + ERROR("Failed to add device %s", dev->path); + return -1; + } + } + + return 0; +} + static int nri_adjust_cgroup_path(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) { if (adjust->linux == NULL || adjust->linux->cgroups_path == NULL) { @@ -500,7 +547,50 @@ static int nri_adjust_mounts(const nri_container_adjustment *adjust, oci_runtime return 0; } -static int nri_adjust_rlimit(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) +static int host_spec_add_linux_resources_rlimit(host_config *spec, const char *type, uint64_t hard, uint64_t soft) +{ + size_t j; + bool exists = false; + host_config_nri_rlimits_element *rlimit = NULL; + + if (spec == NULL || type == NULL) { + ERROR("Invalid arguments"); + return -1; + } + + rlimit = util_common_calloc_s(sizeof(host_config_nri_rlimits_element)); + if (rlimit == NULL) { + ERROR("Out of memory"); + return -1; + } + rlimit->type = util_strdup_s(type); + rlimit->hard = hard; + rlimit->soft = soft; + + for (j = 0; j < spec->nri_rlimits_len; j++) { + if (spec->nri_rlimits[j]->type == NULL) { + ERROR("rlimit type is empty"); + free_host_config_nri_rlimits_element(rlimit); + return -1; + } + if (strcmp(spec->nri_rlimits[j]->type, rlimit->type) == 0) { + exists = true; + break; + } + } + if (exists) { + /* override ulimit */ + free_host_config_nri_rlimits_element(spec->nri_rlimits[j]); + spec->nri_rlimits[j] = rlimit; + } else { + spec->nri_rlimits[spec->nri_rlimits_len] = rlimit; + spec->nri_rlimits_len++; + } + + return 0; +} + +static int nri_adjust_oci_spec_rlimit(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) { if (adjust->rlimits == NULL || adjust->rlimits_len == 0) { return 0; @@ -522,6 +612,36 @@ static int nri_adjust_rlimit(const nri_container_adjustment *adjust, oci_runtime return 0; } +static int nri_adjust_host_spec_rlimit(const nri_container_adjustment *adjust, host_config *spec) +{ + if (adjust->rlimits == NULL || adjust->rlimits_len == 0) { + return 0; + } + + if (spec->nri_rlimits == NULL) { + spec->nri_rlimits = util_common_calloc_s(sizeof(host_config_nri_rlimits_element *) * adjust->rlimits_len); + if (spec->nri_rlimits == NULL) { + ERROR("Out of memory"); + return -1; + } + } + + size_t i; + for (i = 0; i < adjust->rlimits_len; i++) { + nri_posix_rlimit *rlimit = adjust->rlimits[i]; + if (rlimit->type == NULL) { + ERROR("Invalid rlimit type"); + return -1; + } + if (host_spec_add_linux_resources_rlimit(spec, rlimit->type, rlimit->soft, rlimit->hard) != 0) { + ERROR("Failed to add rlimit"); + return -1; + } + } + + return 0; +} + // todo: we do not support it blockio_class static int nri_adjust_blockio_class(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) { @@ -554,7 +674,7 @@ int nri_adjust_oci_spec(const nri_container_adjustment *adjust, oci_runtime_spec return -1; } - if (nri_adjust_devices(adjust, oci_spec) != 0) { + if (nri_adjust_oci_spec_devices(adjust, oci_spec) != 0) { ERROR("Failed to do nri adjust devices in oci spec"); return -1; } @@ -580,7 +700,7 @@ int nri_adjust_oci_spec(const nri_container_adjustment *adjust, oci_runtime_spec return -1; } - if (nri_adjust_rlimit(adjust, oci_spec) != 0) { + if (nri_adjust_oci_spec_rlimit(adjust, oci_spec) != 0) { ERROR("Failed to do nri adjust rlimit in oci spec"); return -1; } @@ -598,5 +718,100 @@ int nri_adjust_oci_spec(const nri_container_adjustment *adjust, oci_runtime_spec return -1; } + return 0; +} + +int nri_adjust_host_spec(const nri_container_adjustment *adjust, host_config *host_spec) +{ + if (adjust == NULL || host_spec == NULL) { + ERROR("Invalid input arguments"); + return -1; + } + + if (nri_adjust_host_spec_devices(adjust, host_spec) != 0) { + ERROR("Failed to do nri adjust devices in host config"); + return -1; + } + + if (nri_adjust_host_spec_rlimit(adjust, host_spec) != 0) { + ERROR("Failed to do nri adjust rlimit in host config"); + return -1; + } + + return 0; +} + +static defs_device *copy_def_devices(const defs_device *dev) +{ + defs_device *tmp_dev = util_common_calloc_s(sizeof(defs_device)); + if (tmp_dev == NULL) { + return NULL; + } + tmp_dev->type = util_strdup_s(dev->type); + tmp_dev->path = util_strdup_s(dev->path); + tmp_dev->file_mode = dev->file_mode; + tmp_dev->major = dev->major; + tmp_dev->minor = dev->minor; + tmp_dev->uid = dev->uid; + tmp_dev->gid = dev->gid; + return tmp_dev; +} + +static int merge_nri_devices(oci_runtime_spec *oci_spec, host_config *host_spec) +{ + size_t i; + + if (oci_spec == NULL || host_spec == NULL) { + ERROR("Invalid input arguments"); + return -1; + } + + for (i = 0; i < host_spec->nri_devices_len; i++) { + if (spec_add_device(oci_spec, copy_def_devices(host_spec->nri_devices[i])) != 0) { + ERROR("Failed to add device %s", host_spec->nri_devices[i]->path); + return -1; + } + } + + return 0; +} + +static int merge_nri_ulimits(oci_runtime_spec *oci_spec, host_config *host_spec) +{ + size_t i; + + if (oci_spec == NULL || host_spec == NULL) { + ERROR("Invalid input arguments"); + return -1; + } + + for (i = 0; i < host_spec->nri_rlimits_len; i++) { + host_config_nri_rlimits_element *rlimit = host_spec->nri_rlimits[i]; + if (spec_add_linux_resources_rlimit(oci_spec, rlimit->type, rlimit->hard, rlimit->soft) != 0) { + ERROR("Failed to add rlimit %s", rlimit->type); + return -1; + } + } + + return 0; +} + +int update_oci_nri(oci_runtime_spec *oci_spec, host_config *host_spec) +{ + if (oci_spec == NULL || host_spec == NULL) { + ERROR("Invalid input arguments"); + return -1; + } + + if (merge_nri_devices(oci_spec, host_spec) != 0) { + ERROR("Failed to merge nri devices"); + return -1; + } + + if (merge_nri_ulimits(oci_spec, host_spec) != 0) { + ERROR("Failed to merge nri ulimits"); + return -1; + } + return 0; } \ No newline at end of file diff --git a/src/daemon/common/nri/nri_spec.h b/src/daemon/common/nri/nri_spec.h index e7c5035d..1a622284 100644 --- a/src/daemon/common/nri/nri_spec.h +++ b/src/daemon/common/nri/nri_spec.h @@ -18,7 +18,13 @@ #include #include +#include int nri_adjust_oci_spec(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec); +// the device and ulimit will be updated when starting, so they need to be stored in host-spec +int nri_adjust_host_spec(const nri_container_adjustment *adjust, host_config *host_spec); + +int update_oci_nri(oci_runtime_spec *oci_spec, host_config *host_spec); + #endif // DAEMON_COMMON_NRI_NRI_SPEC_H \ No newline at end of file diff --git a/src/daemon/common/nri/nri_utils.c b/src/daemon/common/nri/nri_utils.c index 51054e32..d2b62117 100644 --- a/src/daemon/common/nri/nri_utils.c +++ b/src/daemon/common/nri/nri_utils.c @@ -19,218 +19,235 @@ #include "utils.h" -static bool copy_nri_hugepage_limit(const nri_hugepage_limit* src, nri_hugepage_limit** dest) +static nri_hugepage_limit*copy_nri_hugepage_limit(const nri_hugepage_limit *src) { - if (src == NULL || dest == NULL) { + nri_hugepage_limit *dest = NULL; + if (src == NULL) { ERROR("Invalid input arguments"); return false; } - *dest = (nri_hugepage_limit *)util_common_calloc_s(sizeof(nri_hugepage_limit)); - if (*dest == NULL) { + dest = (nri_hugepage_limit *)util_common_calloc_s(sizeof(nri_hugepage_limit)); + if (dest == NULL) { ERROR("Out of memory"); return false; } - (*dest)->limit = src->limit; - (*dest)->page_size = util_strdup_s(src->page_size); - return true; + dest->limit = src->limit; + dest->page_size = util_strdup_s(src->page_size); + return dest; } -static bool copy_nri_hook(const nri_hook *src, nri_hook **dest) +static nri_hook *copy_nri_hook(const nri_hook *src) { - if (src == NULL || dest == NULL) { + nri_hook *dest = NULL; + if (src == NULL) { ERROR("Invalid input arguments"); return false; } - *dest = (nri_hook *)util_common_calloc_s(sizeof(nri_hook)); + dest = (nri_hook *)util_common_calloc_s(sizeof(nri_hook)); if (dest == NULL) { ERROR("Out of memory"); return false; } - (*dest)->args = util_copy_array_by_len(src->args, src->args_len); - (*dest)->args_len = src->args_len; - (*dest)->env = util_copy_array_by_len(src->env, src->env_len); - (*dest)->env_len = src->env_len; - (*dest)->path = util_strdup_s(src->path); - return true; + dest->args = util_copy_array_by_len(src->args, src->args_len); + dest->args_len = src->args_len; + dest->env = util_copy_array_by_len(src->env, src->env_len); + dest->env_len = src->env_len; + dest->path = util_strdup_s(src->path); + return dest; } -static bool copy_nri_linux_device_cgroup(const nri_linux_device_cgroup *src, nri_linux_device_cgroup **dest) +static nri_linux_device_cgroup *copy_nri_linux_device_cgroup(const nri_linux_device_cgroup *src) { - if (src == NULL || dest == NULL) { + nri_linux_device_cgroup *dest = NULL; + if (src == NULL) { ERROR("Invalid input arguments"); return false; } - *dest = (nri_linux_device_cgroup *)util_common_calloc_s(sizeof(nri_linux_device_cgroup)); + dest = (nri_linux_device_cgroup *)util_common_calloc_s(sizeof(nri_linux_device_cgroup)); if (dest == NULL) { ERROR("Out of memory"); return false; } - (*dest)->allow = src->allow; - (*dest)->type = util_strdup_s(src->type); - (*dest)->major = (int64_t *)util_common_calloc_s(sizeof(int64_t)); - if ((*dest)->major == NULL) { + dest->allow = src->allow; + dest->type = util_strdup_s(src->type); + dest->major = (int64_t *)util_common_calloc_s(sizeof(int64_t)); + if (dest->major == NULL) { ERROR("Out of memory"); - return false; + goto free_out; } - (*dest)->minor = (int64_t *)util_common_calloc_s(sizeof(int64_t)); - if ((*dest)->minor == NULL) { + dest->minor = (int64_t *)util_common_calloc_s(sizeof(int64_t)); + if (dest->minor == NULL) { ERROR("Out of memory"); - return false; + goto free_out; } - (*dest)->access = util_strdup_s(src->access); - return true; + dest->access = util_strdup_s(src->access); + return dest; +free_out: + free_nri_linux_device_cgroup(dest); + return NULL; } -static bool copy_nri_linux_cpu(const nri_linux_cpu *src, nri_linux_cpu **dest) +static nri_linux_cpu *copy_nri_linux_cpu(const nri_linux_cpu *src) { - if (src == NULL || dest == NULL) { + nri_linux_cpu *dest = NULL; + if (src == NULL) { ERROR("Invalid input arguments"); return false; } - (*dest) = (nri_linux_cpu *)util_common_calloc_s(sizeof(nri_linux_cpu)); - if ((*dest) == NULL) { + dest = (nri_linux_cpu *)util_common_calloc_s(sizeof(nri_linux_cpu)); + if (dest == NULL) { ERROR("Out of memory"); return false; } - (*dest)->cpus = util_strdup_s(src->cpus); - (*dest)->mems = util_strdup_s(src->mems); + dest->cpus = util_strdup_s(src->cpus); + dest->mems = util_strdup_s(src->mems); if (src->period != NULL) { - (*dest)->period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); - if ((*dest)->period == NULL) { + dest->period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); + if (dest->period == NULL) { ERROR("Out of memory"); - return false; + goto free_out; } - *(*dest)->period = *src->period; + *dest->period = *src->period; } if (src->quota != NULL) { - (*dest)->quota = (int64_t *)util_common_calloc_s(sizeof(int64_t)); - if ((*dest)->quota == NULL) { + dest->quota = (int64_t *)util_common_calloc_s(sizeof(int64_t)); + if (dest->quota == NULL) { ERROR("Out of memory"); - return false; + goto free_out; } - *(*dest)->quota = *src->quota; + *dest->quota = *src->quota; } if (src->realtime_period != NULL) { - (*dest)->realtime_period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); - if ((*dest)->realtime_period == NULL) { + dest->realtime_period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); + if (dest->realtime_period == NULL) { ERROR("Out of memory"); - return false; + goto free_out; } - *(*dest)->realtime_period = *src->realtime_period; + *dest->realtime_period = *src->realtime_period; } if (src->realtime_runtime != NULL) { - (*dest)->realtime_runtime = (int64_t *)util_common_calloc_s(sizeof(int64_t)); - if ((*dest)->realtime_runtime == NULL) { + dest->realtime_runtime = (int64_t *)util_common_calloc_s(sizeof(int64_t)); + if (dest->realtime_runtime == NULL) { ERROR("Out of memory"); - return false; + goto free_out; } - *(*dest)->realtime_runtime = *src->realtime_runtime; + *dest->realtime_runtime = *src->realtime_runtime; } if (src->shares != NULL) { - (*dest)->shares = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); - if ((*dest)->shares == NULL) { + dest->shares = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); + if (dest->shares == NULL) { ERROR("Out of memory"); - return false; + goto free_out; } - *(*dest)->shares = *src->shares; + *dest->shares = *src->shares; } - return true; + return dest; + +free_out: + free_nri_linux_cpu(dest); + return NULL; } -static bool copy_nri_linux_memory(const nri_linux_memory *src, nri_linux_memory **dest) +static nri_linux_memory *copy_nri_linux_memory(const nri_linux_memory *src) { - if (src == NULL || dest == NULL) { + nri_linux_memory *dest = NULL; + if (src == NULL) { ERROR("Invalid input arguments"); return false; } - *dest = (nri_linux_memory *)util_common_calloc_s(sizeof(nri_linux_memory)); + + dest = (nri_linux_memory *)util_common_calloc_s(sizeof(nri_linux_memory)); if (dest == NULL) { ERROR("Out of memory"); return false; } if (src->limit != NULL) { - (*dest)->limit = (int64_t *)util_common_calloc_s(sizeof(int64_t)); - if ((*dest)->limit == NULL) { + dest->limit = (int64_t *)util_common_calloc_s(sizeof(int64_t)); + if (dest->limit == NULL) { ERROR("Out of memory"); - return false; + goto free_out; } - *(*dest)->limit = *src->limit; + *dest->limit = *src->limit; } if (src->reservation != NULL) { - (*dest)->reservation = (int64_t *)util_common_calloc_s(sizeof(int64_t)); - if ((*dest)->reservation == NULL) { + dest->reservation = (int64_t *)util_common_calloc_s(sizeof(int64_t)); + if (dest->reservation == NULL) { ERROR("Out of memory"); - return false; + goto free_out; } - *(*dest)->reservation = *src->reservation; + *dest->reservation = *src->reservation; } if (src->swap != NULL) { - (*dest)->swap = (int64_t *)util_common_calloc_s(sizeof(int64_t)); - if ((*dest)->swap == NULL) { + dest->swap = (int64_t *)util_common_calloc_s(sizeof(int64_t)); + if (dest->swap == NULL) { ERROR("Out of memory"); - return false; + goto free_out; } - *(*dest)->swap = *src->swap; + *dest->swap = *src->swap; } if (src->kernel != NULL) { - (*dest)->kernel = (int64_t *)util_common_calloc_s(sizeof(int64_t)); - if ((*dest)->kernel == NULL) { + dest->kernel = (int64_t *)util_common_calloc_s(sizeof(int64_t)); + if (dest->kernel == NULL) { ERROR("Out of memory"); - return false; + goto free_out; } - *(*dest)->kernel = *src->kernel; + *dest->kernel = *src->kernel; } if (src->kernel_tcp != NULL) { - (*dest)->kernel_tcp = (int64_t *)util_common_calloc_s(sizeof(int64_t)); - if ((*dest)->kernel_tcp == NULL) { + dest->kernel_tcp = (int64_t *)util_common_calloc_s(sizeof(int64_t)); + if (dest->kernel_tcp == NULL) { ERROR("Out of memory"); - return false; + goto free_out; } - *(*dest)->kernel_tcp = *src->kernel_tcp; + *dest->kernel_tcp = *src->kernel_tcp; } if (src->swappiness != NULL) { - (*dest)->swappiness = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); - if ((*dest)->swappiness == NULL) { + dest->swappiness = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); + if (dest->swappiness == NULL) { ERROR("Out of memory"); - return false; + goto free_out; } - *(*dest)->swappiness = *src->swappiness; + *dest->swappiness = *src->swappiness; } if (src->disable_oom_killer != NULL) { - (*dest)->disable_oom_killer = (uint8_t *)util_common_calloc_s(sizeof(uint8_t)); - if ((*dest)->disable_oom_killer == NULL) { + dest->disable_oom_killer = (uint8_t *)util_common_calloc_s(sizeof(uint8_t)); + if (dest->disable_oom_killer == NULL) { ERROR("Out of memory"); - return false; + goto free_out; } - *(*dest)->disable_oom_killer = *src->disable_oom_killer; + *dest->disable_oom_killer = *src->disable_oom_killer; } if (src->use_hierarchy != NULL) { - (*dest)->use_hierarchy = (uint8_t *)util_common_calloc_s(sizeof(uint8_t)); - if ((*dest)->use_hierarchy == NULL) { + dest->use_hierarchy = (uint8_t *)util_common_calloc_s(sizeof(uint8_t)); + if (dest->use_hierarchy == NULL) { ERROR("Out of memory"); - return false; + goto free_out; } - *(*dest)->use_hierarchy = *src->use_hierarchy; + *dest->use_hierarchy = *src->use_hierarchy; } - return true; + return dest; + +free_out: + free_nri_linux_memory(dest); + return NULL; } bool is_marked_for_removal(const char* key, char **out) @@ -241,7 +258,7 @@ bool is_marked_for_removal(const char* key, char **out) } if (!util_has_prefix(key, "-")) { - *out = (char*)key; + *out = util_strdup_s(key); return false; } @@ -254,94 +271,153 @@ bool is_marked_for_removal(const char* key, char **out) return true; } -bool copy_nri_mount(const nri_mount *src, nri_mount **dest) +nri_mount *copy_nri_mount(const nri_mount *src) { - if (src == NULL || dest == NULL) { + nri_mount *dest = NULL; + if (src == NULL) { ERROR("Invalid input arguments"); return false; } - *dest = (nri_mount *)util_common_calloc_s(sizeof(nri_mount)); + + dest = (nri_mount *)util_common_calloc_s(sizeof(nri_mount)); if (dest == NULL) { ERROR("Out of memory"); return false; } - (*dest)->destination = util_strdup_s(src->destination); - (*dest)->options = util_copy_array_by_len(src->options, src->options_len); - (*dest)->options_len = src->options_len; - (*dest)->source = util_strdup_s(src->source); - (*dest)->type = util_strdup_s(src->type); - return true; + dest->destination = util_strdup_s(src->destination); + dest->options = util_copy_array_by_len(src->options, src->options_len); + dest->options_len = src->options_len; + dest->source = util_strdup_s(src->source); + dest->type = util_strdup_s(src->type); + return dest; } -bool copy_nri_key_value(const nri_key_value *src, nri_key_value **dest) +nri_linux_device *copy_nri_device(const nri_linux_device *src) { - if (src == NULL || dest == NULL) { + nri_linux_device *dest = NULL; + + if (src == NULL) { ERROR("Invalid input arguments"); return false; } - *dest = (nri_key_value *)util_common_calloc_s(sizeof(nri_key_value)); + + dest = (nri_linux_device *)util_common_calloc_s(sizeof(nri_linux_device)); if (dest == NULL) { ERROR("Out of memory"); return false; } - (*dest)->key = util_strdup_s(src->key); - (*dest)->value = util_strdup_s(src->value); - return true; + if (src->file_mode != NULL) { + dest->file_mode = (uint32_t *)util_common_calloc_s(sizeof(uint32_t)); + if (dest->file_mode == NULL) { + ERROR("Out of memory"); + goto free_out; + } + *dest->file_mode = *src->file_mode; + } + + if (src->uid != NULL) { + dest->uid = (uint32_t *)util_common_calloc_s(sizeof(uint32_t)); + if (dest->uid == NULL) { + ERROR("Out of memory"); + goto free_out; + } + *dest->uid = *src->uid; + } + + if (src->gid != NULL) { + dest->gid = (uint32_t *)util_common_calloc_s(sizeof(uint32_t)); + if (dest->gid == NULL) { + ERROR("Out of memory"); + goto free_out; + } + *dest->gid = *src->gid; + } + + dest->major = src->major; + dest->minor = src->minor; + dest->path = util_strdup_s(src->path); + dest->type = util_strdup_s(src->type); + + return dest; +free_out: + free_nri_linux_device(dest); + return NULL; } -bool copy_nri_posix_rlimit(const nri_posix_rlimit *src, nri_posix_rlimit **dest) +nri_key_value *copy_nri_key_value(const nri_key_value *src) { - if (src == NULL || dest == NULL) { + nri_key_value *dest = NULL; + if (src == NULL) { ERROR("Invalid input arguments"); return false; } - *dest = (nri_posix_rlimit *)util_common_calloc_s(sizeof(nri_posix_rlimit)); + dest = (nri_key_value *)util_common_calloc_s(sizeof(nri_key_value)); if (dest == NULL) { ERROR("Out of memory"); return false; } - (*dest)->hard = src->hard; - (*dest)->soft = src->soft; - (*dest)->type = util_strdup_s(src->type); - return true; + dest->key = util_strdup_s(src->key); + dest->value = util_strdup_s(src->value); + return dest; } -bool copy_nri_linux_resources(const nri_linux_resources *src, nri_linux_resources **dest) +nri_posix_rlimit *copy_nri_posix_rlimit(const nri_posix_rlimit *src) { - if (src == NULL || dest == NULL) { + nri_posix_rlimit *dest = NULL; + if (src == NULL) { ERROR("Invalid input arguments"); return false; } - - *dest = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); - if (*dest == NULL) { + dest = (nri_posix_rlimit *)util_common_calloc_s(sizeof(nri_posix_rlimit)); + if (dest == NULL) { ERROR("Out of memory"); return false; } + dest->hard = src->hard; + dest->soft = src->soft; + dest->type = util_strdup_s(src->type); + return dest; +} - if (!init_nri_linux_resources(dest)) { - ERROR("Failed to init dest nri linux resources"); - goto free_out; +nri_linux_resources *copy_nri_linux_resources(const nri_linux_resources *src) +{ + nri_linux_resources *dest = NULL; + if (src == NULL) { + ERROR("Invalid input arguments"); + return false; } - if (!copy_nri_linux_cpu(src->cpu, &(*dest)->cpu)) { - ERROR("Failed to copy nri_linux_cpu"); - goto free_out; + dest = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); + if (dest == NULL) { + ERROR("Out of memory"); + return false; } - if (!copy_nri_linux_memory(src->memory, &(*dest)->memory)) { - ERROR("Failed to copy nri_linux_memory"); - goto free_out; + if (src->cpu != NULL) { + dest->cpu = copy_nri_linux_cpu(src->cpu); + if (dest->cpu == NULL) { + ERROR("Failed to copy nri_linux_cpu"); + goto free_out; + } } - (*dest)->blockio_class = util_strdup_s(src->blockio_class); - (*dest)->rdt_class = util_strdup_s(src->rdt_class); + if (src->memory != NULL) { + dest->memory = copy_nri_linux_memory(src->memory); + if (dest->memory == NULL) { + ERROR("Failed to copy nri_linux_memory"); + goto free_out; + } + } + + dest->blockio_class = util_strdup_s(src->blockio_class); + dest->rdt_class = util_strdup_s(src->rdt_class); if (src->hugepage_limits_len > 0) { - (*dest)->hugepage_limits = (nri_hugepage_limit**)util_smart_calloc_s(sizeof(nri_hugepage_limit*), + dest->hugepage_limits = (nri_hugepage_limit**)util_smart_calloc_s(sizeof(nri_hugepage_limit*), src->hugepage_limits_len); - for (size_t i = 0; i < src->hugepage_limits_len; ++i) { - if (!copy_nri_hugepage_limit(src->hugepage_limits[i], &((*dest)->hugepage_limits[i]))) { + for (size_t i = 0; i < src->hugepage_limits_len; i++) { + dest->hugepage_limits[i] = copy_nri_hugepage_limit(src->hugepage_limits[i]); + if (dest->hugepage_limits[i] == NULL) { ERROR("Failed to copy nri_hugepage_limit"); goto free_out; } @@ -349,25 +425,26 @@ bool copy_nri_linux_resources(const nri_linux_resources *src, nri_linux_resource } if (src->devices_len > 0) { - (*dest)->devices = (nri_linux_device_cgroup**)util_smart_calloc_s(sizeof(nri_linux_device_cgroup*), src->devices_len); - for (size_t i = 0; i < src->devices_len; ++i) { - if (!copy_nri_linux_device_cgroup(src->devices[i], &((*dest)->devices[i]))) { + dest->devices = (nri_linux_device_cgroup**)util_smart_calloc_s(sizeof(nri_linux_device_cgroup*), src->devices_len); + for (size_t i = 0; i < src->devices_len; i++) { + dest->devices[i] = copy_nri_linux_device_cgroup(src->devices[i]); + if (dest->devices[i] == NULL) { ERROR("Failed to copy nri_linux_device_cgroup"); goto free_out; } } } - if (dup_json_map_string_string(src->unified, (*dest)->unified)) { + if (dup_json_map_string_string(src->unified, dest->unified)) { ERROR("Failed to copy json_map_string_string"); goto free_out; } - return true; + return dest; free_out: - free_nri_linux_resources(*dest); - return false; + free_nri_linux_resources(dest); + return NULL; } bool merge_nri_hooks(nri_hook **targetHooks, size_t targetSize, const nri_hook **sourceHooks, @@ -381,140 +458,67 @@ bool merge_nri_hooks(nri_hook **targetHooks, size_t targetSize, const nri_hook * return false; } - if (util_mem_realloc((void**)&targetHooks, newSize, (void**)&targetHooks, oldSize) != 0) { + if (util_mem_realloc((void**)&targetHooks, newSize, targetHooks, oldSize) != 0) { ERROR("Failed to realloc and assign hook array"); return false; } for (size_t i = 0; i < sourceLen; i++) { - if (!copy_nri_hook(sourceHooks[i], &targetHooks[targetSize++])) { + targetHooks[targetSize] = copy_nri_hook(sourceHooks[i]); + if (targetHooks[targetSize] == NULL) { ERROR("Failed to copy hook"); return false; } + targetSize++; } return true; } -bool init_nri_container_adjust(nri_container_adjustment **adjust) +nri_container_update *init_nri_container_update(const char *id, const uint8_t ignore_failure) { - if (adjust == NULL) { + nri_container_update *update = NULL; + if (id == NULL) { ERROR("Invalid input arguments"); return false; } - *adjust = (nri_container_adjustment *)util_common_calloc_s(sizeof(nri_container_adjustment)); - if (*adjust == NULL) { + update = (nri_container_update *)util_common_calloc_s(sizeof(nri_container_update)); + if (update == NULL) { ERROR("Out of memory"); return false; } - (*adjust)->annotations = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); - if ((*adjust)->annotations == NULL) { - goto free_out; - } - - (*adjust)->env = (nri_key_value **)util_common_calloc_s(sizeof(nri_key_value *)); - if ((*adjust)->env == NULL) { - goto free_out; - } - (*adjust)->env_len = 0; - - (*adjust)->hooks = (nri_hooks *)util_common_calloc_s(sizeof(nri_hooks)); - if ((*adjust)->hooks == NULL) { - goto free_out; - } - - (*adjust)->linux = (nri_linux_container_adjustment *)util_common_calloc_s(sizeof(nri_linux_container_adjustment)); - if ((*adjust)->linux == NULL) { - goto free_out; - } - - (*adjust)->linux->resources = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); - if ((*adjust)->linux->resources == NULL) { - ERROR("Out of memory"); - return false; - } - - (*adjust)->mounts = (nri_mount **)util_common_calloc_s(sizeof(nri_mount *)); - if ((*adjust)->mounts == NULL) { - goto free_out; - } - (*adjust)->mounts_len = 0; - - (*adjust)->rlimits = (nri_posix_rlimit **)util_common_calloc_s(sizeof(nri_posix_rlimit *)); - if ((*adjust)->rlimits == NULL) { - goto free_out; - } - (*adjust)->rlimits_len = 0; + update->container_id = util_strdup_s(id); - return true; - -free_out: - ERROR("Out of memory"); - free_nri_container_adjustment(*adjust); - return false; + update->ignore_failure = ignore_failure; + return update; } -bool init_nri_container_update(nri_container_update **update, const char *id, uint8_t ignore_failure) +nri_linux_resources *init_nri_linux_resources() { - if (update == NULL || id == NULL) { - ERROR("Invalid input arguments"); - return false; - } - - *update = (nri_container_update *)util_common_calloc_s(sizeof(nri_container_update)); - if (*update == NULL) { - ERROR("Out of memory"); - return false; - } - - (*update)->container_id = util_strdup_s(id); - (*update)->linux = (nri_linux_container_update *)util_common_calloc_s(sizeof(nri_linux_container_update)); - if ((*update)->linux == NULL) { - goto free_out; - } + nri_linux_resources *resources = NULL; - (*update)->ignore_failure = ignore_failure; - return true; - -free_out: - ERROR("Out of memory"); - free_nri_container_update(*update); - return false; -} - -bool init_nri_linux_resources(nri_linux_resources **resources) -{ + resources = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); if (resources == NULL) { - ERROR("Invalid input arguments"); - return false; - } - - *resources = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); - if (*resources == NULL) { ERROR("Out of memory"); return false; } - (*resources)->cpu = (nri_linux_cpu *)util_common_calloc_s(sizeof(nri_linux_cpu)); - if ((*resources)->cpu == NULL) { + resources->cpu = (nri_linux_cpu *)util_common_calloc_s(sizeof(nri_linux_cpu)); + if (resources->cpu == NULL) { goto free_out; } - (*resources)->memory = (nri_linux_memory *)util_common_calloc_s(sizeof(nri_linux_memory)); - if ((*resources)->memory == NULL) { + resources->memory = (nri_linux_memory *)util_common_calloc_s(sizeof(nri_linux_memory)); + if (resources->memory == NULL) { goto free_out; } - (*resources)->unified = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); - if ((*resources)->unified == NULL) { - goto free_out; - } - return true; + return resources; free_out: ERROR("Out of memory"); - free_nri_linux_resources(*resources); - return false; + free_nri_linux_resources(resources); + return NULL; } \ No newline at end of file diff --git a/src/daemon/common/nri/nri_utils.h b/src/daemon/common/nri/nri_utils.h index 7bf54a71..171055df 100644 --- a/src/daemon/common/nri/nri_utils.h +++ b/src/daemon/common/nri/nri_utils.h @@ -49,19 +49,19 @@ typedef enum { LAST = 12, } NRI_Event; -bool copy_nri_mount(const nri_mount *src, nri_mount **dest); -bool copy_nri_key_value(const nri_key_value *src, nri_key_value **dest); -bool copy_nri_posix_rlimit(const nri_posix_rlimit *src, nri_posix_rlimit **dest); -bool copy_nri_linux_resources(const nri_linux_resources *src, nri_linux_resources **dest); +nri_mount *copy_nri_mount(const nri_mount *src); +nri_posix_rlimit *copy_nri_posix_rlimit(const nri_posix_rlimit *src); +nri_linux_resources *copy_nri_linux_resources(const nri_linux_resources *src); +nri_key_value *copy_nri_key_value(const nri_key_value *src); +nri_linux_device *copy_nri_device(const nri_linux_device *src); bool is_marked_for_removal(const char* key, char **out); bool merge_nri_hooks(nri_hook **targetHooks, size_t targetSize, const nri_hook **sourceHooks, size_t sourceLen); -bool init_nri_container_adjust(nri_container_adjustment **adjust); -bool init_nri_container_update(nri_container_update **update, const char *id, uint8_t ignore_failure); -bool init_nri_linux_resources(nri_linux_resources **resources); +nri_container_update *init_nri_container_update(const char *id, const uint8_t ignore_failure); +nri_linux_resources *init_nri_linux_resources(void); #ifdef __cplusplus } diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c index 9ba1c8a0..020c9f32 100644 --- a/src/daemon/config/isulad_config.c +++ b/src/daemon/config/isulad_config.c @@ -1938,7 +1938,7 @@ int merge_json_confs_into_global(struct service_arguments *args) override_string_value(&args->json_confs->plugin_path, &tmp_json_confs->plugin_path); args->json_confs->plugin_registration_timeout = tmp_json_confs->plugin_registration_timeout; args->json_confs->plugin_requst_timeout = tmp_json_confs->plugin_requst_timeout; - override_string_value(&args->json_confs->nri_socket_path, &tmp_json_confs->nri_socket_path); + // Setting socket plugin path is not supported now #endif args->json_confs->enable_pod_events = tmp_json_confs->enable_pod_events; #endif diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc index 1cee68ec..d3fdd76a 100644 --- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc @@ -27,6 +27,11 @@ #include "stream_server.h" #include "sandbox_manager.h" +#ifdef ENABLE_NRI +#include "nri_adaption.h" +#include "nri_helpers.h" +#endif + namespace CRIV1 { auto ContainerManagerService::GetContainerOrSandboxRuntime(const std::string &realID, Errors &error) -> std::string { @@ -505,11 +510,24 @@ std::string ContainerManagerService::CreateContainer(const std::string &podSandb return response_id; } +#ifdef ENABLE_NRI + Errors nriErr; + nri_container_adjustment *adjust = NULL; + if (!NRIAdaptation::GetInstance()->CreateContainer(sandbox, response_id, containerConfig, &adjust, nriErr)) { + ERROR("Failed to get NRI adjustment for container: %s", nriErr.GetCMessage()); + NRIAdaptation::GetInstance()->UndoCreateContainer(sandbox, response_id, nriErr); + return response_id; + } +#endif + request = GenerateCreateContainerRequest(*sandbox, containerConfig, podSandboxConfig, error); if (error.NotEmpty()) { error.SetError("Failed to generate create container request"); goto cleanup; } +#ifdef ENABLE_NRI + request->adjust = adjust; +#endif if (m_cb->container.create(request, &response) != 0) { if (response != nullptr && (response->errmsg != nullptr)) { @@ -522,6 +540,12 @@ std::string ContainerManagerService::CreateContainer(const std::string &podSandb response_id = response->id; +#ifdef ENABLE_NRI + if (!NRIAdaptation::GetInstance()->PostCreateContainer(response_id, nriErr)) { + ERROR("NRI post-create notification failed: %s", nriErr.GetCMessage()); + } +#endif + cleanup: free_container_create_request(request); free_container_create_response(response); @@ -530,6 +554,9 @@ cleanup: void ContainerManagerService::StartContainer(const std::string &containerID, Errors &error) { +#ifdef ENABLE_NRI + Errors nriErr; +#endif if (containerID.empty()) { error.SetError("Invalid empty container id."); return; @@ -563,6 +590,14 @@ void ContainerManagerService::StartContainer(const std::string &containerID, Err goto cleanup; } +#ifdef ENABLE_NRI + if (!NRIAdaptation::GetInstance()->StartContainer(containerID, nriErr)) { + ERROR("NRI container start failed: %s", nriErr.GetCMessage()); + NRIAdaptation::GetInstance()->StopContainer(containerID, nriErr); + goto cleanup; + } +#endif + if (ret != 0) { if (response != nullptr && response->errmsg != nullptr) { error.SetError(response->errmsg); @@ -571,6 +606,11 @@ void ContainerManagerService::StartContainer(const std::string &containerID, Err } goto cleanup; } +#ifdef ENABLE_NRI + if (!NRIAdaptation::GetInstance()->PostStartContainer(containerID, nriErr)) { + ERROR("NRI PostStartContainer notification failed: %s", nriErr.GetCMessage()); + } +#endif cleanup: free_container_start_request(request); free_container_start_response(response); @@ -578,11 +618,25 @@ cleanup: void ContainerManagerService::StopContainer(const std::string &containerID, int64_t timeout, Errors &error) { +#ifdef ENABLE_NRI + Errors nriErr; +#endif CRIHelpers::StopContainer(m_cb, containerID, timeout, error); +#ifdef ENABLE_NRI + if (!NRIAdaptation::GetInstance()->StopContainer(containerID, nriErr)) { + ERROR("NRI StopContainer notification failed: %s", nriErr.GetCMessage()); + } +#endif } void ContainerManagerService::RemoveContainer(const std::string &containerID, Errors &error) { +#ifdef ENABLE_NRI + Errors nriErr; + if (!NRIAdaptation::GetInstance()->RemoveContainer(containerID, nriErr)) { + ERROR("NRI RemoveContainer notification failed: %s", nriErr.GetCMessage()); + } +#endif CRIHelpers::RemoveContainer(m_cb, containerID, error); if (error.NotEmpty()) { WARN("Failed to remove container %s", containerID.c_str()); @@ -1048,6 +1102,18 @@ void ContainerManagerService::UpdateContainerResources(const std::string &contai struct parser_context ctx { OPT_GEN_SIMPLIFY, 0 }; + + runtime::v1::LinuxContainerResources updateRes = resources; +#ifdef ENABLE_NRI + Errors nriErr; + runtime::v1::LinuxContainerResources adjust; + if (!NRIAdaptation::GetInstance()->UpdateContainer(containerID, resources, adjust, nriErr)) { + ERROR("NRI UpdateContainer notification failed: %s", nriErr.GetCMessage()); + goto cleanup; + } + updateRes = adjust; +#endif + request = (container_update_request *)util_common_calloc_s(sizeof(container_update_request)); if (request == nullptr) { error.SetError("Out of memory"); @@ -1061,17 +1127,17 @@ void ContainerManagerService::UpdateContainerResources(const std::string &contai goto cleanup; } - hostconfig->cpu_period = resources.cpu_period(); - hostconfig->cpu_quota = resources.cpu_quota(); - hostconfig->cpu_shares = resources.cpu_shares(); + hostconfig->cpu_period = updateRes.cpu_period(); + hostconfig->cpu_quota = updateRes.cpu_quota(); + hostconfig->cpu_shares = updateRes.cpu_shares(); - if (!resources.unified().empty()) { + if (!updateRes.unified().empty()) { hostconfig->unified = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); if (hostconfig->unified == nullptr) { error.SetError("Out of memory"); goto cleanup; } - for (auto &iter : resources.unified()) { + for (auto &iter : updateRes.unified()) { if (append_json_map_string_string(hostconfig->unified, iter.first.c_str(), iter.second.c_str()) != 0) { error.SetError("Failed to append string"); goto cleanup; @@ -1079,30 +1145,30 @@ void ContainerManagerService::UpdateContainerResources(const std::string &contai } } - hostconfig->memory = resources.memory_limit_in_bytes(); - hostconfig->memory_swap = resources.memory_swap_limit_in_bytes(); - if (!resources.cpuset_cpus().empty()) { - hostconfig->cpuset_cpus = util_strdup_s(resources.cpuset_cpus().c_str()); + hostconfig->memory = updateRes.memory_limit_in_bytes(); + hostconfig->memory_swap = updateRes.memory_swap_limit_in_bytes(); + if (!updateRes.cpuset_cpus().empty()) { + hostconfig->cpuset_cpus = util_strdup_s(updateRes.cpuset_cpus().c_str()); } - if (!resources.cpuset_mems().empty()) { - hostconfig->cpuset_mems = util_strdup_s(resources.cpuset_mems().c_str()); + if (!updateRes.cpuset_mems().empty()) { + hostconfig->cpuset_mems = util_strdup_s(updateRes.cpuset_mems().c_str()); } - if (resources.hugepage_limits_size() != 0) { + if (updateRes.hugepage_limits_size() != 0) { hostconfig->hugetlbs = (host_config_hugetlbs_element **)util_smart_calloc_s( - sizeof(host_config_hugetlbs_element *), resources.hugepage_limits_size()); + sizeof(host_config_hugetlbs_element *), updateRes.hugepage_limits_size()); if (hostconfig->hugetlbs == nullptr) { error.SetError("Out of memory"); goto cleanup; } - for (int i = 0; i < resources.hugepage_limits_size(); i++) { + for (int i = 0; i < updateRes.hugepage_limits_size(); i++) { hostconfig->hugetlbs[i] = (host_config_hugetlbs_element *)util_common_calloc_s(sizeof(host_config_hugetlbs_element)); if (hostconfig->hugetlbs[i] == nullptr) { error.SetError("Out of memory"); goto cleanup; } - hostconfig->hugetlbs[i]->page_size = util_strdup_s(resources.hugepage_limits(i).page_size().c_str()); - hostconfig->hugetlbs[i]->limit = resources.hugepage_limits(i).limit(); + hostconfig->hugetlbs[i]->page_size = util_strdup_s(updateRes.hugepage_limits(i).page_size().c_str()); + hostconfig->hugetlbs[i]->limit = updateRes.hugepage_limits(i).limit(); hostconfig->hugetlbs_len++; } } @@ -1121,6 +1187,12 @@ void ContainerManagerService::UpdateContainerResources(const std::string &contai error.SetError("Failed to call update container callback"); } } +#ifdef ENABLE_NRI + if (!NRIAdaptation::GetInstance()->PostUpdateContainer(containerID, nriErr)) { + ERROR("NRI PostUpdateContainer notification failed: %s", nriErr.GetCMessage()); + goto cleanup; + } +#endif cleanup: free_container_update_request(request); free_container_update_response(response); diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc index 77faf48a..0140eb99 100644 --- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc @@ -37,6 +37,9 @@ #include "transform.h" #include "isulad_config.h" #include "mailbox.h" +#ifdef ENABLE_NRI +#include "nri_adaption.h" +#endif namespace CRIV1 { void PodSandboxManagerService::PrepareSandboxData(const runtime::v1::PodSandboxConfig &config, @@ -304,6 +307,9 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1::PodSandboxConfig std::string network_setting_json; runtime::v1::PodSandboxConfig copyConfig = config; cri_container_message_t msg = { 0 }; +#ifdef ENABLE_NRI + Errors nriErr; +#endif // Step 1: Parepare sandbox name, runtime and networkMode PrepareSandboxData(config, runtimeHandler, sandboxName, runtimeInfo, networkMode, error); @@ -401,6 +407,14 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1::PodSandboxConfig msg.type = CRI_CONTAINER_MESSAGE_TYPE_STARTED; mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &msg); +#ifdef ENABLE_NRI + if (!NRIAdaptation::GetInstance()->RunPodSandbox(sandbox, nriErr)) { + ERROR("NRI RunPodSandbox failed: %s", nriErr.GetCMessage()); + error.Errorf("NRI RunPodSandbox failed: %s", nriErr.GetCMessage()); + return response_id; + } +#endif + return sandbox->GetId(); cleanup_network: @@ -418,6 +432,11 @@ cleanup_sandbox: if (error.NotEmpty()) { ERROR("Failed to delete sandbox: %s", sandbox->GetId().c_str()); } +#ifdef ENABLE_NRI + if (!NRIAdaptation::GetInstance()->RemovePodSandbox(sandbox, nriErr)) { + DEBUG("NRI RemovePodSandbox failed: %s", nriErr.GetCMessage()); + } +#endif return response_id; } @@ -618,7 +637,15 @@ void PodSandboxManagerService::StopPodSandbox(const std::string &podSandboxID, E return; } - sandbox->Stop(sandbox::DEFAULT_STOP_TIMEOUT, error); + if (!sandbox->Stop(sandbox::DEFAULT_STOP_TIMEOUT, error)) { + return; + } +#ifdef ENABLE_NRI + Errors nriStopPodErr; + if (!NRIAdaptation::GetInstance()->StopPodSandbox(sandbox, nriStopPodErr)) { + ERROR("NRI sandbox stop notification failed %s", nriStopPodErr.GetCMessage()); + } +#endif } void PodSandboxManagerService::RemoveAllContainersInSandbox(const std::string &readSandboxID, @@ -656,6 +683,9 @@ void PodSandboxManagerService::RemovePodSandbox(const std::string &podSandboxID, { std::vector errors; std::string realSandboxID; +#ifdef ENABLE_NRI + Errors nriErr; +#endif if (m_cb == nullptr || m_cb->container.remove == nullptr) { ERROR("Unimplemented callback"); @@ -724,6 +754,11 @@ void PodSandboxManagerService::RemovePodSandbox(const std::string &podSandboxID, msg.type = CRI_CONTAINER_MESSAGE_TYPE_DELETED; mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &msg); } +#ifdef ENABLE_NRI + if (!NRIAdaptation::GetInstance()->RemovePodSandbox(sandbox, nriErr)) { + ERROR("NRI RemovePodSandbox failed: %s", nriErr.GetCMessage()); + } +#endif } auto PodSandboxManagerService::SharesHostNetwork(const container_inspect *inspect) -> runtime::v1::NamespaceMode diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c index 2b6b2a15..dcbdd1d3 100644 --- a/src/daemon/executor/container_cb/execution_create.c +++ b/src/daemon/executor/container_cb/execution_create.c @@ -65,6 +65,10 @@ #include "mailbox.h" #include "specs_mount.h" +#ifdef ENABLE_NRI +#include "nri_spec.h" +#endif + #ifdef ENABLE_CRI_API_V1 static bool validate_sandbox_info(const container_sandbox_info *sandbox) { @@ -1465,6 +1469,14 @@ int container_create_cb(const container_create_request *request, container_creat skip_sandbox_key_manage = (is_sandbox_container(request->sandbox) && namespace_is_cni(host_spec->network_mode)); #endif +#ifdef ENABLE_NRI + if (request->adjust != NULL && nri_adjust_host_spec(request->adjust, host_spec) != 0) { + ERROR("Failed to adjust host spec"); + cc = ISULAD_ERR_INPUT; + goto clean_container_root_dir; + } +#endif + if (save_container_config_before_create(id, runtime_root, host_spec, v2_spec) != 0) { ERROR("Failed to malloc container_config_v2_common_config"); cc = ISULAD_ERR_INPUT; @@ -1553,6 +1565,15 @@ int container_create_cb(const container_create_request *request, container_creat } #endif +#ifdef ENABLE_NRI + // modify oci spec by nri plugin + if (request->adjust != NULL && nri_adjust_oci_spec(request->adjust, oci_spec) != 0) { + ERROR("Failed to adjust oci spec"); + cc = ISULAD_ERR_EXEC; + goto clean_netns; + } +#endif + host_channel = dup_host_channel(host_spec->host_channel); if (prepare_host_channel(host_channel, host_spec->user_remap)) { ERROR("Failed to prepare host channel"); diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c index 0b95cdad..4157c631 100644 --- a/src/daemon/modules/service/service_container.c +++ b/src/daemon/modules/service/service_container.c @@ -76,6 +76,9 @@ #include "sandbox_ops.h" #include "vsock_io_handler.h" #endif +#ifdef ENABLE_NRI +#include "nri_spec.h" +#endif #define KATA_RUNTIME "kata-runtime" @@ -726,6 +729,15 @@ static int do_oci_spec_update(const char *id, oci_runtime_spec *oci_spec, contai return -1; } +#ifdef ENABLE_NRI + // update oci spec with nri + ret = update_oci_nri(oci_spec, hostconfig); + if (ret != 0) { + ERROR("Failed to update oci spec with nri"); + return -1; + } +#endif + // renew_oci_config() will update process->user and share namespace after. return 0; diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c index 002431d8..36e89343 100644 --- a/src/daemon/modules/spec/specs.c +++ b/src/daemon/modules/spec/specs.c @@ -2883,7 +2883,12 @@ int spec_add_linux_resources_rlimit(oci_runtime_spec *oci_spec, const char *type return -1; } - ret = make_sure_oci_spec_linux_resources(oci_spec); + ret = make_sure_oci_spec_process(oci_spec); + if (ret < 0) { + return -1; + } + + ret = merge_ulimits_pre(oci_spec, 1); if (ret < 0) { return -1; } diff --git a/src/daemon/nri/nri_adaption.cc b/src/daemon/nri/nri_adaption.cc new file mode 100644 index 00000000..3190767f --- /dev/null +++ b/src/daemon/nri/nri_adaption.cc @@ -0,0 +1,1165 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * iSulad licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: zhongtao + * Create: 2024-03-15 + * Description: provide plugin manager(NRI adaption) class definition + *********************************************************************************/ + +#include "nri_adaption.h" + +#include + +#include +#include + +#include "isulad_config.h" +#include "utils_file.h" +#include "utils_string.h" +#include "utils.h" +#include "nri_convert.h" +#include "nri_plugin.h" +#include "nri_result.h" +#include "sandbox_manager.h" + +std::atomic NRIAdaptation::m_instance; + +NRIAdaptation *NRIAdaptation::GetInstance() noexcept +{ + static std::once_flag flag; + + std::call_once(flag, [] { m_instance = new NRIAdaptation; }); + + return m_instance; +} + +NRIAdaptation::~NRIAdaptation() +{ + for (const auto &pair : m_storeMap) { + auto plugin = pair.second; + plugin->shutdown(); + } +} + +auto NRIAdaptation::Init(Errors &error) -> bool +{ + std::map> tmp_storeMap; + + m_support = conf_get_nri_support(); + if (!m_support) { + return true; + } + + m_external_support = conf_get_nri_external_support(); + service_executor_t *cb = get_service_executor(); + if (cb == nullptr) { + ERROR("Init isulad service executor failure."); + return false; + } + + m_containerManager = std::make_unique(cb); + + m_pluginConfigPath = GetNRIPluginConfigPath(); + m_pluginPath = GetNRIPluginPath(); + m_plugin_registration_timeout = conf_get_nri_plugin_registration_timeout(); + m_plugin_requst_timeout = conf_get_nri_plugin_requst_timeout(); + m_sock_path = GetNRISockPath(); + + if (!StartPlugin()) { + ERROR("Failed to do StartPlugin"); + return false; + } + + if (!SortPlugins()) { + ERROR("Failed to do SortPlugins"); + return false; + } + + return true; +} + +void NRIAdaptation::RemoveClosedPlugins() +{ + std::vector closedPlugin; + + GetClosedPlugins(closedPlugin); + + for (const auto &key : closedPlugin) { + RemovePluginByIndex(key); + } +} + +void NRIAdaptation::GetClosedPlugins(std::vector &closedPlugin) +{ + ReadGuard lock(m_mutex); + for (const auto &pair : m_storeMap) { + auto plugin = pair.second; + if (plugin->IsClose()) { + closedPlugin.push_back(pair.first); + } + } +} + +auto NRIAdaptation::IsSupport() -> bool +{ + return m_support; +} + +auto NRIAdaptation::GetPluginByIndex(const std::string &index) -> std::shared_ptr +{ + ReadGuard lock(m_mutex); + return m_storeMap[index]; +} + +void NRIAdaptation::RemovePluginByIndex(const std::string &index) +{ + WriteGuard lock(m_mutex); + m_storeMap.erase(index); +} + +void NRIAdaptation::AddPluginByIndex(const std::string &index, std::shared_ptr plugin) +{ + WriteGuard lock(m_mutex); + m_storeMap[index] = plugin; +} + +auto NRIAdaptation::ApplyUpdates(const std::vector &update, + std::vector &failed, bool getFailed, Errors &error) -> bool +{ + for (auto &u : update) { + runtime::v1::LinuxContainerResources resources; + if (!LinuxResourcesFromNRI(u->linux->resources, resources)) { + ERROR("Failed to convert Linux resources from NRI"); + error.Errorf("Failed to convert Linux resources from NRI"); + return false; + } + + m_containerManager->UpdateContainerResources(u->container_id, resources, error); + + if (error.NotEmpty()) { + ERROR("Failed to update container: %s resources: %s", u->container_id, error.GetCMessage()); + if (!u->ignore_failure && getFailed) { + failed.push_back(u); + } + continue; + } + + TRACE("NRI update of container %s successful", u->container_id); + } + + if (failed.size() != 0) { + error.Errorf("NRI update of some containers failed"); + return false; + } + return true; +} + +auto NRIAdaptation::NewExternalPlugin(int fd) -> bool +{ + if (!m_external_support) { + ERROR("External plugin support is disabled"); + return false; + } + if (fd < 0) { + ERROR("Invalid fd"); + return false; + } + + std::string plugin_name; + NRIHelpers::GenerateRandomExternalName(plugin_name); + if (plugin_name.empty()) { + ERROR("Failed to generate random external name"); + return false; + } + + auto plugin = std::make_shared(fd, plugin_name); + + AddPluginByIndex(plugin_name, plugin); + + if (!plugin->Start(m_plugin_registration_timeout, m_plugin_requst_timeout)) { + ERROR("Failed to start plugin ready for conn fd %d", fd); + RemovePluginByIndex(plugin_name); + return false; + } + + std::vector pods; + std::vector cons; + nri_container_update **updateRes; + size_t update_len = 0; + + std::vector> sandboxes; + runtime::v1::PodSandboxFilter podFilter; + std::vector> containers; + Errors tmpError; + + std::vector updates; + std::vector failed; + + sandbox::SandboxManager::GetInstance()->ListAllSandboxes(podFilter, sandboxes); + + if (!PodSandboxesToNRI(sandboxes, pods)) { + ERROR("Failed to convert podsandbox to nri"); + NRIHelpers::FreeNriPodVector(pods); + return false; + } + + m_containerManager->ListContainers(nullptr, containers, tmpError); + + if (!ContainersToNRI(containers, cons)) { + ERROR("Failed to convert container to nri"); + NRIHelpers::FreeNriPodVector(pods); + NRIHelpers::FreeNriContainerVector(cons); + return false; + } + + // pods and cons's memory transfer to nri_synchronize_request, + // and is automatically freed when nri_synchronize_request is freed + if (!plugin->Synchronize(pods, cons, &updateRes, update_len, tmpError)) { + ERROR("Failed to synchronize plugin"); + return false; + } + + for (size_t i = 0; i < update_len; i++) { + updates.push_back(updateRes[i]); + } + + if (!ApplyUpdates(updates, failed, false, tmpError)) { + ERROR("Failed to update post-sync"); + } + + NRIHelpers::FreeNriContainerUpdateVector(updates); + NRIHelpers::FreeNriContainerUpdateVector(failed); + INFO("plugin %s connected", plugin_name.c_str()); + return true; +} + +auto NRIAdaptation::RunPodSandbox(std::shared_ptr sandbox, Errors &error) -> bool +{ + if (!m_support) { + return true; + } + + auto pod = NRIPodSandbox(sandbox, error); + if (pod == nullptr) { + ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); + return false; + } + + auto runPodEvent = makeUniquePtrCStructWrapper(free_nri_state_change_event); + if (runPodEvent == nullptr) { + ERROR("Out of memory"); + return false; + } + + runPodEvent->get()->pod = pod->move(); + runPodEvent->get()->event = RUN_POD_SANDBOX; + return StateChange(runPodEvent->get(), error); +} + +auto NRIAdaptation::StopPodSandbox(std::shared_ptr sandbox, Errors &error) -> bool +{ + if (!m_support) { + return true; + } + + auto pod = NRIPodSandbox(sandbox, error); + if (pod == nullptr) { + ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); + return false; + } + + auto stopPodEvent = makeUniquePtrCStructWrapper(free_nri_state_change_event); + if (stopPodEvent == nullptr) { + ERROR("Out of memory"); + return false; + } + + stopPodEvent->get()->pod = pod->move(); + stopPodEvent->get()->event = STOP_POD_SANDBOX; + return StateChange(stopPodEvent->get(), error); +} + +auto NRIAdaptation::RemovePodSandbox(std::shared_ptr sandbox, Errors &error) -> bool +{ + if (!m_support) { + return true; + } + + auto pod = NRIPodSandbox(sandbox, error); + if (pod == nullptr) { + ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); + return false; + } + + auto removePodEvent = makeUniquePtrCStructWrapper(free_nri_state_change_event); + if (removePodEvent == nullptr) { + ERROR("Out of memory"); + return false; + } + + removePodEvent->get()->pod = pod->move(); + removePodEvent->get()->event = REMOVE_POD_SANDBOX; + return StateChange(removePodEvent->get(), error); +} + +auto NRIAdaptation::CreateContainer(std::shared_ptr sandbox, const std::string &conId, + const runtime::v1::ContainerConfig &containerConfig, nri_container_adjustment **adjust, Errors &error) -> bool +{ + if (!m_support) { + return true; + } + + auto pod = NRIPodSandbox(sandbox, error); + if (pod == nullptr) { + ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); + return false; + } + + auto con = NRIContainerByConConfig(sandbox, containerConfig, error); + if (con == nullptr) { + ERROR("Failed to covert container to nri: %s", conId.c_str()); + return false; + } + + auto req = makeUniquePtrCStructWrapper(free_nri_create_container_request); + if (req == nullptr) { + ERROR("Out of memory"); + return false; + } + + req->get()->container = con->move(); + req->get()->pod = pod->move(); + + pluginResult result; + result.InitByConId(conId); + + if (!PluginsCreateContainer(req->get(), conId, result)) { + ERROR("Failed to call create container to all plugins"); + return false; + } + + RemoveClosedPlugins(); + + // TODO:evict container do not aply + + // TODO:how can i rollback on failure + std::vector failed; + if (!ApplyUpdates(result.GetReplyUpdate(), failed, false, error)) { + ERROR("Failed to apply updates"); + NRIHelpers::FreeNriContainerUpdateVector(failed); + return false; + } + + *adjust = result.MoveReplyAdjust(); + NRIHelpers::FreeNriContainerUpdateVector(failed); + return true; +} + +bool NRIAdaptation::PluginsCreateContainer(nri_create_container_request *req, const std::string &conId, + pluginResult &result) +{ + ReadGuard lock(m_mutex); + + for (const auto &pair : m_storeMap) { + auto plugin = pair.second; + Errors tmpError; + nri_create_container_response *resp = nullptr; + + if (!plugin->CreateContainer(req, &resp, tmpError)) { + ERROR("Failed to call create container: %s to pliugin: %s", conId.c_str(), plugin->GetName().c_str()); + (void)plugin->shutdown(); + continue; + } + + if (resp == nullptr) { + ERROR("Empty CreateContainer resp : %s", plugin->GetName().c_str()); + continue; + } + + auto resp_wrapper = + makeUniquePtrCStructWrapper(resp, free_nri_create_container_response); + if (resp_wrapper == nullptr) { + ERROR("Out of memory"); + return false; + } + + result.Apply(CREATE_CONTAINER, resp_wrapper->get()->adjust, resp_wrapper->get()->update, + resp_wrapper->get()->update_len, plugin->GetName()); + } + return true; +} + +auto NRIAdaptation::PostCreateContainer(const std::string &conId, Errors &error) -> bool +{ + if (!m_support) { + return true; + } + + auto con = NRIContainerByID(conId, error); + if (con == nullptr) { + ERROR("Failed to covert container to nri: %s", conId.c_str()); + return false; + } + + auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(con->get()->pod_sandbox_id); + if (sandbox == nullptr) { + ERROR("Failed to get sandbox info for nri"); + return false; + } + + auto pod = NRIPodSandbox(sandbox, error); + if (pod == nullptr) { + ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); + return false; + } + + auto postCreateConEvent = makeUniquePtrCStructWrapper(free_nri_state_change_event); + if (postCreateConEvent == nullptr) { + ERROR("Out of memory"); + return false; + } + + postCreateConEvent->get()->container = con->move(); + postCreateConEvent->get()->pod = pod->move(); + postCreateConEvent->get()->event = POST_CREATE_CONTAINER; + return StateChange(postCreateConEvent->get(), error); +} + +auto NRIAdaptation::StartContainer(const std::string &conId, Errors &error) -> bool +{ + if (!m_support) { + return true; + } + + auto con = NRIContainerByID(conId, error); + if (con == nullptr) { + ERROR("Failed to covert container to nri: %s", conId.c_str()); + return false; + } + + auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(con->get()->pod_sandbox_id); + if (sandbox == nullptr) { + ERROR("Failed to get sandbox info for nri"); + return false; + } + + if (sandbox == nullptr) { + ERROR("Failed to get sandbox for container: %s, sandbox id: %s", conId.c_str(), con->get()->pod_sandbox_id); + return false; + } + + auto pod = NRIPodSandbox(sandbox, error); + if (pod == nullptr) { + ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); + return false; + } + + auto startConEvent = makeUniquePtrCStructWrapper(free_nri_state_change_event); + if (startConEvent == nullptr) { + ERROR("Out of memory"); + return false; + } + + startConEvent->get()->container = con->move(); + startConEvent->get()->pod = pod->move(); + startConEvent->get()->event = START_CONTAINER; + return StateChange(startConEvent->get(), error); +} + +auto NRIAdaptation::PostStartContainer(const std::string &conId, Errors &error) -> bool +{ + if (!m_support) { + return true; + } + + auto con = NRIContainerByID(conId, error); + if (con == nullptr) { + ERROR("Failed to covert container to nri: %s", conId.c_str()); + return false; + } + + auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(con->get()->pod_sandbox_id); + if (sandbox == nullptr) { + ERROR("Failed to get sandbox info for nri"); + return false; + } + + auto pod = NRIPodSandbox(sandbox, error); + if (pod == nullptr) { + ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); + return false; + } + + auto postStartConEvent = makeUniquePtrCStructWrapper(free_nri_state_change_event); + if (postStartConEvent == nullptr) { + ERROR("Out of memory"); + return false; + } + + postStartConEvent->get()->container = con->move(); + postStartConEvent->get()->pod = pod->move(); + postStartConEvent->get()->event = POST_START_CONTAINER; + return StateChange(postStartConEvent->get(), error); +} + +auto NRIAdaptation::UndoCreateContainer(std::shared_ptr sandbox, const std::string &conId, + Errors &error) -> bool +{ + if (!m_support) { + return true; + } + + if (!StopContainer(conId, error)) { + ERROR("container creation undo (stop) failed: %s", conId.c_str()); + } + + if (!RemoveContainer(conId, error)) { + ERROR("container creation undo (remove) failed: %s", conId.c_str()); + } + + return true; +} + +auto NRIAdaptation::UpdateContainer(const std::string &conId, const runtime::v1::LinuxContainerResources &resources, + runtime::v1::LinuxContainerResources &adjust, Errors &error) -> bool +{ + if (!m_support) { + return true; + } + + auto con = NRIContainerByID(conId, error); + if (con == nullptr) { + ERROR("Failed to covert container to nri: %s", conId.c_str()); + return false; + } + + auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(con->get()->pod_sandbox_id); + if (sandbox == nullptr) { + ERROR("Failed to get sandbox info for nri"); + return false; + } + + auto pod = NRIPodSandbox(sandbox, error); + if (pod == nullptr) { + ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); + return false; + } + + auto req = makeUniquePtrCStructWrapper(free_nri_update_container_request); + if (req == nullptr) { + ERROR("Out of memory"); + return false; + } + + auto reqRes = LinuxResourcesToNRI(resources); + + req->get()->container = con->move(); + req->get()->pod = pod->move(); + req->get()->linux_resources = reqRes; + + pluginResult result; + result.InitByUpdateReq(req->get()); + + if (!PluginsUpdateContainer(req->get(), conId, result)) { + ERROR("Failed to call update container to all plugins"); + return false; + } + + RemoveClosedPlugins(); + + // TODO:evict container do not aply + + // TODO:how can i rollback on failure + std::vector failed; + if (!ApplyUpdates(result.GetReplyUpdate(), failed, false, error)) { + ERROR("Failed to apply updates"); + NRIHelpers::FreeNriContainerUpdateVector(failed); + return false; + } + + NRIHelpers::FreeNriContainerUpdateVector(failed); + + if (!LinuxResourcesFromNRI(result.GetReplyResources(conId), adjust)) { + ERROR("Failed to convert Linux resources from NRI"); + return false; + } + + return true; +} + +bool NRIAdaptation::PluginsUpdateContainer(nri_update_container_request *req, const std::string &conId, + pluginResult &result) +{ + ReadGuard lock(m_mutex); + + for (const auto &pair : m_storeMap) { + auto plugin = pair.second; + Errors tmpError; + nri_update_container_response *resp = nullptr; + + if (!plugin->UpdateContainer(req, &resp, tmpError)) { + ERROR("Failed to call create container: %s to pliugin: %s", conId.c_str(), plugin->GetName().c_str()); + plugin->shutdown(); + continue; + } + + if (resp == nullptr) { + ERROR("Empty UpdateContainer resp : %s", plugin->GetName().c_str()); + continue; + } + + auto resp_wrapper = + makeUniquePtrCStructWrapper(resp, free_nri_update_container_response); + if (resp_wrapper == nullptr) { + ERROR("Out of memory"); + return false; + } + + result.Apply(UPDATE_CONTAINER, nullptr, resp_wrapper->get()->update, resp_wrapper->get()->update_len, + plugin->GetName()); + } + return true; +} + +bool NRIAdaptation::PluginsStopContainer(nri_stop_container_request *req, const std::string &conId, + pluginResult &result) +{ + ReadGuard lock(m_mutex); + + for (const auto &pair : m_storeMap) { + auto plugin = pair.second; + Errors tmpError; + nri_stop_container_response *resp = nullptr; + + if (!plugin->StopContainer(req, &resp, tmpError)) { + ERROR("Failed to call create container: %s to pliugin: %s", conId.c_str(), plugin->GetName().c_str()); + plugin->shutdown(); + continue; + } + + if (resp == nullptr) { + ERROR("Empty StopContainer resp : %s", plugin->GetName().c_str()); + continue; + } + + auto resp_wrapper = makeUniquePtrCStructWrapper(resp, free_nri_stop_container_response); + if (resp_wrapper == nullptr) { + ERROR("Out of memory"); + return false; + } + + result.Apply(STOP_CONTAINER, nullptr, resp_wrapper->get()->update, resp_wrapper->get()->update_len, plugin->GetName()); + } + return true; +} + +auto NRIAdaptation::PostUpdateContainer(const std::string &conId, Errors &error) ->bool +{ + if (!m_support) { + return true; + } + + auto con = NRIContainerByID(conId, error); + if (con == nullptr) { + ERROR("Failed to covert container to nri: %s", conId.c_str()); + return false; + } + + auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(con->get()->pod_sandbox_id); + if (sandbox == nullptr) { + ERROR("Failed to get sandbox info for nri"); + return false; + } + + auto pod = NRIPodSandbox(sandbox, error); + if (pod == nullptr) { + ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); + return false; + } + + auto postUpdateConEvent = makeUniquePtrCStructWrapper(free_nri_state_change_event); + if (postUpdateConEvent == nullptr) { + ERROR("Out of memory"); + return false; + } + + postUpdateConEvent->get()->container = con->move(); + postUpdateConEvent->get()->pod = pod->move(); + postUpdateConEvent->get()->event = POST_UPDATE_CONTAINER; + return StateChange(postUpdateConEvent->get(), error); +} + +auto NRIAdaptation::StopContainer(const std::string &conId, Errors &error) ->bool +{ + if (!m_support) { + return true; + } + + auto con = NRIContainerByID(conId, error); + if (con == nullptr) { + ERROR("Failed to covert container to nri: %s", conId.c_str()); + return false; + } + + auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(con->get()->pod_sandbox_id); + if (sandbox == nullptr) { + ERROR("Failed to get sandbox info for nri"); + return false; + } + + auto pod = NRIPodSandbox(sandbox, error); + if (pod == nullptr) { + ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); + return false; + } + + auto req = makeUniquePtrCStructWrapper(free_nri_stop_container_request); + if (req == nullptr) { + ERROR("Out of memory"); + return false; + } + + req->get()->pod = pod->move(); + req->get()->container = con->move(); + + pluginResult result; + result.Init(); + + if (!PluginsStopContainer(req->get(), conId, result)) { + ERROR("Failed to call stop container to all plugins"); + return false; + } + + RemoveClosedPlugins(); + + // TODO:how can i rollback on failure + std::vector failed; + if (!ApplyUpdates(result.GetReplyUpdate(), failed, false, error)) { + ERROR("Failed to apply updates"); + NRIHelpers::FreeNriContainerUpdateVector(failed); + return false; + } + + NRIHelpers::FreeNriContainerUpdateVector(failed); + return true; +} + +auto NRIAdaptation::RemoveContainer(const std::string &conId, Errors &error) ->bool +{ + if (!m_support) { + return true; + } + + auto con = NRIContainerByID(conId, error); + if (con == nullptr) { + ERROR("Failed to covert container to nri: %s", conId.c_str()); + return false; + } + + auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(con->get()->pod_sandbox_id); + if (sandbox == nullptr) { + ERROR("Failed to get sandbox info for nri"); + return false; + } + + auto pod = NRIPodSandbox(sandbox, error); + if (pod == nullptr) { + ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); + return false; + } + + auto removeConEvent = makeUniquePtrCStructWrapper(free_nri_state_change_event); + if (removeConEvent == nullptr) { + ERROR("Out of memory"); + return false; + } + + removeConEvent->get()->container = con->move(); + removeConEvent->get()->pod = pod->move(); + removeConEvent->get()->event = REMOVE_CONTAINER; + return StateChange(removeConEvent->get(), error); +} + +auto NRIAdaptation::StateChange(nri_state_change_event *evt, Errors &error) ->bool +{ + if (evt->event == UNKNOWN) { + ERROR("invalid (unset) event in state change notification"); + error.SetError("invalid (unset) event in state change notification"); + return false; + } + + PluginsStateChange(evt); + + RemoveClosedPlugins(); + return true; +} + +void NRIAdaptation::PluginsStateChange(nri_state_change_event *evt) +{ + ReadGuard lock(m_mutex); + + for (const auto &pair : m_storeMap) { + auto plugin = pair.second; + Errors tmpError; + if (!plugin->StateChange(evt, tmpError)) { + ERROR("invalid (unset) event in state change notification: %s", plugin->GetName().c_str()); + plugin->shutdown(); + continue; + } + } +} + +// Perform a set of unsolicited container updates requested by a plugin. +auto NRIAdaptation::updateContainers(const nri_update_containers_request *req, + nri_update_containers_response **resp) ->bool +{ + std::vector failed; + std::vector vec; + size_t i; + Errors error; + bool ret = false; + + if (req == nullptr) { + ERROR("Invalid request"); + return false; + } + + for (i = 0; i < req->update_len; i++) { + vec.push_back(req->update[i]); + } + + if (!ApplyUpdates(vec, failed, false, error)) { + ERROR("Failed to apply updates: %s", error.GetCMessage()); + goto free_out; + } + + if (failed.size() == 0) { + ret = true; + goto free_out; + } + + (*resp)->failed = (nri_container_update **)util_common_calloc_s(failed.size() * sizeof(nri_container_update *)); + if ((*resp)->failed == nullptr) { + ERROR("Out of memory"); + goto free_out; + } + + for (i = 0; i < failed.size(); i++) { + (*resp)->failed[i] = failed[i]; + failed[i] = nullptr; + (*resp)->failed_len++; + } + ret = true; + +free_out: + NRIHelpers::FreeNriContainerUpdateVector(vec); + NRIHelpers::FreeNriContainerUpdateVector(failed); + return ret; +} + +auto NRIAdaptation::StartPlugin() -> bool +{ + std::map> tmp_storeMap; + + if (!DiscoverPlugins(tmp_storeMap) != 0) { + ERROR("Failed to do DiscoverPlugins"); + return false; + } + + for (const auto &pair : tmp_storeMap) { + const std::string &index = pair.first; + auto plugin = pair.second; + + if (!NewLaunchedPlugin(plugin)) { + ERROR("Failed to do NewLaunchedPlugin for %s", plugin->GetName().c_str()); + continue; + } + + AddPluginByIndex(index, plugin); + + if (!plugin->Start(m_plugin_registration_timeout, m_plugin_requst_timeout)) { + ERROR("Failed to start plugin %s ready", plugin->GetName().c_str()); + RemovePluginByIndex(index); + (void)plugin->shutdown(); + continue; + } + } + + return true; +} + +// return true always, failure to obtain one plugin does not affect other plugin +static auto walk_plugin_dir_cb(const char *path_name, const struct dirent *sub_dir, void *context) -> bool +{ + std::string full_path = std::string(path_name) + "/" + sub_dir->d_name; + struct stat file_stat = { 0 }; + + if (stat(full_path.c_str(), &file_stat) != 0) { + WARN("Failed to get NRI plugin %s stat", sub_dir->d_name); + return true; + } + + if (S_ISDIR(file_stat.st_mode)) { + INFO("Skip dir in plugin path %s", sub_dir->d_name); + return true; + } + + // 1. Verify plugin permissions for exec + if (!(file_stat.st_mode & S_IXUSR)) { + ERROR("NRI plugin %s has no permission for exec", sub_dir->d_name); + return true; + } + + // 2. Parse plugin name + __isula_auto_array_t char **arr = util_string_split_n(sub_dir->d_name, '-', 2); + if (arr == nullptr) { + ERROR("Invalid plugin name %s, idx-pluginname expected", sub_dir->d_name); + return true; + } + + if (!NRIHelpers::CheckPluginIndex(arr[0])) { + ERROR("Invalid plugin name %s, invalid idx", sub_dir->d_name); + return true; + } + + // 3. init plugin + std::string index(arr[0]); + std::string pluginName(arr[1]); + std::string config; + + std::map> &map = + *static_cast>*>(context); + if (!NRIHelpers::GetPluginConfig(index, pluginName, config)) { + ERROR("Failed to get plugin %s config", pluginName.c_str()); + return true; + } + + auto plugin = std::make_shared(index, pluginName, config); + + // todo:use random id str + map[pluginName] = plugin; + return true; +} + +static void plugin_exec_func(nri_plugin_exec_args_t * plugin_args) +{ + const char *params[PARAM_NUM] = {0}; + int i = 0; + std::string sock = std::to_string(plugin_args->sockFd); + + if (plugin_args == nullptr) { + ERROR("Missing plugin exec info"); + _exit(EXIT_FAILURE); + } + + if (chdir(plugin_args->workdir) < 0) { + ERROR("Failed to chdir to %s", plugin_args->workdir); + _exit(EXIT_FAILURE); + } + + if (setenv(PluginNameEnv.c_str(), plugin_args->name, 1) != 0) { + ERROR("%s: failed to set PluginNameEnv for process %d", plugin_args->name, getpid()); + exit(EXIT_FAILURE); + } + + if (setenv(PluginIdxEnv.c_str(), plugin_args->index, 1) != 0) { + ERROR("%s: failed to set PluginIdxEnv for process %d", plugin_args->name, getpid()); + exit(EXIT_FAILURE); + } + + if (setenv(PluginSocketEnv.c_str(), sock.c_str(), 1) != 0) { + ERROR("%s: failed to set PluginSocketEnv for process %d", plugin_args->name, getpid()); + exit(EXIT_FAILURE); + } + + if (util_check_inherited(true, plugin_args->sockFd) != 0) { + ERROR("Failed to close inherited fds"); + exit(EXIT_FAILURE); + } + + if (setsid() < 0) { + ERROR("Failed to setsid for nri plugin: %s", plugin_args->name); + exit(EXIT_FAILURE); + } + + params[i++] = plugin_args->name; + + execvp(plugin_args->cmd, (char * const *)params); + ERROR("Failed to exec %s", plugin_args->cmd); + _exit(EXIT_FAILURE); +} + +// create socket, and call plugin start +auto NRIAdaptation::NewLaunchedPlugin(const std::shared_ptr &plugin) -> bool +{ + // 1. create socket for plugin + if (!plugin->CreateSocketPair()) { + ERROR("Failed to create socket pair"); + return false; + } + + std::string name = plugin->GetQualifiedName(); + std::string cmd = m_pluginPath + "/" + name; + + DEBUG("Plugin %s start", cmd.c_str()); + + // 2. exec plugin + nri_plugin_exec_args_t p_args = { + .workdir = m_pluginPath.c_str(), + .cmd = cmd.c_str(), + .name = name.c_str(), + .index = plugin->GetIndex().c_str(), + .sockFd = plugin->GetPeerSockFd(), + }; + + int pid = fork(); + if (pid == (pid_t) -1) { + SYSERROR("Failed to fork"); + return false; + } + + if (pid == (pid_t)0) { + set_child_process_pdeathsig(); + + plugin_exec_func(&p_args); + } + + close(plugin->GetPeerSockFd()); + + plugin->SetPid(pid); + + return true; +} + +// find plugin and create plugin +auto NRIAdaptation::DiscoverPlugins(std::map> &map) -> bool +{ + int nret = 0; + + // 1. get all plugin + nret = util_scan_subdirs(m_pluginPath.c_str(), walk_plugin_dir_cb, static_cast(&map)); + if (nret != 0) { + ERROR("Failed to scan nri plugin subdirs"); + } + return true; +} + +auto NRIAdaptation::SortPlugins() -> bool +{ + RemoveClosedPlugins(); + + std::vector>> sortedPlugins(m_storeMap.begin(), m_storeMap.end()); + + std::sort(sortedPlugins.begin(), sortedPlugins.end(), [](const auto & a, const auto & b) { + return a.first < b.first; + }); + + WriteGuard lock(m_mutex); + m_storeMap.clear(); + for (const auto &pair : sortedPlugins) { + m_storeMap.insert(pair); + } + + return true; +} + +auto NRIAdaptation::GetNRIPluginConfigPath(void) -> std::string +{ + __isula_auto_free char *config_path = nullptr; + std::string ret; + + config_path = conf_get_nri_plugin_config_path(); + if (config_path == nullptr) { + return ret; + } + ret = std::string(config_path); + return ret; +} + +auto NRIAdaptation::GetNRIPluginPath(void) -> std::string +{ + __isula_auto_free char *plugin_path = nullptr; + std::string ret; + + plugin_path = conf_get_nri_plugin_path(); + if (plugin_path == nullptr) { + return ret; + } + ret = std::string(plugin_path); + return ret; +} + +auto NRIAdaptation::GetNRISockPath(void) -> std::string +{ + __isula_auto_free char *sock_path = nullptr; + std::string ret; + + sock_path = conf_get_socket_path(); + if (sock_path == nullptr) { + return ret; + } + ret = std::string(sock_path); + return ret; +} + +auto NRIAdaptation::NRIPodSandbox(const std::shared_ptr &sandbox, + Errors &error) -> std::unique_ptr> +{ + auto pod = makeUniquePtrCStructWrapper(free_nri_pod_sandbox); + if (pod == nullptr) { + ERROR("Out of memory"); + return nullptr; + } + + if (!PodSandboxToNRI(sandbox, *pod->get())) { + error.Errorf("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); + ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); + return nullptr; + } + + return pod; +} + +auto NRIAdaptation::NRIContainerByID(const std::string &id, + Errors &error) -> std::unique_ptr> +{ + auto con = makeUniquePtrCStructWrapper(free_nri_container); + if (con == nullptr) { + ERROR("Out of memory"); + return nullptr; + } + + if (!ContainerToNRIByID(id, *con->get())) { + error.Errorf("Failed to covert container to nri: %s", id.c_str()); + ERROR("Failed to covert container to nri: %s", id.c_str()); + return nullptr; + } + + return con; +} + +auto NRIAdaptation::NRIContainerByConConfig(const std::shared_ptr &sandbox, + const runtime::v1::ContainerConfig &containerConfig, Errors &error) -> std::unique_ptr> +{ + auto con = makeUniquePtrCStructWrapper(free_nri_container); + if (con == nullptr) { + ERROR("Out of memory"); + return nullptr; + } + + if (!ContainerToNRIByConConfig(containerConfig, *con->get())) { + error.Errorf("Failed to covert container to nri: %s", con->get()->name); + ERROR("Failed to covert container to nri: %s", con->get()->name); + return nullptr; + } + con->get()->pod_sandbox_id = isula_strdup_s(sandbox->GetId().c_str()); + + return con; +} \ No newline at end of file diff --git a/src/daemon/nri/nri_adaption.h b/src/daemon/nri/nri_adaption.h index 874662cf..27a6d93e 100644 --- a/src/daemon/nri/nri_adaption.h +++ b/src/daemon/nri/nri_adaption.h @@ -46,10 +46,6 @@ public: auto GetSockpath(std::vector &paths) -> bool; - auto StopPlugins() -> bool; - - void RemoveClosedPlugins(); - auto GetPluginByIndex(const std::string &index) -> std::shared_ptr; void AddPluginByIndex(const std::string &index, std::shared_ptr plugin); void RemovePluginByIndex(const std::string &index); @@ -65,7 +61,8 @@ public: Errors &error) -> bool; auto StartContainer(const std::string &conId, Errors &error) -> bool; auto PostStartContainer(const std::string &conId, Errors &error) -> bool; - auto UpdateContainer(const std::string &conId, Errors &error) -> bool; + auto UpdateContainer(const std::string &conId, const runtime::v1::LinuxContainerResources &resources, + runtime::v1::LinuxContainerResources &adjust, Errors &error) -> bool; auto PostUpdateContainer(const std::string &conId, Errors &error) -> bool; auto StopContainer(const std::string &conId, Errors &error) -> bool; auto RemoveContainer(const std::string &conId, Errors &error) -> bool; @@ -87,6 +84,8 @@ private: auto SyncPlugin() -> bool; auto SortPlugins() -> bool; + void RemoveClosedPlugins(); + void GetClosedPlugins(std::vector &closedPlugin); auto ApplyUpdates(const std::vector &update, std::vector &failed, @@ -107,18 +106,16 @@ private: void PluginsStateChange(nri_state_change_event *evt); bool PluginsCreateContainer(nri_create_container_request *req, const std::string &conId, pluginResult &result); bool PluginsUpdateContainer(nri_update_container_request *req, const std::string &conId, pluginResult &result); + bool PluginsStopContainer(nri_stop_container_request *req, const std::string &conId, pluginResult &result); private: RWMutex m_mutex; static std::atomic m_instance; bool m_support; bool m_external_support; - std::string m_version; std::string m_sock_path; std::string m_pluginConfigPath; std::string m_pluginPath; - std::vector m_socketPathArr; - std::string m_disableConnections; // id --> NRIPlugin map std::map> m_storeMap; // TODO:plugin monitor thread id?? diff --git a/src/daemon/nri/nri_helpers.cc b/src/daemon/nri/nri_helpers.cc index ff9d67c1..b660e7a7 100644 --- a/src/daemon/nri/nri_helpers.cc +++ b/src/daemon/nri/nri_helpers.cc @@ -90,4 +90,25 @@ bool CheckPluginIndex(const std::string &idx) return true; } + +void FreeNriContainerUpdateVector(std::vector &vec) +{ + for (auto ptr : vec) { + free_nri_container_update(ptr); + } +} + +void FreeNriContainerVector(std::vector &vec) +{ + for (auto ptr : vec) { + free_nri_container(ptr); + } +} + +void FreeNriPodVector(std::vector &vec) +{ + for (auto ptr : vec) { + free_nri_pod_sandbox(ptr); + } +} }// namespace NRIHelpers \ No newline at end of file diff --git a/src/daemon/nri/nri_helpers.h b/src/daemon/nri/nri_helpers.h index 1a2f488e..e776987b 100644 --- a/src/daemon/nri/nri_helpers.h +++ b/src/daemon/nri/nri_helpers.h @@ -41,6 +41,10 @@ void GenerateRandomExternalName(std::string &ret); bool CheckPluginIndex(const std::string &idx); +void FreeNriContainerUpdateVector(std::vector &vec); +void FreeNriContainerVector(std::vector &vec); +void FreeNriPodVector(std::vector &vec); + template void freeArray(T ** &arr, int size) { diff --git a/src/daemon/nri/nri_plugin_ops.cc b/src/daemon/nri/nri_plugin_ops.cc new file mode 100644 index 00000000..e2f88b63 --- /dev/null +++ b/src/daemon/nri/nri_plugin_ops.cc @@ -0,0 +1,123 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * iSulad licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: zhongtao + * Create: 2024-03-26 + * Description: provide nri plugin api definition + ******************************************************************************/ +#include "nri_plugin_ops.h" + +#include +#include + +#include "nri_adaption.h" +#include "nri_plugin.h" +#include "isulad_config.h" + +static bool start_external_listener() +{ + __isula_auto_free char *sock_path = NULL; + + sock_path = conf_get_socket_path(); + if (sock_path == NULL) { + ERROR("Failed to get socket path"); + return false; + } + + if (nri_external_service_start(sock_path, nri_external_plugin_connect) != 0) { + ERROR("Failed to lauch external service"); + return false; + } + return true; +} + +bool nri_adaption_init(void) +{ + Errors error; + + if (!conf_get_nri_support()) { + return true; + } + + nri_runtime_callbacks callbacks; + callbacks.register_plugin = nri_registry_containers; + callbacks.update_containers = nri_update_containers; + if (nri_runtime_service_init(callbacks) != 0) { + ERROR("Failed to init runtime service\n"); + return false; + } + + if (conf_get_nri_external_support()) { + if (!start_external_listener()) { + ERROR("Failed to start external listener\n"); + goto clean_out; + } + } + + NRIAdaptation::GetInstance()->Init(error); + if (error.NotEmpty()) { + ERROR("Failed to init NRIAdaptation: %s", error.GetCMessage()); + goto clean_out; + } + return true; +clean_out: + nri_runtime_service_destroy(); + return false; +} + +bool nri_adaption_shutdown(void) +{ + nri_external_service_shutdown(); + nri_runtime_service_destroy(); + return true; +} + +int nri_update_containers(const char *plugin_id, const nri_update_containers_request *request, + nri_update_containers_response **response) +{ + if (request == nullptr || response == nullptr || plugin_id == nullptr) { + ERROR("Invalid input arguments"); + return -1; + } + + if (!NRIAdaptation::GetInstance()->updateContainers(request, response)) { + ERROR("Failed to update containers by plugin %s", plugin_id); + return -1; + } + + return 0; +} + +int nri_registry_containers(const char *plugin_id, const nri_register_plugin_request *request) +{ + if (request == nullptr || plugin_id == nullptr) { + ERROR("Invalid input arguments"); + return -1; + } + + auto plugin = NRIAdaptation::GetInstance()->GetPluginByIndex(plugin_id); + if (plugin == nullptr) { + ERROR("Failed to get plugin by index %s", plugin_id); + return -1; + } + + plugin->SetReady(); + return 0; +} + +int nri_external_plugin_connect(int fd) +{ + if (fd < 0) { + ERROR("Invalid input arguments"); + return -1; + } + + return NRIAdaptation::GetInstance()->NewExternalPlugin(fd) ? 0 : -1; +} \ No newline at end of file diff --git a/src/daemon/nri/nri_plugin_ops.h b/src/daemon/nri/nri_plugin_ops.h index 37d437d2..3a5393ba 100644 --- a/src/daemon/nri/nri_plugin_ops.h +++ b/src/daemon/nri/nri_plugin_ops.h @@ -25,13 +25,15 @@ extern "C" { #endif bool nri_adaption_init(void); +bool nri_adaption_shutdown(void); #ifdef __cplusplus } #endif -int nri_update_containers(const nri_update_containers_request *request, nri_update_containers_response **response); -int nri_registry_containers(const nri_register_plugin_request *request); +int nri_update_containers(const char *plugin_id, const nri_update_containers_request *request, + nri_update_containers_response **response); +int nri_registry_containers(const char *plugin_id, const nri_register_plugin_request *request); int nri_external_plugin_connect(int fd); diff --git a/src/daemon/nri/nri_result.cc b/src/daemon/nri/nri_result.cc new file mode 100644 index 00000000..7da4451d --- /dev/null +++ b/src/daemon/nri/nri_result.cc @@ -0,0 +1,977 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * iSulad licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: zhongtao + * Create: 2024-06-29 + * Description: provide nri result definition + *********************************************************************************/ + +#include "nri_result.h" + +#include +#include + +#include "cxxutils.h" +#include "transform.h" +#include "utils.h" + +pluginResult::~pluginResult() +{ + free_nri_linux_resources(m_update_req); + free_nri_container_adjustment(m_reply.adjust); + for (size_t i = 0; i < m_reply.update.size(); i++) { + free_nri_container_update(m_reply.update[i]); + } +} + +auto pluginResult::InitReply() -> bool +{ + m_reply.adjust = (nri_container_adjustment *)util_common_calloc_s(sizeof(nri_container_adjustment)); + if (m_reply.adjust == NULL) { + ERROR("Out of memory"); + return false; + } + return true; +} + +auto pluginResult::Init() -> bool +{ + if (!InitReply()) { + ERROR("Failed to init reply"); + return false; + } + m_update_req = nullptr; + return true; +} + +auto pluginResult::InitByConId(std::string conId) -> bool +{ + m_conId = conId; + + if (!InitReply()) { + ERROR("Failed to init reply"); + return false; + } + + m_update_req = nullptr; + return true; +} + +auto pluginResult::InitByUpdateReq(nri_update_container_request *req) -> bool +{ + m_conId = req->container->id; + m_update_req = copy_nri_linux_resources(req->linux_resources); + if (m_update_req == nullptr) { + ERROR("Failed to copy nri linux resources"); + return false; + } + + if (!InitReply()) { + ERROR("Failed to init reply"); + return false; + } + m_update_req = nullptr; + return true; +} + +auto pluginResult::GetReplyUpdate() -> std::vector +{ + return m_reply.update; +} + +auto pluginResult::MoveReplyAdjust() -> nri_container_adjustment * +{ + nri_container_adjustment *ret = m_reply.adjust; + m_reply.adjust = nullptr; + return ret; +} + +auto pluginResult::GetReplyResources(const std::string &id) -> const nri_linux_resources * +{ + nri_linux_resources *ret = NULL; + nri_container_update *update = m_updates[id]; + ret = update->linux->resources; + return ret; +} + +auto pluginResult::Apply(int32_t event, const nri_container_adjustment *adjust, nri_container_update **update, + size_t update_len, const std::string &plugin) -> bool +{ + if (plugin.length() == 0) { + ERROR("Empty plugin name"); + return false; + } + if (event == CREATE_CONTAINER) { + if (!Adjust(adjust, plugin)) { + ERROR("Failed to do adjust to plugin: %s", plugin.c_str()); + return false; + } + + if (!Update(update, update_len, plugin)) { + ERROR("Failed to do update to plugin: %s", plugin.c_str()); + return false; + } + return true; + } else if (event == UPDATE_CONTAINER) { + if (!Update(update, update_len, plugin)) { + ERROR("Failed to do update to plugin: %s", plugin.c_str()); + return false; + } + return true; + } else if (event == STOP_CONTAINER) { + if (!Update(update, update_len, plugin)) { + ERROR("Failed to do update to plugin: %s", plugin.c_str()); + return false; + } + return true; + } else { + ERROR("Cannot apply response of invalid type %d", event); + return false; + } + return true; +} + +auto pluginResult::Adjust(const nri_container_adjustment *adjust, const std::string &plugin) -> bool +{ + if (adjust == nullptr) { + return true; + } + + if (!AdjustAnnotations(adjust->annotations, plugin)) { + ERROR("Cannot adajust annotations by plugin %s", plugin.c_str()); + return false; + } + + if (!AdjustMounts(adjust->mounts, adjust->mounts_len, plugin)) { + ERROR("Cannot adajust mounts by plugin %s", plugin.c_str()); + return false; + } + + if (!AdjustEnv(adjust->env, adjust->env_len, plugin)) { + ERROR("Cannot adajust mounts by plugin %s", plugin.c_str()); + return false; + } + + if (!AdjustHooks(adjust->hooks, plugin)) { + ERROR("Cannot adajust hooks by plugin %s", plugin.c_str()); + return false; + } + + if (adjust->linux != nullptr) { + if (m_reply.adjust->linux == nullptr) { + m_reply.adjust->linux = (nri_linux_container_adjustment *)util_common_calloc_s(sizeof(nri_linux_container_adjustment)); + if (m_reply.adjust->linux == nullptr) { + ERROR("Out of memory"); + return false; + } + } + + if (!AdjustDevices(adjust->linux->devices, adjust->linux->devices_len, plugin)) { + ERROR("Cannot adajust devices by plugin %s", plugin.c_str()); + return false; + } + + if (!AdjustResources(adjust->linux->resources, plugin)) { + ERROR("Cannot adajust devices by plugin %s", plugin.c_str()); + return false; + } + + if (!AdjustCgroupsPath(adjust->linux->cgroups_path, plugin)) { + ERROR("Cannot adajust cgroups path by plugin %s", plugin.c_str()); + return false; + } + } + + if (!AdjustRlimits(adjust->rlimits, adjust->rlimits_len, plugin)) { + ERROR("Cannot adajust rlimits path by plugin %s", plugin.c_str()); + return false; + } + + return true; +} + +auto pluginResult::AdjustAnnotations(json_map_string_string *annos, const std::string &plugin) -> bool +{ + if (annos == nullptr || annos->len == 0) { + return true; + } + + if (m_reply.adjust->annotations == nullptr) { + m_reply.adjust->annotations = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); + if (m_reply.adjust->annotations == nullptr) { + ERROR("Out of memory"); + return false; + } + } + + google::protobuf::Map del; + const char *id = m_conId.c_str(); + google::protobuf::Map mapAnno; + Transform::JsonMapToProtobufMapForString(annos, mapAnno); + + // if key is marked for remove, add pair to del, and delete key from annos + for (auto it = mapAnno.begin(); it != mapAnno.end();) { + const std::string &key = it->first; + char *out = NULL; + if (is_marked_for_removal(key.c_str(), &out)) { + del[out] = ""; + it = mapAnno.erase(it); + } else { + ++it; + } + } + + for (const auto &iter : mapAnno) { + std::string key = iter.first; + std::string value = iter.second; + auto it = del.find(key); + if (it != del.end()) { + auto owner = m_owners.find(id); + if (owner != m_owners.end()) { + m_owners[id].annotations.erase(key); + } + append_json_map_string_string(m_reply.adjust->annotations, NRIHelpers::MarkForRemoval(key).c_str(), ""); + } + + // set annotations's owner plugin + auto owner = m_owners.find(id); + if (owner != m_owners.end()) { + auto anno = m_owners[id].annotations.find(key); + if (anno != m_owners[id].annotations.end()) { + ERROR("plugins %s and %s both tried to set annotation: %s", plugin.c_str(), anno->second.c_str(), key.c_str()); + return false; + } + m_owners[id].annotations[key] = plugin; + } + + // add pair to m_reply.adjust + append_json_map_string_string(m_reply.adjust->annotations, key.c_str(), value.c_str()); + del.erase(key); + } + + // add del to m_reply.adjust + for (auto &pair : del) { + append_json_map_string_string(m_reply.adjust->annotations, NRIHelpers::MarkForRemoval(pair.first).c_str(), ""); + } + + return true; +} + +auto pluginResult::AdjustMounts(nri_mount **mounts, size_t mounts_size, const std::string &plugin) -> bool +{ + if (mounts == nullptr || mounts_size == 0) { + return true; + } + + size_t i; + std::vector add; + std::map del; + std::string id = m_conId.c_str(); + + // first split removals from the rest of adjustments + for (i = 0; i < mounts_size; i++) { + char *out = NULL; + if (is_marked_for_removal(mounts[i]->destination, &out)) { + del[out] = mounts[i]; + } else { + add.push_back(mounts[i]); + } + } + + // next remove marked mounts from collected adjustments + nri_mount** cleard = nullptr; + size_t clearLen = 0; + + if (m_reply.adjust->mounts_len > 0) { + cleard = (nri_mount **)util_common_calloc_s(m_reply.adjust->mounts_len * sizeof(nri_mount *)); + if (cleard == nullptr) { + ERROR("Out of memory"); + return false; + } + + for (i = 0; i < m_reply.adjust->mounts_len; i++) { + auto removed = del.find(m_reply.adjust->mounts[i]->destination); + if (removed != del.end()) { + auto owner = m_owners.find(id); + if (owner != m_owners.end()) { + m_owners[id].mounts.erase(m_reply.adjust->mounts[i]->destination); + } + continue; + } + cleard[clearLen] = copy_nri_mount(m_reply.adjust->mounts[i]); + if (cleard[clearLen] == nullptr) { + ERROR("Failed to copy nri mounts to cleard"); + return false; + } + clearLen++; + } + + NRIHelpers::freeArray(m_reply.adjust->mounts, m_reply.adjust->mounts_len); + m_reply.adjust->mounts = cleard; + m_reply.adjust->mounts_len = clearLen; + } + + // finally, apply additions to collected adjustments + size_t oldSize, newSize; + oldSize = m_reply.adjust->mounts_len * sizeof(nri_mount *); + newSize = oldSize + add.size() * sizeof(nri_mount *); + int ret = util_mem_realloc((void **)(&m_reply.adjust->mounts), newSize, (void *)m_reply.adjust->mounts, oldSize); + if (ret != 0) { + ERROR("Failed to realloc and assign nri mounts array"); + return false; + } + for (i = 0; i < add.size(); i++) { + // set mounts's owner plugin + auto owner = m_owners.find(id); + if (owner != m_owners.end()) { + auto mount = m_owners[id].mounts.find(add[i]->destination); + if (mount != m_owners[id].mounts.end()) { + ERROR("plugins %s and %s both tried to set mount: %s", plugin.c_str(), mount->second.c_str(), add[i]->destination); + return false; + } + m_owners[id].mounts[add[i]->destination] = plugin; + } + m_reply.adjust->mounts[m_reply.adjust->mounts_len] = copy_nri_mount(add[i]); + if (m_reply.adjust->mounts[m_reply.adjust->mounts_len] == nullptr) { + ERROR("Failed to copy add nri mounts to reply adjust"); + return false; + } + m_reply.adjust->mounts_len++; + } + + return true; +} + +auto pluginResult::AdjustEnv(nri_key_value **envs, size_t envs_size, const std::string &plugin) -> bool +{ + if (envs == nullptr || envs_size == 0) { + return true; + } + + size_t i; + std::vector add; + std::map del; + std::string id = m_conId.c_str(); + + // first split removals from the rest of adjustments + for (i = 0; i < envs_size; i++) { + char *out = NULL; + if (is_marked_for_removal(envs[i]->key, &out)) { + del[out] = envs[i]; + } else { + add.push_back(envs[i]); + } + } + + // next remove marked mounts from collected adjustments + nri_key_value** cleard; + size_t clearLen = 0; + + if(m_reply.adjust->env_len > 0) { + cleard = (nri_key_value **)util_common_calloc_s(m_reply.adjust->env_len * sizeof(nri_key_value *)); + if (cleard == nullptr) { + ERROR("Out of memory"); + return false; + } + + for (i = 0; i < m_reply.adjust->env_len; i++) { + auto removed = del.find(m_reply.adjust->env[i]->key); + if (removed != del.end()) { + auto owner = m_owners.find(id); + if (owner != m_owners.end()) { + m_owners[id].env.erase(m_reply.adjust->env[i]->key); + } + continue; + } + cleard[clearLen] = copy_nri_key_value(m_reply.adjust->env[i]); + if (cleard[clearLen] == nullptr) { + ERROR("Failed to copy nri env key value to cleard"); + return false; + } + clearLen++; + } + + NRIHelpers::freeArray(m_reply.adjust->env, m_reply.adjust->env_len); + m_reply.adjust->env = cleard; + m_reply.adjust->env_len = clearLen; + } + + // finally, apply additions to collected adjustments + size_t oldSize, newSize; + oldSize = m_reply.adjust->env_len * sizeof(nri_key_value *); + newSize = oldSize + add.size() * sizeof(nri_key_value *); + int ret = util_mem_realloc((void **)(&m_reply.adjust->env), newSize, m_reply.adjust->env, oldSize); + if (ret != 0) { + ERROR("Failed to realloc and assign nri env array"); + return false; + } + for (i = 0; i < add.size(); i++) { + // set env's owner plugin + auto owner = m_owners.find(id); + if (owner != m_owners.end()) { + auto env = m_owners[id].env.find(add[i]->key); + if (env != m_owners[id].env.end()) { + ERROR("plugins %s and %s both tried to set env: %s", plugin.c_str(), env->second.c_str(), add[i]->key); + return false; + } + m_owners[id].env[add[i]->key] = plugin; + } + m_reply.adjust->env[m_reply.adjust->env_len] = copy_nri_key_value(add[i]); + if (m_reply.adjust->env[m_reply.adjust->env_len] == nullptr) { + ERROR("Failed to copy add nri env to reply adjust"); + return false; + } + m_reply.adjust->env_len++; + } + + return true; +} + +auto pluginResult::AdjustHooks(const nri_hooks *hooks, const std::string &plugin) -> bool +{ + if (hooks == nullptr) { + return true; + } + + if (m_reply.adjust->hooks == nullptr) { + m_reply.adjust->hooks = (nri_hooks *)util_common_calloc_s(sizeof(nri_hooks)); + if (m_reply.adjust->hooks == nullptr) { + ERROR("Out of memory"); + return false; + } + } + + nri_hooks * reply = m_reply.adjust->hooks; + + if (!merge_nri_hooks(reply->prestart, reply->prestart_len, (const nri_hook**)hooks->prestart, hooks->prestart_len)) { + ERROR("Failed to realloc and copy prestart hooks"); + return false; + } + + if (!merge_nri_hooks(reply->poststart, reply->poststart_len, (const nri_hook**)hooks->poststart, + hooks->poststart_len)) { + ERROR("Failed to realloc and copy poststart hooks"); + return false; + } + + if (!merge_nri_hooks(reply->poststop, reply->poststop_len, (const nri_hook**)hooks->poststop, hooks->poststop_len)) { + ERROR("Failed to realloc and copy poststop hooks"); + return false; + } + + /* TODO:zhongtao + * The OCI being used by the iSulad not supportes + * createRuntime/createContainer/startContainer currently. + */ + if (!merge_nri_hooks(reply->create_runtime, reply->create_runtime_len, (const nri_hook**)hooks->create_runtime, + hooks->create_runtime_len)) { + ERROR("Failed to realloc and copy create_runtime hooks"); + return false; + } + + if (!merge_nri_hooks(reply->create_container, reply->create_container_len, (const nri_hook**)hooks->create_container, + hooks->create_container_len)) { + ERROR("Failed to realloc and copy create_container hooks"); + return false; + } + + if (!merge_nri_hooks(reply->start_container, reply->start_container_len, (const nri_hook**)hooks->start_container, + hooks->start_container_len)) { + ERROR("Failed to realloc and copy start_container hooks"); + return false; + } + + return false; +} + +auto pluginResult::AdjustDevices(nri_linux_device **devices, size_t devices_size, const std::string &plugin) -> bool +{ + if (devices_size == 0) { + return true; + } + + size_t i; + std::vector add; + std::map del; + std::string id = m_conId.c_str(); + + // first split removals from the rest of adjustments + for (i = 0; i < devices_size; i++) { + char *out = NULL; + if (is_marked_for_removal(devices[i]->path, &out)) { + del[out] = devices[i]; + } else { + add.push_back(devices[i]); + } + } + + // next remove marked mounts from collected adjustments + nri_linux_device** cleard; + size_t clearLen = 0; + + if (m_reply.adjust->linux->devices_len > 0) { + cleard = (nri_linux_device **)util_common_calloc_s(m_reply.adjust->linux->devices_len * sizeof(nri_linux_device *)); + if (cleard == nullptr) { + ERROR("Out of memory"); + return false; + } + for (i = 0; i < m_reply.adjust->linux->devices_len; i++) { + auto removed = del.find(m_reply.adjust->linux->devices[i]->path); + if (removed != del.end()) { + auto owner = m_owners.find(id); + if (owner != m_owners.end()) { + m_owners[id].devices.erase(m_reply.adjust->linux->devices[i]->path); + } + continue; + } + cleard[clearLen] = copy_nri_device(m_reply.adjust->linux->devices[i]); + if (cleard[clearLen] == nullptr) { + ERROR("Failed to copy nri linux device to cleard"); + return false; + } + clearLen++; + } + + NRIHelpers::freeArray(m_reply.adjust->linux->devices, m_reply.adjust->linux->devices_len); + m_reply.adjust->linux->devices = cleard; + m_reply.adjust->linux->devices_len = clearLen; + } + + // finally, apply additions to collected adjustments + size_t oldSize, newSize; + oldSize = m_reply.adjust->linux->devices_len * sizeof(nri_linux_device *); + newSize = oldSize + add.size() * sizeof(nri_linux_device *); + int ret = util_mem_realloc((void **)(&m_reply.adjust->linux->devices), newSize, m_reply.adjust->linux->devices, oldSize); + if (ret != 0) { + ERROR("Failed to realloc and assign nri devices array"); + return false; + } + for (i = 0; i < add.size(); i++) { + // set mounts's owner plugin + auto owner = m_owners.find(id); + if (owner != m_owners.end()) { + auto device = m_owners[id].devices.find(add[i]->path); + if (device != m_owners[id].devices.end()) { + ERROR("plugins %s and %s both tried to set devices: %s", plugin.c_str(), device->second.c_str(), add[i]->path); + return false; + } + m_owners[id].devices[add[i]->path] = plugin; + } + m_reply.adjust->linux->devices[m_reply.adjust->linux->devices_len] = copy_nri_device(add[i]); + if (m_reply.adjust->linux->devices[m_reply.adjust->linux->devices_len] == nullptr) { + ERROR("Failed to copy add nri devices to reply adjust"); + return false; + } + m_reply.adjust->linux->devices_len++; + } + + return true; +} + +auto pluginResult::AdjustResources(nri_linux_resources *resources, const std::string &plugin) -> bool +{ + if (resources == nullptr) { + return true; + } + + if (m_reply.adjust->linux->resources == nullptr) { + m_reply.adjust->linux->resources = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); + if (m_reply.adjust->linux->resources == nullptr) { + ERROR("Out of memory"); + return false; + } + } + + std::string id = m_conId.c_str(); + nri_linux_resources *reply = m_reply.adjust->linux->resources; + + return ClaimAndCopyResources(resources, id, plugin, reply); +} + +bool pluginResult::ClaimAndCopyResources(nri_linux_resources *src, std::string &id, const std::string &plugin, + nri_linux_resources *dest) +{ + size_t i; + if (src->memory != nullptr) { + if (src->memory->limit != nullptr) { + auto memLimit = m_owners[id].memLimit; + if (!memLimit.empty()) { + ERROR("plugins %s and %s both tried to set devices's memory limit", plugin.c_str(), memLimit.c_str()); + return false; + } + m_owners[id].memLimit = plugin; + *dest->memory->limit = *src->memory->limit; + } + + if (src->memory->reservation != nullptr) { + auto memReservation = m_owners[id].memReservation; + if (!memReservation.empty()) { + ERROR("plugins %s and %s both tried to set devices's memory reservation", plugin.c_str(), + memReservation.c_str()); + return false; + } + m_owners[id].memReservation = plugin; + *dest->memory->reservation = *src->memory->reservation; + } + + if (src->memory->swap != nullptr) { + auto memSwapLimit = m_owners[id].memSwapLimit; + if (!memSwapLimit.empty()) { + ERROR("plugins %s and %s both tried to set devices's memory swap limit", plugin.c_str(), + memSwapLimit.c_str()); + return false; + } + m_owners[id].memSwapLimit = plugin; + *dest->memory->swap = *src->memory->swap; + } + + if (src->memory->kernel != nullptr) { + auto memKernelLimit = m_owners[id].memKernelLimit; + if (!memKernelLimit.empty()) { + ERROR("plugins %s and %s both tried to set devices's memory kernel limit", plugin.c_str(), + memKernelLimit.c_str()); + return false; + } + m_owners[id].memKernelLimit = plugin; + *dest->memory->kernel = *src->memory->kernel; + } + + if (src->memory->kernel_tcp != nullptr) { + auto memTCPLimit = m_owners[id].memTCPLimit; + if (!memTCPLimit.empty()) { + ERROR("plugins %s and %s both tried to set devices's memory tcp limit", plugin.c_str(), + memTCPLimit.c_str()); + return false; + } + m_owners[id].memTCPLimit = plugin; + *dest->memory->kernel_tcp = *src->memory->kernel_tcp; + } + + if (src->memory->swappiness != nullptr) { + auto memSwappiness = m_owners[id].memSwappiness; + if (!memSwappiness.empty()) { + ERROR("plugins %s and %s both tried to set devices's memory swappiness", plugin.c_str(), + memSwappiness.c_str()); + return false; + } + m_owners[id].memSwappiness = plugin; + *dest->memory->swappiness = *src->memory->swappiness; + } + + if (src->memory->disable_oom_killer != nullptr) { + auto memDisableOomKiller = m_owners[id].memDisableOomKiller; + if (!memDisableOomKiller.empty()) { + ERROR("plugins %s and %s both tried to set devices's memory disable_oom_killer", plugin.c_str(), + memDisableOomKiller.c_str()); + return false; + } + m_owners[id].memDisableOomKiller = plugin; + *dest->memory->disable_oom_killer = *src->memory->disable_oom_killer; + } + + if (src->memory->use_hierarchy != nullptr) { + auto memUseHierarchy = m_owners[id].memUseHierarchy; + if (!memUseHierarchy.empty()) { + ERROR("plugins %s and %s both tried to set devices's memory use_hierarchy", plugin.c_str(), + memUseHierarchy.c_str()); + return false; + } + m_owners[id].memUseHierarchy = plugin; + *dest->memory->use_hierarchy = *src->memory->use_hierarchy; + } + } + + if (src->cpu != nullptr) { + if (src->cpu->shares != nullptr) { + auto cpuShares = m_owners[id].cpuShares; + if (!cpuShares.empty()) { + ERROR("plugins %s and %s both tried to set devices's cpu shares", plugin.c_str(), cpuShares.c_str()); + return false; + } + m_owners[id].cpuShares = plugin; + *dest->cpu->shares = *src->cpu->shares; + } + + if (src->cpu->quota != nullptr) { + auto cpuQuota = m_owners[id].cpuQuota; + if (!cpuQuota.empty()) { + ERROR("plugins %s and %s both tried to set devices's cpu quota", plugin.c_str(), cpuQuota.c_str()); + return false; + } + m_owners[id].cpuQuota = plugin; + *dest->cpu->quota = *src->cpu->quota; + } + + if (src->cpu->period != nullptr) { + auto cpuPeriod = m_owners[id].cpuPeriod; + if (!cpuPeriod.empty()) { + ERROR("plugins %s and %s both tried to set devices's cpu period", plugin.c_str(), cpuPeriod.c_str()); + return false; + } + m_owners[id].cpuPeriod = plugin; + *dest->cpu->period = *src->cpu->period; + } + + if (src->cpu->realtime_runtime != nullptr) { + auto cpuRealtimePeriod = m_owners[id].cpuRealtimePeriod; + if (!cpuRealtimePeriod.empty()) { + ERROR("plugins %s and %s both tried to set devices's cpu realtime_runtime", plugin.c_str(), + cpuRealtimePeriod.c_str()); + return false; + } + m_owners[id].cpuRealtimePeriod = plugin; + *dest->cpu->realtime_runtime = *src->cpu->realtime_runtime; + } + + if (src->cpu->realtime_period != nullptr) { + auto cpuRealtimePeriod = m_owners[id].cpuRealtimePeriod; + if (!cpuRealtimePeriod.empty()) { + ERROR("plugins %s and %s both tried to set devices's cpu realtime_period", plugin.c_str(), + cpuRealtimePeriod.c_str()); + return false; + } + m_owners[id].cpuRealtimePeriod = plugin; + *dest->cpu->realtime_period = *src->cpu->realtime_period; + } + + if (src->cpu->cpus != nullptr) { + auto cpusetCpus = m_owners[id].cpusetCpus; + if (!cpusetCpus.empty()) { + ERROR("plugins %s and %s both tried to set devices's cpu cpus", plugin.c_str(), cpusetCpus.c_str()); + return false; + } + m_owners[id].cpusetCpus = plugin; + *dest->cpu->cpus = *src->cpu->cpus; + } + + if (src->cpu->mems != nullptr) { + auto cpusetMems = m_owners[id].cpusetMems; + if (!cpusetMems.empty()) { + ERROR("plugins %s and %s both tried to set devices's cpu mems", plugin.c_str(), cpusetMems.c_str()); + return false; + } + m_owners[id].cpusetMems = plugin; + *dest->cpu->mems = *src->cpu->mems; + } + } + + for (i = 0; i < src->hugepage_limits_len; i++) { + auto owner = m_owners.find(id); + if (owner != m_owners.end()) { + auto find = m_owners[id].hugepageLimits.find(src->hugepage_limits[i]->page_size); + if (find != m_owners[id].hugepageLimits.end()) { + ERROR("plugins %s and %s both tried to set hugepageLimits: %s", plugin.c_str(), find->second.c_str(), + src->hugepage_limits[i]->page_size); + return false; + } + m_owners[id].hugepageLimits[src->hugepage_limits[i]->page_size] = plugin; + } + } + + if (src->unified->len != 0) { + google::protobuf::Map mapAnno; + Transform::JsonMapToProtobufMapForString(src->unified, mapAnno); + for (const auto &iter : mapAnno) { + std::string key = iter.first; + std::string value = iter.second; + auto owner = m_owners.find(id); + if (owner != m_owners.end()) { + auto anno = m_owners[id].unified.find(key); + if (anno != m_owners[id].unified.end()) { + ERROR("plugins %s and %s both tried to set unified: %s", plugin.c_str(), anno->second.c_str(), + key.c_str()); + return false; + } + m_owners[id].unified[key] = plugin; + } + // add pair to m_reply.adjust + append_json_map_string_string(dest->unified, key.c_str(), value.c_str()); + } + } + + if (src->blockio_class != nullptr) { + auto blockioClass = m_owners[id].blockioClass; + if (!blockioClass.empty()) { + ERROR("plugins %s and %s both tried to set devices's blockio_class", plugin.c_str(), blockioClass.c_str()); + return false; + } + m_owners[id].blockioClass = plugin; + dest->blockio_class = util_strdup_s(src->blockio_class); + } + + if (src->rdt_class != nullptr) { + auto rdtClass = m_owners[id].rdtClass; + if (!rdtClass.empty()) { + ERROR("plugins %s and %s both tried to set devices's rdt_class", plugin.c_str(), rdtClass.c_str()); + return false; + } + m_owners[id].rdtClass = plugin; + dest->rdt_class = util_strdup_s(src->rdt_class); + } + return true; +} + +auto pluginResult::AdjustCgroupsPath(char *path, const std::string &plugin) -> bool +{ + if (path == nullptr || strcmp(path, "") == 0) { + return true; + } + + std::string id = m_conId.c_str(); + + auto cgroupsPath = m_owners[id].cgroupsPath; + if (!cgroupsPath.empty()) { + ERROR("plugins %s and %s both tried to set devices's cgroups path", plugin.c_str(), cgroupsPath.c_str()); + return false; + } + m_owners[id].cgroupsPath = plugin; + m_reply.adjust->linux->cgroups_path = util_strdup_s(path); + + return true; +} + +auto pluginResult::AdjustRlimits(nri_posix_rlimit **rlimits, size_t rlimits_len, const std::string &plugin) -> bool +{ + if (rlimits_len == 0) { + return true; + } + + size_t i; + std::string id = m_conId.c_str(); + + size_t oldSize, newSize; + oldSize = m_reply.adjust->rlimits_len * sizeof(nri_posix_rlimit *); + newSize = oldSize + rlimits_len * sizeof(nri_posix_rlimit *); + int ret = util_mem_realloc((void **)(&m_reply.adjust->rlimits), newSize, m_reply.adjust->rlimits, oldSize); + if (ret != 0) { + ERROR("Failed to realloc and assign nri rlimits array"); + return false; + } + + for (i = 0; i < rlimits_len; i++) { + auto owner = m_owners.find(id); + if (owner != m_owners.end()) { + auto find = m_owners[id].rlimits.find(rlimits[i]->type); + if (find != m_owners[id].rlimits.end()) { + ERROR("plugins %s and %s both tried to set rlimits type: %s", plugin.c_str(), find->second.c_str(), rlimits[i]->type); + return false; + } + m_owners[id].rlimits[rlimits[i]->type] = plugin; + } + m_reply.adjust->rlimits[m_reply.adjust->rlimits_len] = copy_nri_posix_rlimit(rlimits[i]); + if (m_reply.adjust->rlimits[m_reply.adjust->rlimits_len] == nullptr) { + ERROR("Failed to copy add nri rlimits to reply adjust"); + return false; + } + m_reply.adjust->rlimits_len++; + } + return true; +} + +auto pluginResult::Update(nri_container_update **updates, size_t update_len, const std::string &plugin) -> bool +{ + if (update_len == 0) { + return true; + } + + size_t i; + + for (i = 0; i < update_len; i++) { + nri_container_update *reply; + if (!GetContainerUpdate(updates[i], plugin, &reply)) { + ERROR("Failed to get container update in plugin result"); + return false; + } + + if (!UpdateResources(reply, updates[i], plugin)) { + ERROR("Failed to update container resources in plugin result"); + return false; + } + } + + return true; +} + +auto pluginResult::GetContainerUpdate(nri_container_update *update, const std::string &plugin, + nri_container_update **out) -> bool +{ + if (update == nullptr || out == nullptr || plugin.empty()) { + ERROR("Empyt input args"); + return false; + } + + auto id = update->container_id; + + if (std::string(id) == m_conId) { + ERROR("Plugin %s asked update of %s during creation", plugin.c_str(), id); + return false; + } + + auto find = m_updates.find(id); + if (find != m_updates.end()) { + *out = m_updates[id]; + (*out)->ignore_failure = (*out)->ignore_failure && update->ignore_failure; + return true; + } + + *out = init_nri_container_update(id, update->ignore_failure); + if (*out == nullptr) { + ERROR("Failed to init nri container update"); + return false; + } + + m_updates[id] = *out; + + // for update requests delay appending the requested container (in the response getter) + if (m_conId != id) { + m_reply.update.push_back(*out); + } + + return true; +} + +auto pluginResult::UpdateResources(nri_container_update *reply, nri_container_update *u, + const std::string &plugin) -> bool +{ + if (u->linux == nullptr || u->linux->resources == nullptr) { + return true; + } + + std::string id = u->container_id; + nri_linux_resources *resources; + + if (m_conId == id) { + resources = copy_nri_linux_resources(m_update_req); + if (resources == nullptr) { + ERROR("Failed to copy request's nri linux resources"); + return false; + } + } else { + resources = copy_nri_linux_resources(reply->linux->resources); + if (resources == nullptr) { + ERROR("Failed to copy reply's nri linux resources"); + return false; + } + } + + if (!ClaimAndCopyResources(u->linux->resources, id, plugin, resources)) { + ERROR("Failed to claim and copy resources in plugin result"); + return false; + } + + // update reply from copy on success + reply->linux->resources = copy_nri_linux_resources(resources); + if (reply->linux->resources == nullptr) { + ERROR("Failed to copy resources's nri linux resources to reply"); + return false; + } + + return true; +} \ No newline at end of file diff --git a/src/daemon/nri/nri_result.h b/src/daemon/nri/nri_result.h index f2896ea0..4f26385a 100644 --- a/src/daemon/nri/nri_result.h +++ b/src/daemon/nri/nri_result.h @@ -63,7 +63,7 @@ struct owners { struct resultReply { nri_container_adjustment* adjust; - std::vector update; + std::vector update; }; using resultOwners = std::map; @@ -71,16 +71,19 @@ using resultOwners = std::map; class pluginResult { public: pluginResult() = default; + pluginResult(std::string conId); - ~pluginResult() = default; + virtual ~pluginResult(); + auto Init() -> bool; auto InitByConId(std::string conId) -> bool; auto InitByUpdateReq(nri_update_container_request *req) -> bool; auto GetReplyUpdate() -> std::vector; - auto GetReplyAdjust() -> nri_container_adjustment *; + auto MoveReplyAdjust() -> nri_container_adjustment *; + auto GetReplyResources(const std::string &id) -> const nri_linux_resources *; - auto Apply(int32_t event, nri_container_adjustment *adjust, nri_container_update **update, size_t update_len, + auto Apply(int32_t event, const nri_container_adjustment *adjust, nri_container_update **update, size_t update_len, const std::string &plugin) -> bool; auto Update(nri_container_update **updates, size_t update_len, const std::string &plugin) -> bool; @@ -90,12 +93,12 @@ private: auto InitReply(void) -> bool; - auto Adjust(nri_container_adjustment *adjust, const std::string &plugin) -> bool; + auto Adjust(const nri_container_adjustment *adjust, const std::string &plugin) -> bool; auto AdjustAnnotations(json_map_string_string *annos, const std::string &plugin) -> bool; auto AdjustMounts(nri_mount **mounts, size_t mounts_size, const std::string &plugin) -> bool; auto AdjustEnv(nri_key_value **envs, size_t envs_size, const std::string &plugin) -> bool; - auto AdjustHooks(nri_hooks *hooks, const std::string &plugin) -> bool; + auto AdjustHooks(const nri_hooks *hooks, const std::string &plugin) -> bool; auto AdjustDevices(nri_linux_device **devices, size_t devices_size, const std::string &plugin) -> bool; auto AdjustResources(nri_linux_resources *resources, const std::string &plugin) -> bool; bool ClaimAndCopyResources(nri_linux_resources *src, std::string &id, const std::string &plugin, @@ -107,7 +110,9 @@ private: std::string m_conId; nri_linux_resources *m_update_req; resultReply m_reply; + // update plugin -> update context std::map m_updates; + // adjust plugin -> adjust context resultOwners m_owners; }; diff --git a/src/daemon/nri/plugin.cc b/src/daemon/nri/plugin.cc new file mode 100644 index 00000000..904677c4 --- /dev/null +++ b/src/daemon/nri/plugin.cc @@ -0,0 +1,417 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * iSulad licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: zhongtao + * Create: 2024-03-15 + * Description: provide plugin class definition + *********************************************************************************/ + +#include "plugin.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "utils.h" +#include "cstruct_wrapper.h" + +// same as containerd +std::string DefaultNRIVersion = "2.0.0-beta.2+unknown"; +// same as containerd +std::string DefaultNRIRuntimeName = "v2"; +// defualt timeout for wait: 2s +const int64_t DefaultWaitTimeout = 2000; +const uint64_t SECOND_TO_NANOS = 1000000000; + +// init client conn +NRIPlugin::NRIPlugin(std::string &idx, std::string &name, std::string &config) +{ + m_idx = idx; + m_name = name; + m_config = config; + m_closed = false; + m_external = false; + m_pid = -1; +} + +NRIPlugin::NRIPlugin(int fd, std::string &name) +{ + m_sockFds.push_back(fd); + m_name = name; + m_closed = false; + m_external = true; + m_pid = -1; +} + +// wait for plugin to register, then configure it. +auto NRIPlugin::Start(int64_t registry_timeout, int64_t request_timeout) -> bool +{ + Errors error; + + // todo: what if timeout is 0 or other invalid value? + + if (!Connect(request_timeout)) { + ERROR("Failed to connect nri plugin %s", m_name.c_str()); + return false; + } + + if (!WaitForReady(registry_timeout)) { + ERROR("Failed to wait plugin %s ready with timeout %ld", m_name.c_str(), registry_timeout); + return false; + } + + if (!Configure(error)) { + ERROR("Failed to configure nri plugin %s", m_name.c_str()); + return false; + } + + return true; +} + +auto NRIPlugin::shutdown() -> void +{ + if (!Close()) { + ERROR("Failed to close plugin %s", m_name.c_str()); + } + + if (!Stop()) { + ERROR("Failed to stop plugin %s", m_name.c_str()); + } +} + +// create client connect +auto NRIPlugin::Connect(int64_t timeout) -> bool +{ + if (m_name.empty()) { + ERROR("Empty nri plugin name"); + return false; + } + + if (nri_plugin_connect(m_name.c_str(), m_sockFds[0], timeout * SECOND_TO_NANOS) != 0) { + ERROR("Failed to create a new client for plugin %s", m_name.c_str()); + return false; + } + + return true; +} + +// close a plugin shutting down its multiplexed ttrpc connections. +auto NRIPlugin::Close() -> bool +{ + if (IsClose()) { + return true; + } + + if (nri_plugin_disconnect(m_name.c_str()) != 0) { + ERROR("Failed to close plugin %s", m_name.c_str()); + return false; + } + + SetClose(); + return true; +} + +// stop a plugin (if it was launched by us) +auto NRIPlugin::Stop() -> bool +{ + if (m_external) { + return true; + } + + if (m_pid <= 0) { + WARN("Invalid pid %d", m_pid); + return false; + } + + int nret = kill(m_pid, SIGKILL); + if (nret < 0 && errno != ESRCH) { + SYSWARN("Can not kill process (pid=%d) with SIGKILL", m_pid); + return false; + } + + if (util_waitpid_with_timeout(m_pid, DefaultWaitTimeout, NULL) != 0) { + WARN("Failed to wait for plugin %s to exit", m_name.c_str()); + return false; + } + return true; +} + +// Name returns a string indentication for the plugin. +auto NRIPlugin::GetName() -> const std::string & +{ + return m_name; +} + +auto NRIPlugin::GetIndex() -> const std::string & +{ + return m_idx; +} + +auto NRIPlugin::GetPeerSockFd() -> uint32_t +{ + return m_sockFds[1]; +} + +auto NRIPlugin::GetQualifiedName() -> std::string +{ + return m_idx + "-" + m_name; +} + +void NRIPlugin::SetReady(void) +{ + std::unique_lock lock(m_readyMutex); + m_ready = true; + m_condition.notify_one(); +} + +void NRIPlugin::SetPid(int pid) +{ + m_pid = pid; +} + +auto NRIPlugin::CreateSocketPair() -> bool +{ + int fds[2]; + + if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) == -1) { + ERROR("Failed to create socketpair"); + return false; + } + + m_sockFds.push_back(fds[0]); + m_sockFds.push_back(fds[1]); + return true; +} + +auto NRIPlugin::Configure(Errors &error) -> bool +{ + auto req = makeUniquePtrCStructWrapper(free_nri_configure_request); + if (req == nullptr) { + ERROR("Out of memory"); + return false; + } + + req->get()->config = isula_strdup_s(m_config.c_str()); + req->get()->runtime_name = isula_strdup_s(NRIRruntime.c_str()); + req->get()->runtime_version = isula_strdup_s(NRIVersion.c_str()); + + nri_configure_response *resp = nullptr; + if (nri_plugin_configure(m_name.c_str(), req->get(), &resp) != 0) { + ERROR("Failed to configure plugin %s", m_name.c_str()); + return false; + } + + auto resp_wrapper = makeUniquePtrCStructWrapper(resp, free_nri_configure_response); + if (resp_wrapper == nullptr) { + ERROR("Out of memory"); + return false; + } + + EventMask events = resp_wrapper->get()->events; + if (events != 0) { + EventMask extra = events & ~ValidEvents; + if (extra != 0) { + ERROR("Invalid plugin events: %d", extra); + return false; + } + } else { + events = ValidEvents; + } + + m_events = events; + return true; +} + +auto NRIPlugin::Synchronize(std::vector &pods, std::vector &containers, + nri_container_update ***update, size_t update_len, Errors &error) -> bool +{ + size_t i; + + auto req = makeUniquePtrCStructWrapper(free_nri_synchronize_request); + if (req == nullptr) { + ERROR("Out of memory"); + return false; + } + + if (pods.size() != 0) { + req->get()->pods = (nri_pod_sandbox **)util_common_calloc_s(pods.size() * sizeof(nri_pod_sandbox *)); + if (req->get()->pods == nullptr) { + ERROR("Out of memory"); + return false; + } + + for (i = 0; i < pods.size(); i++) { + req->get()->pods[i] = pods[i]; + req->get()->pods_len++; + } + } + + if (containers.size() != 0) { + req->get()->containers = (nri_container **)util_common_calloc_s(containers.size() * sizeof(nri_container *)); + if (req->get()->containers == nullptr) { + ERROR("Out of memory"); + return false; + } + + for (i = 0; i < containers.size(); i++) { + req->get()->containers[i] = containers[i]; + req->get()->containers_len++; + } + } + + nri_synchronize_response *resp = nullptr; + if (nri_plugin_synchronize(m_name.c_str(), req->get(), &resp) != 0) { + ERROR("Failed to synchronize plugin %s", m_name.c_str()); + return false; + } + + auto resp_wrapper = makeUniquePtrCStructWrapper(resp, free_nri_synchronize_response); + if (resp_wrapper == nullptr) { + ERROR("Out of memory"); + return false; + } + + *update = resp->update; + resp->update = nullptr; + update_len = resp->update_len; + resp->update_len = 0; + return true; +} + +auto NRIPlugin::CreateContainer(nri_create_container_request *req, nri_create_container_response **resp, + Errors &error) -> bool +{ + if (req == nullptr) { + ERROR("Invalid input"); + return false; + } + + if (IsSetEvent(CREATE_CONTAINER) == false) { + return true; + } + + if (nri_plugin_create_container(m_name.c_str(), req, resp) != 0) { + ERROR("Failed to create container by plugin %s", m_name.c_str()); + return false; + } + + return true; +} + +auto NRIPlugin::UpdateContainer(nri_update_container_request *req, nri_update_container_response **resp, + Errors &error) -> bool +{ + if (req == nullptr) { + ERROR("Invalid input"); + return false; + } + + if (!IsSetEvent(UPDATE_CONTAINER)) { + return true; + } + + if (nri_plugin_update_container(m_name.c_str(), req, resp) != 0) { + ERROR("Failed to update container by plugin %s", m_name.c_str()); + return false; + } + return true; +} + +auto NRIPlugin::StopContainer(nri_stop_container_request *req, nri_stop_container_response **resp, + Errors &error) -> bool +{ + if (req == nullptr) { + ERROR("Invalid input"); + return false; + } + + if (!IsSetEvent(STOP_CONTAINER)) { + return true; + } + + if (nri_plugin_stop_container(m_name.c_str(), req, resp) != 0) { + ERROR("Failed to stop container by plugin %s", m_name.c_str()); + return false; + } + return true; +} + +// do nothing with event +auto NRIPlugin::StateChange(nri_state_change_event *evt, Errors &error) -> bool +{ + if (evt == nullptr) { + ERROR("Invalid input"); + return false; + } + + if (!IsSetEvent(evt->event)) { + return true; + } + + if (nri_plugin_state_change(m_name.c_str(), evt) != 0) { + ERROR("Failed to state change by plugin %s", m_name.c_str()); + return false; + } + return true; +} + +auto NRIPlugin::WaitForReady(int64_t timeout) -> bool +{ + auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(timeout * 1000); + std::unique_lock readyMutex(m_readyMutex); + + if (timeout == 0) { + m_condition.wait(readyMutex); + return true; + } + + if (m_condition.wait_until(readyMutex, deadline) == std::cv_status::timeout) { + return false; + } + + return true; +} + +auto NRIPlugin::IsSetEvent(EventMask e) -> bool +{ + return (m_events & (1 << (e - 1))) != 0; +} + +auto NRIPlugin::IsClose() -> bool +{ + ReadGuard lock(m_mutex); + return m_closed; +} + +void NRIPlugin::SetClose() +{ + WriteGuard lock(m_mutex); + m_closed = true; +} \ No newline at end of file diff --git a/src/daemon/nri/plugin.h b/src/daemon/nri/plugin.h index f60a9b3d..ed298be6 100644 --- a/src/daemon/nri/plugin.h +++ b/src/daemon/nri/plugin.h @@ -38,17 +38,14 @@ public: virtual ~NRIPlugin() = default; // wait for plugin to register, then configure it. auto Start(int64_t registry_timeout, int64_t request_timeout) -> bool; - // close a plugin shutting down its multiplexed ttrpc connections. - auto Close(void) -> bool; - // stop a plugin (if it was launched by us) - auto Stop(void) -> bool; + + auto shutdown() -> void; // Name returns a string indentication for the plugin. auto GetName(void) -> const std::string &; auto GetIndex(void) -> const std::string &; auto GetPeerSockFd(void) -> uint32_t; auto GetQualifiedName(void) -> std::string; - void SetReady(void); void SetPid(int pid); @@ -58,8 +55,8 @@ public: auto Configure(Errors &error) -> bool; // Only called in external plugin scenario - auto Synchronize(std::vector> pods, - std::vector> &containers, nri_container_update ***update, size_t update_len, + auto Synchronize(std::vector &pods, + std::vector &containers, nri_container_update ***update, size_t update_len, Errors &error) -> bool; auto CreateContainer(nri_create_container_request *req, nri_create_container_response **resp, Errors &error) -> bool; auto UpdateContainer(nri_update_container_request *req, nri_update_container_response **resp, Errors &error) -> bool; @@ -72,6 +69,13 @@ private: auto WaitForReady(int64_t timeout) -> bool; auto IsSetEvent(EventMask e) -> bool; + // close a plugin shutting down its multiplexed ttrpc connections. + auto Close(void) -> bool; + // stop a plugin (if it was launched by us) + auto Stop(void) -> bool; + + void SetClose(void); + private: RWMutex m_mutex; bool m_external; @@ -83,7 +87,7 @@ private: std::vector m_sockFds; std::string m_localFileName; std::string m_peerFileName; - // TODO:zhontao monitor? + // TODO: plugin monitor? bool m_closed; std::mutex m_readyMutex; bool m_ready; diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c index f81a9141..69f6dbf0 100644 --- a/src/utils/cutils/utils.c +++ b/src/utils/cutils/utils.c @@ -1705,4 +1705,4 @@ void set_child_process_pdeathsig(void) if (prctl(PR_SET_PDEATHSIG, SIGKILL) < 0) { SYSERROR("Failed to set child process pdeathsig"); } -} \ No newline at end of file +} -- 2.25.1