From 0674bfac4dd1ab812432334c779ab718dc54bc8b Mon Sep 17 00:00:00 2001 From: liuxu Date: Thu, 11 Apr 2024 11:02:19 +0800 Subject: [PATCH 68/69] cdi:invoke cdi operate when init isulad and create container --- src/cmd/isulad/main.c | 11 +++ src/daemon/common/cri/v1/v1_cri_helpers.cc | 79 +++++++++++++++++++ src/daemon/common/cri/v1/v1_cri_helpers.h | 3 + src/daemon/config/daemon_arguments.c | 4 + src/daemon/config/isulad_config.c | 8 ++ .../v1/v1_cri_container_manager_service.cc | 8 ++ .../executor/container_cb/execution_create.c | 9 +++ .../modules/service/service_container.c | 10 +++ src/daemon/modules/spec/specs_mount.c | 43 +++++++++- src/daemon/modules/spec/specs_mount.h | 4 + src/daemon/modules/spec/verify.c | 2 +- 11 files changed, 179 insertions(+), 2 deletions(-) diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c index 9fa87bdb..3e2249d7 100644 --- a/src/cmd/isulad/main.c +++ b/src/cmd/isulad/main.c @@ -83,6 +83,9 @@ #endif #include "id_name_manager.h" #include "cgroup.h" +#ifdef ENABLE_CDI +#include "cdi_operate_api.h" +#endif /* ENABLE_CDI */ sem_t g_daemon_shutdown_sem; sem_t g_daemon_wait_shutdown_sem; @@ -1400,6 +1403,14 @@ static int isulad_server_init_common() } #endif +#ifdef ENABLE_CDI + if (args->json_confs->enable_cdi && + cdi_operate_registry_init(args->json_confs->cdi_spec_dirs, args->json_confs->cdi_spec_dirs_len) != 0) { + ERROR("Failed to init CDI module"); + goto out; + } +#endif /* ENABLE_CDI */ + if (spec_module_init() != 0) { ERROR("Failed to init spec module"); goto out; diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.cc b/src/daemon/common/cri/v1/v1_cri_helpers.cc index ea5c8bb5..520d23d4 100644 --- a/src/daemon/common/cri/v1/v1_cri_helpers.cc +++ b/src/daemon/common/cri/v1/v1_cri_helpers.cc @@ -22,6 +22,7 @@ #include #include +#include #include "v1_cri_security_context.h" #include "cri_helpers.h" @@ -33,6 +34,9 @@ #include "isulad_config.h" #include "sha256.h" #include "v1_naming.h" +#ifdef ENABLE_CDI +#include "cdi_operate_api.h" +#endif /* ENABLE_CDI */ namespace CRIHelpersV1 { @@ -666,4 +670,79 @@ std::unique_ptr GetContainerStatus(service_executo return contStatus; } +#ifdef ENABLE_CDI +static int InsertCDIDevices(std::unordered_set &fromCRI, const std::string &devName, + string_array *requested, Errors &err) +{ + if (fromCRI.find(devName) == fromCRI.end()) { + fromCRI.insert(devName); + if (util_append_string_array(requested, devName.c_str()) != 0) { + ERROR("Out of memory"); + err.Errorf("Out of memory"); + return -1; + } + DEBUG("Appended device: %s", devName.c_str()); + } else { + INFO("Skipping duplicate CDI device %s", devName.c_str()); + } + return 0; +} + +void GenerateCDIRequestedDevices(const runtime::v1::ContainerConfig &config, host_config *hostconfig, Errors &err) +{ + std::unordered_set fromCRI; + __isula_auto_string_array_t string_array *requested = nullptr; + __isula_auto_string_array_t string_array *keys = nullptr; + __isula_auto_string_array_t string_array *devices = nullptr; + json_map_string_string *annotations = nullptr; + __isula_auto_free char *error = nullptr; + + if (hostconfig == nullptr) { + ERROR("Invalid input arguments"); + err.Errorf("Invalid input arguments"); + return; + } + + if (config.cdi_devices().empty() && config.annotations().empty()) { + return; + } + requested = (string_array *)util_common_calloc_s(sizeof(*requested)); + if (requested == nullptr) { + ERROR("Out of memory"); + err.Errorf("Out of memory"); + return; + } + if (!config.cdi_devices().empty()) { + for (int i = 0; i < config.cdi_devices().size(); i++) { + if (InsertCDIDevices(fromCRI, config.cdi_devices(i).name(), requested, err) != 0) { + goto free_out; + } + } + } + if (!config.annotations().empty()) { + annotations = CRIHelpers::MakeAnnotations(config.annotations(), err); + if (err.NotEmpty()) { + goto free_out; + } + if (cdi_operate_parse_annotations(annotations, &keys, &devices, &error) != 0) { + ERROR("Failed to parse CDI annotations: %s", error); + err.Errorf("Failed to parse CDI annotations: %s", error); + goto free_out; + } + for (size_t i = 0; i < devices->len; i++) { + if (InsertCDIDevices(fromCRI, std::string(devices->items[i]), requested, err) != 0) { + goto free_out; + } + } + } + hostconfig->cdi_requested_devices = requested->items; + requested->items = nullptr; + hostconfig->cdi_requested_devices_len = requested->len; + requested->len = 0; + +free_out: + free_json_map_string_string(annotations); +} +#endif /* ENABLE_CDI */ + } // v1 namespace CRIHelpers diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.h b/src/daemon/common/cri/v1/v1_cri_helpers.h index 1578c428..22cffd0d 100644 --- a/src/daemon/common/cri/v1/v1_cri_helpers.h +++ b/src/daemon/common/cri/v1/v1_cri_helpers.h @@ -79,6 +79,9 @@ std::string CRISandboxerConvert(const std::string &runtime); void ApplySandboxSecurityContextToHostConfig(const runtime::v1::LinuxSandboxSecurityContext &context, host_config *hc, Errors &error); +#ifdef ENABLE_CDI +void GenerateCDIRequestedDevices(const runtime::v1::ContainerConfig &config, host_config *hostconfig, Errors &err); +#endif /* ENABLE_CDI */ auto GetContainerStatus(service_executor_t *m_cb, const std::string &containerID, Errors &error) -> std::unique_ptr; diff --git a/src/daemon/config/daemon_arguments.c b/src/daemon/config/daemon_arguments.c index 0ae6268a..ef15934a 100644 --- a/src/daemon/config/daemon_arguments.c +++ b/src/daemon/config/daemon_arguments.c @@ -173,6 +173,10 @@ int service_arguments_init(struct service_arguments *args) goto free_out; } +#ifdef ENABLE_CDI + args->json_confs->enable_cdi = false; +#endif /* ENABLE_CDI */ + ret = 0; free_out: diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c index 778ff921..695a0d95 100644 --- a/src/daemon/config/isulad_config.c +++ b/src/daemon/config/isulad_config.c @@ -1830,6 +1830,14 @@ int merge_json_confs_into_global(struct service_arguments *args) args->json_confs->metrics_port = tmp_json_confs->metrics_port; #endif +#ifdef ENABLE_CDI + args->json_confs->enable_cdi = tmp_json_confs->enable_cdi; + args->json_confs->cdi_spec_dirs = tmp_json_confs->cdi_spec_dirs; + tmp_json_confs->cdi_spec_dirs = NULL; + args->json_confs->cdi_spec_dirs_len = tmp_json_confs->cdi_spec_dirs_len; + tmp_json_confs->cdi_spec_dirs_len = 0; +#endif /* ENABLE_CDI */ + out: free(err); free_isulad_daemon_configs(tmp_json_confs); diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc index e86dafae..1097c32c 100644 --- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc @@ -199,6 +199,14 @@ auto ContainerManagerService::GenerateCreateContainerHostConfig( } } +#ifdef ENABLE_CDI + CRIHelpersV1::GenerateCDIRequestedDevices(containerConfig, hostconfig, error); + if (error.NotEmpty()) { + ERROR("Failed to generate CDI requested devices"); + goto cleanup; + } +#endif /* ENABLE_CDI */ + return hostconfig; cleanup: diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c index a9102226..785b4e27 100644 --- a/src/daemon/executor/container_cb/execution_create.c +++ b/src/daemon/executor/container_cb/execution_create.c @@ -63,6 +63,7 @@ #include "runtime_api.h" #include "id_name_manager.h" #include "mailbox.h" +#include "specs_mount.h" #ifdef ENABLE_CRI_API_V1 static bool validate_sandbox_info(const container_sandbox_info *sandbox) @@ -512,6 +513,14 @@ static oci_runtime_spec *generate_oci_config(host_config *host_spec, const char goto error_out; } +#ifdef ENABLE_CDI + ret = inject_CDI_devcies_for_oci_spec(oci_spec, host_spec); + if (ret != 0) { + ERROR("Failed to inject CDI devices"); + goto error_out; + } +#endif /* ENABLE_CDI */ + return oci_spec; error_out: diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c index eb7ce4f4..b19a134a 100644 --- a/src/daemon/modules/service/service_container.c +++ b/src/daemon/modules/service/service_container.c @@ -2003,6 +2003,16 @@ static defs_process *make_exec_process_spec(const container_config *container_sp } spec->no_new_privileges = oci_spec->process->no_new_privileges; + +#ifdef ENABLE_CDI + // extend step: merge env from oci_spec which comes from injected devices + ret = defs_process_add_multiple_env(spec, (const char **)oci_spec->process->env, + oci_spec->process->env_len); + if (ret != 0) { + ERROR("Failed to dup oci env for exec process spec"); + goto err_out; + } +#endif /* ENABLE_CDI */ } // for oci runtime: diff --git a/src/daemon/modules/spec/specs_mount.c b/src/daemon/modules/spec/specs_mount.c index 50ee9a85..12bd261b 100644 --- a/src/daemon/modules/spec/specs_mount.c +++ b/src/daemon/modules/spec/specs_mount.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -54,6 +55,9 @@ #include "volume_api.h" #include "parse_volume.h" #include "specs_api.h" +#ifdef ENABLE_CDI +#include "cdi_operate_api.h" +#endif /* ENABLE_CDI */ enum update_rw { update_rw_untouch, @@ -3582,6 +3586,15 @@ int update_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostcon oci_spec->linux->resources->devices_len += 1; } + // extend step: inject CDI devcies +#ifdef ENABLE_CDI + ret = inject_CDI_devcies_for_oci_spec(oci_spec, hostconfig); + if (ret != 0) { + ERROR("Failed to inject CDI devices"); + return -1; + } +#endif /* ENABLE_CDI */ + // Step8: do update devices and cgroup device rules at here if (hostconfig->privileged) { // Step8.1: for priviledged container, we should merge all devices under /dev @@ -3592,4 +3605,32 @@ int update_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostcon } return ret; -} \ No newline at end of file +} + +#ifdef ENABLE_CDI +int inject_CDI_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostconfig) +{ + int ret = 0; + string_array devices_array = { 0 }; + __isula_auto_free char *error = NULL; + + if (oci_spec == NULL || hostconfig == NULL) { + ERROR("Invalid params"); + return -1; + } + if (hostconfig->cdi_requested_devices == NULL) { + return 0; + } + devices_array.items = hostconfig->cdi_requested_devices; + devices_array.len = hostconfig->cdi_requested_devices_len; + devices_array.cap = hostconfig->cdi_requested_devices_len; + if (cdi_operate_refresh() != 0) { + WARN("CDI registry has errors, please check past logs"); + } + if (cdi_operate_inject_devices(oci_spec, &devices_array) != 0) { + ERROR("Failed to inject CDI devices"); + ret = -1; + } + return ret; +} +#endif /* ENABLE_CDI */ \ No newline at end of file diff --git a/src/daemon/modules/spec/specs_mount.h b/src/daemon/modules/spec/specs_mount.h index b742ca35..1406c557 100644 --- a/src/daemon/modules/spec/specs_mount.h +++ b/src/daemon/modules/spec/specs_mount.h @@ -49,6 +49,10 @@ int setup_ipc_dirs(host_config *host_spec, container_config_v2_common_config *v2 int update_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostconfig); +#ifdef ENABLE_CDI +int inject_CDI_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostconfig); +#endif /* ENABLE_CDI */ + #ifdef __cplusplus } #endif diff --git a/src/daemon/modules/spec/verify.c b/src/daemon/modules/spec/verify.c index af790d6e..617b7f23 100644 --- a/src/daemon/modules/spec/verify.c +++ b/src/daemon/modules/spec/verify.c @@ -1518,7 +1518,7 @@ static int verify_custom_mount(defs_mount **mounts, size_t len) for (i = 0; i < len; ++i) { iter = *(mounts + i); - if (iter == NULL || strcmp(iter->type, MOUNT_TYPE_BIND)) { + if (iter == NULL || iter->type == NULL || strcmp(iter->type, MOUNT_TYPE_BIND)) { continue; } -- 2.34.1