From 167af3ce0cff3906c9976b249432d41167b15eb2 Mon Sep 17 00:00:00 2001 From: jikai Date: Thu, 11 Jan 2024 17:06:57 +0800 Subject: [PATCH 24/43] add support for systemd cgroup driver Signed-off-by: jikai --- src/cmd/isulad-shim/process.c | 6 + src/cmd/isulad/isulad_commands.h | 2 + src/contrib/config/daemon.json | 3 +- src/daemon/common/cgroup.h | 4 + src/daemon/common/cgroup_v1.c | 116 ++++++++++++++++++ src/daemon/config/isulad_config.c | 23 ++++ src/daemon/config/isulad_config.h | 2 + .../cri/v1/cri_v1_runtime_runtime_service.cc | 25 ++++ .../cri/v1/cri_v1_runtime_runtime_service.h | 4 + .../cri/v1/v1_cri_runtime_manager_service.cc | 12 ++ .../cri/v1/v1_cri_runtime_manager_service.h | 2 + .../entry/cri/v1/v1_cri_runtime_service.h | 2 + .../cri/v1/v1_cri_runtime_service_impl.cc | 5 + .../cri/v1/v1_cri_runtime_service_impl.h | 2 + src/daemon/executor/container_cb/execution.c | 26 ++++ .../modules/runtime/engines/lcr/lcr_rt_ops.c | 7 ++ .../modules/runtime/isula/isula_rt_ops.c | 1 + src/daemon/modules/runtime/shim/shim_rt_ops.c | 6 + src/daemon/modules/spec/specs.c | 22 +++- 19 files changed, 268 insertions(+), 2 deletions(-) diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c index bf67c414..8a4ca175 100644 --- a/src/cmd/isulad-shim/process.c +++ b/src/cmd/isulad-shim/process.c @@ -1287,6 +1287,12 @@ static void get_runtime_cmd(process_t *p, const char *log_path, const char *pid_ params[i++] = p->state->cwd; } } else { + // the --systemd-cgroup argument is not in oci spec, but up to now, + // the latest version of runc, crun, youki, runsc, kata-runtime all support this argument + // should ensure that this is supported for oci runtime + if (p->state->systemd_cgroup) { + params[i++] = "--systemd-cgroup"; + } params[i++] = "create"; params[i++] = "--bundle"; params[i++] = p->bundle; diff --git a/src/cmd/isulad/isulad_commands.h b/src/cmd/isulad/isulad_commands.h index ba4b838d..cf5f65f5 100644 --- a/src/cmd/isulad/isulad_commands.h +++ b/src/cmd/isulad/isulad_commands.h @@ -226,6 +226,8 @@ int command_default_ulimit_append(command_option_t *option, const char *arg); &(cmdargs)->json_confs->storage_opts, \ "Storage driver options", \ command_append_array }, \ + { CMD_OPT_TYPE_BOOL, false, "systemd-cgroup", 0, &(cmdargs)->json_confs->systemd_cgroup, \ + "Use systemd cgroup driver(default false)", NULL }, \ SUP_GROUPS_OPT(cmdargs) \ { CMD_OPT_TYPE_CALLBACK, \ false, \ diff --git a/src/contrib/config/daemon.json b/src/contrib/config/daemon.json index 966e016a..69362c26 100644 --- a/src/contrib/config/daemon.json +++ b/src/contrib/config/daemon.json @@ -36,5 +36,6 @@ "cri-runtimes": { "kata": "io.containerd.kata.v2" }, - "enable-cri-v1": false + "enable-cri-v1": false, + "systemd-cgroup": false } diff --git a/src/daemon/common/cgroup.h b/src/daemon/common/cgroup.h index 6664fb15..fa20f42c 100644 --- a/src/daemon/common/cgroup.h +++ b/src/daemon/common/cgroup.h @@ -141,6 +141,10 @@ typedef struct { int common_get_cgroup_v1_metrics(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics); +char *common_get_init_cgroup(const char *subsystem); + +char *common_get_own_cgroup(const char *subsystem); + #ifdef __cplusplus } #endif diff --git a/src/daemon/common/cgroup_v1.c b/src/daemon/common/cgroup_v1.c index 9004becc..e38fc03e 100644 --- a/src/daemon/common/cgroup_v1.c +++ b/src/daemon/common/cgroup_v1.c @@ -554,3 +554,119 @@ int common_get_cgroup_v1_metrics(const char *cgroup_path, cgroup_metrics_t *cgro return 0; } +/* parse cgroup files, such as /proc/self/cgroup or /proc/1/cgroup */ +static int parse_cgroup_file(const char *path, char ***nlist, char ***plist) +{ + int ret = 0; + size_t length = 0; + __isula_auto_file FILE *fp = NULL; + __isula_auto_free char *pline = NULL; + + fp = util_fopen(path, "r"); + if (fp == NULL) { + return -1; + } + + while (getline(&pline, &length, fp) != -1) { + char *pos = NULL; + char *pos2 = NULL; + char *pos3 = NULL; + char *ptoken = NULL; + char *psave = NULL; + pos = strchr(pline, ':'); + if (pos == NULL) { + ERROR("Invalid cgroup entry: must contain at least two colons: %s", pline); + ret = -1; + goto out; + } + pos++; + pos2 = strchr(pos, ':'); + if (pos2 == NULL) { + ERROR("Invalid cgroup entry: must contain at least two colons: %s", pline); + ret = -1; + goto out; + } + pos3 = strchr(pos2, '\n'); + if (pos3 != NULL) { + *pos3 = '\0'; + } + *pos2 = '\0'; + + if ((pos2 - pos) == 0) { + INFO("Cgroup entry: %s not supported by cgroup v1", pline); + continue; + } + + for (ptoken = strtok_r(pos, ",", &psave); ptoken; ptoken = strtok_r(NULL, ",", &psave)) { + ret = util_array_append(nlist, ptoken); + if (ret != 0) { + ERROR("Failed to append string"); + goto out; + } + + ret = util_array_append(plist, pos2 + 1); + if (ret != 0) { + ERROR("Failed to append string"); + goto out; + } + } + } + +out: + if (ret != 0) { + util_free_array(*nlist); + *nlist = NULL; + util_free_array(*plist); + *plist = NULL; + } + return ret; +} + +static char *common_get_cgroup_path(const char *path, const char *subsystem) +{ + char **nlist = NULL, **plist = NULL; + size_t i = 0; + char *res = NULL; + if (path == NULL) { + ERROR("Invalid NULL param"); + return NULL; + } + + if (parse_cgroup_file(path, &nlist, &plist) < 0) { + return NULL; + } + + for (i = 0; i < util_array_len((const char **)nlist); i++) { + const char *prefix = "name="; + bool find_sub = (strcmp(nlist[i], subsystem) == 0 || (strncmp(nlist[i], prefix, strlen(prefix)) == 0 + && strcmp(nlist[i]+strlen(prefix), subsystem) == 0)); + if (find_sub) { + res = util_strdup_s(plist[i]); + break; + } + } + + util_free_array(nlist); + util_free_array(plist); + return res; +} + +char *common_get_init_cgroup(const char *subsystem) +{ + if (common_get_cgroup_version() != CGROUP_VERSION_1) { + ERROR("Not implemented for cgroup v2 hierarchy"); + return NULL; + } + + return common_get_cgroup_path("/proc/1/cgroup", subsystem); +} + +char *common_get_own_cgroup(const char *subsystem) +{ + if (common_get_cgroup_version() != CGROUP_VERSION_1) { + ERROR("Not implemented for cgroup v2 hierarchy"); + return NULL; + } + + return common_get_cgroup_path("/proc/self/cgroup", subsystem); +} diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c index 51758adb..8179558e 100644 --- a/src/daemon/config/isulad_config.c +++ b/src/daemon/config/isulad_config.c @@ -1762,6 +1762,8 @@ int merge_json_confs_into_global(struct service_arguments *args) args->json_confs->enable_cri_v1 = tmp_json_confs->enable_cri_v1; #endif + args->json_confs->systemd_cgroup = tmp_json_confs->systemd_cgroup; + if (merge_cri_runtimes_into_global(args, tmp_json_confs)) { ret = -1; goto out; @@ -1895,3 +1897,24 @@ isulad_daemon_constants *get_isulad_daemon_constants(void) { return g_isulad_daemon_constants; } + +bool conf_get_systemd_cgroup() +{ + bool systemd_cgroup = false; + struct service_arguments *conf = NULL; + + if (isulad_server_conf_rdlock() != 0) { + return false; + } + + conf = conf_get_server_conf(); + if (conf == NULL || conf->json_confs == NULL) { + goto out; + } + + systemd_cgroup = conf->json_confs->systemd_cgroup; + +out: + (void)isulad_server_conf_unlock(); + return systemd_cgroup; +} diff --git a/src/daemon/config/isulad_config.h b/src/daemon/config/isulad_config.h index 459ea331..f29cd564 100644 --- a/src/daemon/config/isulad_config.h +++ b/src/daemon/config/isulad_config.h @@ -114,6 +114,8 @@ char *conf_get_isulad_monitor_fifo_path(void); int init_isulad_daemon_constants(void); isulad_daemon_constants *get_isulad_daemon_constants(void); +bool conf_get_systemd_cgroup(void); + #ifdef __cplusplus } #endif diff --git a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc index ba9459f6..76e393f3 100644 --- a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc +++ b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc @@ -632,3 +632,28 @@ grpc::Status RuntimeV1RuntimeServiceImpl::Status(grpc::ServerContext *context, return grpc::Status::OK; } + +grpc::Status +RuntimeV1RuntimeServiceImpl::RuntimeConfig(grpc::ServerContext *context, + const runtime::v1::RuntimeConfigRequest *request, + runtime::v1::RuntimeConfigResponse *reply) +{ + Errors error; + + if (request == nullptr) { + ERROR("Invalid input arguments"); + return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, "Invalid input arguments"); + } + + EVENT("Event: {Object: CRI, Type: Runtime Config}"); + + m_rService->RuntimeConfig(reply, error); + if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to get runtime config:%s", error.GetMessage().c_str()); + return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); + } + + EVENT("Event: {Object: CRI, Type: Runtime Config}"); + + return grpc::Status::OK; +} diff --git a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h index 09ebe68b..52cc6b99 100644 --- a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h +++ b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h @@ -101,6 +101,10 @@ public: grpc::Status Status(grpc::ServerContext *context, const runtime::v1::StatusRequest *request, runtime::v1::StatusResponse *reply) override; + grpc::Status RuntimeConfig(grpc::ServerContext *context, + const runtime::v1::RuntimeConfigRequest *request, + runtime::v1::RuntimeConfigResponse *reply) override; + private: std::unique_ptr m_rService; }; diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.cc index 1afac3e2..de489aae 100644 --- a/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.cc +++ b/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.cc @@ -16,6 +16,7 @@ #include "isula_libutils/log.h" #include "v1_cri_helpers.h" #include "cri_helpers.h" +#include "isulad_config.h" namespace CRIV1 { void RuntimeManagerService::UpdateRuntimeConfig(const runtime::v1::RuntimeConfig &config, Errors & /*error*/) @@ -69,4 +70,15 @@ auto RuntimeManagerService::Status(Errors &error) -> std::unique_ptrmutable_linux()->set_cgroup_driver(conf_get_systemd_cgroup() ? runtime::v1::SYSTEMD : runtime::v1::CGROUPFS); +} + } // namespace CRI diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.h b/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.h index 3800ec30..2160064e 100644 --- a/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.h +++ b/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.h @@ -39,6 +39,8 @@ public: auto Status(Errors &error) -> std::unique_ptr; + void RuntimeConfig(runtime::v1::RuntimeConfigResponse *reply, Errors &error); + private: service_executor_t *m_cb; std::shared_ptr m_pluginManager; diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service.h b/src/daemon/entry/cri/v1/v1_cri_runtime_service.h index 87394173..839f6724 100644 --- a/src/daemon/entry/cri/v1/v1_cri_runtime_service.h +++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service.h @@ -89,6 +89,8 @@ public: virtual void UpdateRuntimeConfig(const runtime::v1::RuntimeConfig &config, Errors &error) = 0; virtual auto Status(Errors &error) -> std::unique_ptr = 0; + + virtual void RuntimeConfig(runtime::v1::RuntimeConfigResponse *reply, Errors &error) = 0; }; } // namespace CRIV1 #endif // DAEMON_ENTRY_CRI_V1_CRI_RUNTIME_SERVICE_INTERFACE_H diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc index 67cda5ed..aa5ae516 100644 --- a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc +++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc @@ -161,4 +161,9 @@ auto CRIRuntimeServiceImpl::Status(Errors &error) -> std::unique_ptrStatus(error); } +void CRIRuntimeServiceImpl::RuntimeConfig(runtime::v1::RuntimeConfigResponse *reply, Errors &error) +{ + m_runtimeManager->RuntimeConfig(reply, error); +} + } // namespace CRIV1 diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h index 23866648..0a25749f 100644 --- a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h +++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h @@ -92,6 +92,8 @@ public: auto Status(Errors &error) -> std::unique_ptr override; + void RuntimeConfig(runtime::v1::RuntimeConfigResponse *reply, Errors &error) override; + protected: std::unique_ptr m_runtimeVersioner; std::unique_ptr m_containerManager; diff --git a/src/daemon/executor/container_cb/execution.c b/src/daemon/executor/container_cb/execution.c index 63d8143c..7ed8e837 100644 --- a/src/daemon/executor/container_cb/execution.c +++ b/src/daemon/executor/container_cb/execution.c @@ -427,6 +427,32 @@ static int cpurt_controller_init(const char *id, const host_config *host_spec) return 0; } + if (conf_get_systemd_cgroup()) { + // currently it is the same as docker, yet it is unclear that + // if systemd cgroup is used and cgroup parent is set to a slice rather than system.slice + // should iSulad set cpu.rt_runtime_us and cpu.rt_period_us for the parent path? + // in fact, even if system.slice is used, + // cpu.rt_runtime_us and cpu.rt_period_us might still needed to be set manually + __isula_auto_free char *init_cgroup = common_get_init_cgroup("cpu"); + if (init_cgroup == NULL) { + ERROR("Failed to get init cgroup"); + return -1; + } + // make sure that the own cgroup path for cpu existed + __isula_auto_free char *own_cgroup = common_get_own_cgroup("cpu"); + if (own_cgroup == NULL) { + ERROR("Failed to get own cgroup"); + return -1; + } + char *new_cgroups_path = util_path_join(init_cgroup, cgroups_path); + if (new_cgroups_path == NULL) { + ERROR("Failed to join path"); + return -1; + } + free(cgroups_path); + cgroups_path = new_cgroups_path; + } + mnt_root = sysinfo_cgroup_controller_cpurt_mnt_path(); if (mnt_root == NULL) { ERROR("Failed to get cpu rt controller mnt root path"); diff --git a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c index 8c3c5f1a..6b862958 100644 --- a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c +++ b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c @@ -53,6 +53,13 @@ int rt_lcr_create(const char *name, const char *runtime, const rt_create_params_ char *runtime_root = NULL; struct engine_operation *engine_ops = NULL; + if (conf_get_systemd_cgroup()) { + ERROR("Systemd cgroup not supported for lcr runtime"); + isulad_set_error_message("Systemd cgroup not supported for lcr runtime"); + ret = -1; + goto out; + } + runtime_root = conf_get_routine_rootdir(runtime); if (runtime_root == NULL) { ERROR("Root path is NULL"); diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c index 745154bb..b9aba3e3 100644 --- a/src/daemon/modules/runtime/isula/isula_rt_ops.c +++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c @@ -1157,6 +1157,7 @@ int rt_isula_create(const char *id, const char *runtime, const rt_create_params_ p.runtime_args = (char **)runtime_args; p.runtime_args_len = runtime_args_len; p.attach_socket = attach_socket; + p.systemd_cgroup = conf_get_systemd_cgroup(); copy_process(&p, config->process); copy_annotations(&p, config->annotations); diff --git a/src/daemon/modules/runtime/shim/shim_rt_ops.c b/src/daemon/modules/runtime/shim/shim_rt_ops.c index 81daf224..fc4d8e3a 100644 --- a/src/daemon/modules/runtime/shim/shim_rt_ops.c +++ b/src/daemon/modules/runtime/shim/shim_rt_ops.c @@ -35,6 +35,7 @@ #include "engine.h" #include "shim_rt_monitor.h" #include "supervisor.h" +#include "isulad_config.h" #define EXIT_SIGNAL_OFFSET_X 128 @@ -370,6 +371,11 @@ int rt_shim_create(const char *id, const char *runtime, const rt_create_params_t return -1; } + if (conf_get_systemd_cgroup()) { + ERROR("ShimV2 does not support systemd cgroup yet"); + return -1; + } + exit_fifo_path = util_path_dir(params->exit_fifo); if (exit_fifo_path == NULL) { ERROR("%s: failed to get exit fifo dir from %s", id, params->exit_fifo); diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c index 464b4fb4..b4d2b0f6 100644 --- a/src/daemon/modules/spec/specs.c +++ b/src/daemon/modules/spec/specs.c @@ -188,7 +188,10 @@ static char *do_get_container_cgroup_path(const host_config *host_spec) } if (path == NULL) { - // third, all faild, just use default '/isulad' + // third, all faild, just use default '/isulad' for cgroupfs or "system.slice" for systemd + if (conf_get_systemd_cgroup()) { + return util_strdup_s("system.slice"); + } path = util_strdup_s("/isulad"); } @@ -2288,6 +2291,23 @@ char *merge_container_cgroups_path(const char *id, const host_config *host_spec) path = do_get_container_cgroup_path(host_spec); + if (conf_get_systemd_cgroup()) { + // systemd cgroup path has the form of [slice]:[prefix]:[name] +#define SYSTEMD_CGROUP_PATH_LEN 3 + if (!util_has_suffix(path, ".slice")) { + ERROR("Invalid cgroup path %s for systemd", path); + isulad_set_error_message("Invalid cgroup path %s for systemd", path); + return NULL; + } + + // slice must not contain slashes + // convert test.slice/test-a.slice/test-a-b.slice to become test-a-b.slice + __isula_auto_free char *base = util_path_base(path); + const char *isulad_prefix = "isulad"; + const char *parts[SYSTEMD_CGROUP_PATH_LEN] = {base, isulad_prefix, id}; + return util_string_join(":", parts, SYSTEMD_CGROUP_PATH_LEN); + } + return util_path_join(path, id); } -- 2.34.1