path: root/0024-add-support-for-systemd-cgroup-driver.patch

From 167af3ce0cff3906c9976b249432d41167b15eb2 Mon Sep 17 00:00:00 2001
From: jikai <jikai11@huawei.com>
Date: Thu, 11 Jan 2024 17:06:57 +0800
Subject: [PATCH 24/43] add support for systemd cgroup driver

Signed-off-by: jikai <jikai11@huawei.com>
---
 src/cmd/isulad-shim/process.c                 |   6 +
 src/cmd/isulad/isulad_commands.h              |   2 +
 src/contrib/config/daemon.json                |   3 +-
 src/daemon/common/cgroup.h                    |   4 +
 src/daemon/common/cgroup_v1.c                 | 116 ++++++++++++++++++
 src/daemon/config/isulad_config.c             |  23 ++++
 src/daemon/config/isulad_config.h             |   2 +
 .../cri/v1/cri_v1_runtime_runtime_service.cc  |  25 ++++
 .../cri/v1/cri_v1_runtime_runtime_service.h   |   4 +
 .../cri/v1/v1_cri_runtime_manager_service.cc  |  12 ++
 .../cri/v1/v1_cri_runtime_manager_service.h   |   2 +
 .../entry/cri/v1/v1_cri_runtime_service.h     |   2 +
 .../cri/v1/v1_cri_runtime_service_impl.cc     |   5 +
 .../cri/v1/v1_cri_runtime_service_impl.h      |   2 +
 src/daemon/executor/container_cb/execution.c  |  26 ++++
 .../modules/runtime/engines/lcr/lcr_rt_ops.c  |   7 ++
 .../modules/runtime/isula/isula_rt_ops.c      |   1 +
 src/daemon/modules/runtime/shim/shim_rt_ops.c |   6 +
 src/daemon/modules/spec/specs.c               |  22 +++-
 19 files changed, 268 insertions(+), 2 deletions(-)

diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c
index bf67c414..8a4ca175 100644
--- a/src/cmd/isulad-shim/process.c
+++ b/src/cmd/isulad-shim/process.c
@@ -1287,6 +1287,12 @@ static void get_runtime_cmd(process_t *p, const char *log_path, const char *pid_
             params[i++] = p->state->cwd;
         }
     } else {
+        // the --systemd-cgroup argument is not in oci spec, but up to now,
+        // the latest version of runc, crun, youki, runsc, kata-runtime all support this argument
+        // should ensure that this is supported for oci runtime
+        if (p->state->systemd_cgroup) {
+            params[i++] = "--systemd-cgroup";
+        }
         params[i++] = "create";
         params[i++] = "--bundle";
         params[i++] = p->bundle;
diff --git a/src/cmd/isulad/isulad_commands.h b/src/cmd/isulad/isulad_commands.h
index ba4b838d..cf5f65f5 100644
--- a/src/cmd/isulad/isulad_commands.h
+++ b/src/cmd/isulad/isulad_commands.h
@@ -226,6 +226,8 @@ int command_default_ulimit_append(command_option_t *option, const char *arg);
       &(cmdargs)->json_confs->storage_opts,                                                                       \
       "Storage driver options",                                                                                   \
       command_append_array },                                                                                     \
+    { CMD_OPT_TYPE_BOOL, false, "systemd-cgroup", 0, &(cmdargs)->json_confs->systemd_cgroup,                      \
+      "Use systemd cgroup driver(default false)", NULL },                                                         \
     SUP_GROUPS_OPT(cmdargs)                                                                                       \
     { CMD_OPT_TYPE_CALLBACK,                                                                                      \
       false,                                                                                                      \
diff --git a/src/contrib/config/daemon.json b/src/contrib/config/daemon.json
index 966e016a..69362c26 100644
--- a/src/contrib/config/daemon.json
+++ b/src/contrib/config/daemon.json
@@ -36,5 +36,6 @@
     "cri-runtimes": {
         "kata": "io.containerd.kata.v2"
     },
-    "enable-cri-v1": false
+    "enable-cri-v1": false,
+    "systemd-cgroup": false
 }
diff --git a/src/daemon/common/cgroup.h b/src/daemon/common/cgroup.h
index 6664fb15..fa20f42c 100644
--- a/src/daemon/common/cgroup.h
+++ b/src/daemon/common/cgroup.h
@@ -141,6 +141,10 @@ typedef struct {
 
 int common_get_cgroup_v1_metrics(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics);
 
+char *common_get_init_cgroup(const char *subsystem);
+
+char *common_get_own_cgroup(const char *subsystem);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/daemon/common/cgroup_v1.c b/src/daemon/common/cgroup_v1.c
index 9004becc..e38fc03e 100644
--- a/src/daemon/common/cgroup_v1.c
+++ b/src/daemon/common/cgroup_v1.c
@@ -554,3 +554,119 @@ int common_get_cgroup_v1_metrics(const char *cgroup_path, cgroup_metrics_t *cgro
     return 0;
 }
 
+/* parse cgroup files, such as /proc/self/cgroup or /proc/1/cgroup */
+static int parse_cgroup_file(const char *path, char ***nlist, char ***plist)
+{
+    int ret = 0;
+    size_t length = 0;
+    __isula_auto_file FILE *fp = NULL;
+    __isula_auto_free char *pline = NULL;
+
+    fp = util_fopen(path, "r");
+    if (fp == NULL) {
+        return -1;
+    }
+
+    while (getline(&pline, &length, fp) != -1) {
+        char *pos = NULL;
+        char *pos2 = NULL;
+        char *pos3 = NULL;
+        char *ptoken = NULL;
+        char *psave = NULL;
+        pos = strchr(pline, ':');
+        if (pos == NULL) {
+            ERROR("Invalid cgroup entry: must contain at least two colons: %s", pline);
+            ret = -1;
+            goto out;
+        }
+        pos++;
+        pos2 = strchr(pos, ':');
+        if (pos2 == NULL) {
+            ERROR("Invalid cgroup entry: must contain at least two colons: %s", pline);
+            ret = -1;
+            goto out;
+        }
+        pos3 = strchr(pos2, '\n');
+        if (pos3 != NULL) {
+            *pos3 = '\0';
+        }
+        *pos2 = '\0';
+
+        if ((pos2 - pos) == 0) {
+            INFO("Cgroup entry: %s not supported by cgroup v1", pline);
+            continue;
+        }
+
+        for (ptoken = strtok_r(pos, ",", &psave); ptoken; ptoken = strtok_r(NULL, ",", &psave)) {
+            ret = util_array_append(nlist, ptoken);
+            if (ret != 0) {
+                ERROR("Failed to append string");
+                goto out;
+            }
+
+            ret = util_array_append(plist, pos2 + 1);
+            if (ret != 0) {
+                ERROR("Failed to append string");
+                goto out;
+            }
+        }
+    }
+
+out:
+    if (ret != 0) {
+        util_free_array(*nlist);
+        *nlist = NULL;
+        util_free_array(*plist);
+        *plist = NULL;
+    }
+    return ret;
+}
+
+static char *common_get_cgroup_path(const char *path, const char *subsystem)
+{
+    char **nlist = NULL, **plist = NULL;
+    size_t i = 0;
+    char *res = NULL;
+    if (path == NULL) {
+        ERROR("Invalid NULL param");
+        return NULL;
+    }
+
+    if (parse_cgroup_file(path, &nlist, &plist) < 0) {
+        return NULL;
+    }
+
+    for (i = 0; i < util_array_len((const char **)nlist); i++) {
+        const char *prefix = "name=";
+        bool find_sub = (strcmp(nlist[i], subsystem) == 0 || (strncmp(nlist[i], prefix, strlen(prefix)) == 0
+                        && strcmp(nlist[i]+strlen(prefix), subsystem) == 0));
+        if (find_sub) {
+            res = util_strdup_s(plist[i]);
+            break;
+        }
+    }
+
+    util_free_array(nlist);
+    util_free_array(plist);
+    return res;
+}
+
+char *common_get_init_cgroup(const char *subsystem)
+{
+    if (common_get_cgroup_version() != CGROUP_VERSION_1) {
+        ERROR("Not implemented for cgroup v2 hierarchy");
+        return NULL;
+    }
+
+    return common_get_cgroup_path("/proc/1/cgroup", subsystem);
+}
+
+char *common_get_own_cgroup(const char *subsystem)
+{
+    if (common_get_cgroup_version() != CGROUP_VERSION_1) {
+        ERROR("Not implemented for cgroup v2 hierarchy");
+        return NULL;
+    }
+
+    return common_get_cgroup_path("/proc/self/cgroup", subsystem);
+}
diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c
index 51758adb..8179558e 100644
--- a/src/daemon/config/isulad_config.c
+++ b/src/daemon/config/isulad_config.c
@@ -1762,6 +1762,8 @@ int merge_json_confs_into_global(struct service_arguments *args)
     args->json_confs->enable_cri_v1 = tmp_json_confs->enable_cri_v1;
 #endif
 
+    args->json_confs->systemd_cgroup = tmp_json_confs->systemd_cgroup;
+
     if (merge_cri_runtimes_into_global(args, tmp_json_confs)) {
         ret = -1;
         goto out;
@@ -1895,3 +1897,24 @@ isulad_daemon_constants *get_isulad_daemon_constants(void)
 {
     return g_isulad_daemon_constants;
 }
+
+bool conf_get_systemd_cgroup()
+{
+    bool systemd_cgroup = false;
+    struct service_arguments *conf = NULL;
+
+    if (isulad_server_conf_rdlock() != 0) {
+        return false;
+    }
+
+    conf = conf_get_server_conf();
+    if (conf == NULL || conf->json_confs == NULL) {
+        goto out;
+    }
+
+    systemd_cgroup = conf->json_confs->systemd_cgroup;
+
+out:
+    (void)isulad_server_conf_unlock();
+    return systemd_cgroup;
+}
diff --git a/src/daemon/config/isulad_config.h b/src/daemon/config/isulad_config.h
index 459ea331..f29cd564 100644
--- a/src/daemon/config/isulad_config.h
+++ b/src/daemon/config/isulad_config.h
@@ -114,6 +114,8 @@ char *conf_get_isulad_monitor_fifo_path(void);
 int init_isulad_daemon_constants(void);
 isulad_daemon_constants *get_isulad_daemon_constants(void);
 
+bool conf_get_systemd_cgroup(void);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc
index ba9459f6..76e393f3 100644
--- a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc
+++ b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc
@@ -632,3 +632,28 @@ grpc::Status RuntimeV1RuntimeServiceImpl::Status(grpc::ServerContext *context,
 
     return grpc::Status::OK;
 }
+
+grpc::Status
+RuntimeV1RuntimeServiceImpl::RuntimeConfig(grpc::ServerContext *context,
+                                           const runtime::v1::RuntimeConfigRequest *request,
+                                           runtime::v1::RuntimeConfigResponse *reply)
+{
+    Errors error;
+
+    if (request == nullptr) {
+        ERROR("Invalid input arguments");
+        return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, "Invalid input arguments");
+    }
+
+    EVENT("Event: {Object: CRI, Type: Runtime Config}");
+
+    m_rService->RuntimeConfig(reply, error);
+    if (!error.Empty()) {
+        ERROR("Object: CRI, Type: Failed to get runtime config:%s", error.GetMessage().c_str());
+        return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage());
+    }
+
+    EVENT("Event: {Object: CRI, Type: Runtime Config}");
+
+    return grpc::Status::OK;
+}
diff --git a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h
index 09ebe68b..52cc6b99 100644
--- a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h
+++ b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h
@@ -101,6 +101,10 @@ public:
     grpc::Status Status(grpc::ServerContext *context, const runtime::v1::StatusRequest *request,
                         runtime::v1::StatusResponse *reply) override;
 
+    grpc::Status RuntimeConfig(grpc::ServerContext *context,
+                               const runtime::v1::RuntimeConfigRequest *request,
+                               runtime::v1::RuntimeConfigResponse *reply) override;
+
 private:
     std::unique_ptr<CRIV1::CRIRuntimeService> m_rService;
 };
diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.cc
index 1afac3e2..de489aae 100644
--- a/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.cc
+++ b/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.cc
@@ -16,6 +16,7 @@
 #include "isula_libutils/log.h"
 #include "v1_cri_helpers.h"
 #include "cri_helpers.h"
+#include "isulad_config.h"
 
 namespace CRIV1 {
 void RuntimeManagerService::UpdateRuntimeConfig(const runtime::v1::RuntimeConfig &config, Errors & /*error*/)
@@ -69,4 +70,15 @@ auto RuntimeManagerService::Status(Errors &error) -> std::unique_ptr<runtime::v1
     return status;
 }
 
+void RuntimeManagerService::RuntimeConfig(runtime::v1::RuntimeConfigResponse *reply, Errors &error)
+{
+    if (reply == nullptr) {
+        ERROR("Invaliad params");
+        error.SetError("Invalid params");
+        return;
+    }
+
+    reply->mutable_linux()->set_cgroup_driver(conf_get_systemd_cgroup() ? runtime::v1::SYSTEMD : runtime::v1::CGROUPFS);
+}
+
 } // namespace CRI
diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.h b/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.h
index 3800ec30..2160064e 100644
--- a/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.h
+++ b/src/daemon/entry/cri/v1/v1_cri_runtime_manager_service.h
@@ -39,6 +39,8 @@ public:
 
     auto Status(Errors &error) -> std::unique_ptr<runtime::v1::RuntimeStatus>;
 
+    void RuntimeConfig(runtime::v1::RuntimeConfigResponse *reply, Errors &error);
+
 private:
     service_executor_t *m_cb;
     std::shared_ptr<Network::PluginManager> m_pluginManager;
diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service.h b/src/daemon/entry/cri/v1/v1_cri_runtime_service.h
index 87394173..839f6724 100644
--- a/src/daemon/entry/cri/v1/v1_cri_runtime_service.h
+++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service.h
@@ -89,6 +89,8 @@ public:
     virtual void UpdateRuntimeConfig(const runtime::v1::RuntimeConfig &config, Errors &error) = 0;
 
     virtual auto Status(Errors &error) -> std::unique_ptr<runtime::v1::RuntimeStatus> = 0;
+
+    virtual void RuntimeConfig(runtime::v1::RuntimeConfigResponse *reply, Errors &error) = 0;
 };
 } // namespace CRIV1
 #endif // DAEMON_ENTRY_CRI_V1_CRI_RUNTIME_SERVICE_INTERFACE_H
diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc
index 67cda5ed..aa5ae516 100644
--- a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc
+++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc
@@ -161,4 +161,9 @@ auto CRIRuntimeServiceImpl::Status(Errors &error) -> std::unique_ptr<runtime::v1
     return m_runtimeManager->Status(error);
 }
 
+void CRIRuntimeServiceImpl::RuntimeConfig(runtime::v1::RuntimeConfigResponse *reply, Errors &error)
+{
+    m_runtimeManager->RuntimeConfig(reply, error);
+}
+
 } // namespace CRIV1
diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h
index 23866648..0a25749f 100644
--- a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h
+++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h
@@ -92,6 +92,8 @@ public:
 
     auto Status(Errors &error) -> std::unique_ptr<runtime::v1::RuntimeStatus> override;
 
+    void RuntimeConfig(runtime::v1::RuntimeConfigResponse *reply, Errors &error) override;
+
 protected:
     std::unique_ptr<RuntimeVersionerService> m_runtimeVersioner;
     std::unique_ptr<ContainerManagerService> m_containerManager;
diff --git a/src/daemon/executor/container_cb/execution.c b/src/daemon/executor/container_cb/execution.c
index 63d8143c..7ed8e837 100644
--- a/src/daemon/executor/container_cb/execution.c
+++ b/src/daemon/executor/container_cb/execution.c
@@ -427,6 +427,32 @@ static int cpurt_controller_init(const char *id, const host_config *host_spec)
         return 0;
     }
 
+    if (conf_get_systemd_cgroup()) {
+        // currently it is the same as docker, yet it is unclear that
+        // if systemd cgroup is used and cgroup parent is set to a slice rather than system.slice
+        // should iSulad set cpu.rt_runtime_us and cpu.rt_period_us for the parent path?
+        // in fact, even if system.slice is used,
+        // cpu.rt_runtime_us and cpu.rt_period_us might still needed to be set manually
+        __isula_auto_free char *init_cgroup = common_get_init_cgroup("cpu");
+        if (init_cgroup == NULL) {
+            ERROR("Failed to get init cgroup");
+            return -1;
+        }
+        // make sure that the own cgroup path for cpu existed
+        __isula_auto_free char *own_cgroup = common_get_own_cgroup("cpu");
+        if (own_cgroup == NULL) {
+            ERROR("Failed to get own cgroup");
+            return -1;
+        }
+        char *new_cgroups_path = util_path_join(init_cgroup, cgroups_path);
+        if (new_cgroups_path == NULL) {
+            ERROR("Failed to join path");
+            return -1;
+        }
+        free(cgroups_path);
+        cgroups_path = new_cgroups_path;
+    }
+
     mnt_root = sysinfo_cgroup_controller_cpurt_mnt_path();
     if (mnt_root == NULL) {
         ERROR("Failed to get cpu rt controller mnt root path");
diff --git a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c
index 8c3c5f1a..6b862958 100644
--- a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c
+++ b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c
@@ -53,6 +53,13 @@ int rt_lcr_create(const char *name, const char *runtime, const rt_create_params_
     char *runtime_root = NULL;
     struct engine_operation *engine_ops = NULL;
 
+    if (conf_get_systemd_cgroup()) {
+        ERROR("Systemd cgroup not supported for lcr runtime");
+        isulad_set_error_message("Systemd cgroup not supported for lcr runtime");
+        ret = -1;
+        goto out;
+    }
+
     runtime_root = conf_get_routine_rootdir(runtime);
     if (runtime_root == NULL) {
         ERROR("Root path is NULL");
diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c
index 745154bb..b9aba3e3 100644
--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c
+++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c
@@ -1157,6 +1157,7 @@ int rt_isula_create(const char *id, const char *runtime, const rt_create_params_
     p.runtime_args = (char **)runtime_args;
     p.runtime_args_len = runtime_args_len;
     p.attach_socket = attach_socket;
+    p.systemd_cgroup = conf_get_systemd_cgroup();
     copy_process(&p, config->process);
     copy_annotations(&p, config->annotations);
 
diff --git a/src/daemon/modules/runtime/shim/shim_rt_ops.c b/src/daemon/modules/runtime/shim/shim_rt_ops.c
index 81daf224..fc4d8e3a 100644
--- a/src/daemon/modules/runtime/shim/shim_rt_ops.c
+++ b/src/daemon/modules/runtime/shim/shim_rt_ops.c
@@ -35,6 +35,7 @@
 #include "engine.h"
 #include "shim_rt_monitor.h"
 #include "supervisor.h"
+#include "isulad_config.h"
 
 #define EXIT_SIGNAL_OFFSET_X 128
 
@@ -370,6 +371,11 @@ int rt_shim_create(const char *id, const char *runtime, const rt_create_params_t
         return -1;
     }
 
+    if (conf_get_systemd_cgroup()) {
+        ERROR("ShimV2 does not support systemd cgroup yet");
+        return -1;
+    }
+
     exit_fifo_path = util_path_dir(params->exit_fifo);
     if (exit_fifo_path == NULL) {
         ERROR("%s: failed to get exit fifo dir from %s", id, params->exit_fifo);
diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c
index 464b4fb4..b4d2b0f6 100644
--- a/src/daemon/modules/spec/specs.c
+++ b/src/daemon/modules/spec/specs.c
@@ -188,7 +188,10 @@ static char *do_get_container_cgroup_path(const host_config *host_spec)
     }
 
     if (path == NULL) {
-        // third, all faild, just use default '/isulad'
+        // third, all faild, just use default '/isulad' for cgroupfs or "system.slice" for systemd
+        if (conf_get_systemd_cgroup()) {
+            return util_strdup_s("system.slice");
+        }
         path = util_strdup_s("/isulad");
     }
 
@@ -2288,6 +2291,23 @@ char *merge_container_cgroups_path(const char *id, const host_config *host_spec)
 
     path = do_get_container_cgroup_path(host_spec);
 
+    if (conf_get_systemd_cgroup()) {
+        // systemd cgroup path has the form of [slice]:[prefix]:[name]
+#define SYSTEMD_CGROUP_PATH_LEN 3
+        if (!util_has_suffix(path, ".slice")) {
+            ERROR("Invalid cgroup path %s for systemd", path);
+            isulad_set_error_message("Invalid cgroup path %s for systemd", path);
+            return NULL;
+        }
+
+        // slice must not contain slashes
+        // convert test.slice/test-a.slice/test-a-b.slice to become test-a-b.slice
+        __isula_auto_free char *base = util_path_base(path);
+        const char *isulad_prefix = "isulad";
+        const char *parts[SYSTEMD_CGROUP_PATH_LEN] = {base, isulad_prefix, id};
+        return util_string_join(":", parts, SYSTEMD_CGROUP_PATH_LEN);
+    }
+
     return util_path_join(path, id);
 }
 
-- 
2.34.1