From 835185f7c4739993c2ca26d737bb0a45277ad932 Mon Sep 17 00:00:00 2001 From: jikai Date: Wed, 20 Mar 2024 15:48:42 +0800 Subject: [PATCH 28/43] use supervisor to notify sandbox exit event Signed-off-by: jikai --- src/daemon/modules/api/container_api.h | 2 +- .../modules/container/restore/restore.c | 6 +++- .../modules/container/supervisor/supervisor.c | 15 +++++++- .../modules/service/service_container.c | 9 +++-- .../controller/shim/shim_controller.cc | 35 ++----------------- src/daemon/sandbox/sandbox_ops.cc | 23 ++++++++++++ src/daemon/sandbox/sandbox_ops.h | 2 ++ 7 files changed, 53 insertions(+), 39 deletions(-) diff --git a/src/daemon/modules/api/container_api.h b/src/daemon/modules/api/container_api.h index 4602d244..43d66d64 100644 --- a/src/daemon/modules/api/container_api.h +++ b/src/daemon/modules/api/container_api.h @@ -270,7 +270,7 @@ bool container_is_valid_state_string(const char *state); void container_update_health_monitor(const char *container_id); extern int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_info, const char *name, - const char *runtime); + const char *runtime, bool sandbox_container); extern char *container_exit_fifo_create(const char *cont_state_path); diff --git a/src/daemon/modules/container/restore/restore.c b/src/daemon/modules/container/restore/restore.c index 2669ea22..76868e28 100644 --- a/src/daemon/modules/container/restore/restore.c +++ b/src/daemon/modules/container/restore/restore.c @@ -57,6 +57,7 @@ static int restore_supervisor(const container_t *cont) char *statepath = cont->state_path; char *runtime = cont->runtime; pid_ppid_info_t pid_info = { 0 }; + bool sandbox_container = false; nret = snprintf(container_state, sizeof(container_state), "%s/%s", statepath, id); if (nret < 0 || (size_t)nret >= sizeof(container_state)) { @@ -90,8 +91,11 @@ static int restore_supervisor(const container_t *cont) pid_info.ppid = cont->state->state->p_pid; pid_info.start_time = cont->state->state->start_time; pid_info.pstart_time = cont->state->state->p_start_time; +#ifdef ENABLE_CRI_API_V1 + sandbox_container = is_sandbox_container(cont->common_config->sandbox_info); +#endif - if (container_supervisor_add_exit_monitor(exit_fifo_fd, &pid_info, id, runtime)) { + if (container_supervisor_add_exit_monitor(exit_fifo_fd, &pid_info, id, runtime, sandbox_container)) { ERROR("Failed to add exit monitor to supervisor"); ret = -1; goto out; diff --git a/src/daemon/modules/container/supervisor/supervisor.c b/src/daemon/modules/container/supervisor/supervisor.c index 1f9a043c..63289283 100644 --- a/src/daemon/modules/container/supervisor/supervisor.c +++ b/src/daemon/modules/container/supervisor/supervisor.c @@ -38,6 +38,9 @@ #include "container_api.h" #include "event_type.h" #include "utils_file.h" +#ifdef ENABLE_CRI_API_V1 +#include "sandbox_ops.h" +#endif pthread_mutex_t g_supervisor_lock = PTHREAD_MUTEX_INITIALIZER; struct epoll_descr g_supervisor_descr; @@ -47,6 +50,7 @@ struct supervisor_handler_data { int exit_code; char *name; char *runtime; + bool is_sandbox_container; pid_ppid_info_t pid_info; }; @@ -211,6 +215,14 @@ retry: (void)isulad_monitor_send_container_event(name, STOPPED, (int)pid, data->exit_code, NULL, NULL); +#ifdef ENABLE_CRI_API_V1 + if (data->is_sandbox_container) { + if (sandbox_on_sandbox_exit(name, data->exit_code) < 0) { + ERROR("Failed to handle sandbox %s exit", name); + } + } +#endif + supervisor_handler_data_free(data); DAEMON_CLEAR_ERRMSG(); @@ -259,7 +271,7 @@ static int supervisor_exit_cb(int fd, uint32_t events, void *cbdata, struct epol /* supervisor add exit monitor */ int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_info, const char *name, - const char *runtime) + const char *runtime, bool sandbox_container) { int ret = 0; struct supervisor_handler_data *data = NULL; @@ -285,6 +297,7 @@ int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_inf data->fd = fd; data->name = util_strdup_s(name); data->runtime = util_strdup_s(runtime); + data->is_sandbox_container = sandbox_container; data->pid_info.pid = pid_info->pid; data->pid_info.start_time = pid_info->start_time; data->pid_info.ppid = pid_info->ppid; diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c index a3606a82..7b34cc7f 100644 --- a/src/daemon/modules/service/service_container.c +++ b/src/daemon/modules/service/service_container.c @@ -275,13 +275,14 @@ static void clean_resources_on_failure(const container_t *cont, const char *engi return; } -static int do_post_start_on_success(const char *id, const char *runtime, const char *pidfile, int exit_fifo_fd, +static int do_post_start_on_success(const char *id, const char *runtime, bool sandbox_container, + const char *pidfile, int exit_fifo_fd, const pid_ppid_info_t *pid_info) { int ret = 0; // exit_fifo_fd was closed in container_supervisor_add_exit_monitor - if (container_supervisor_add_exit_monitor(exit_fifo_fd, pid_info, id, runtime)) { + if (container_supervisor_add_exit_monitor(exit_fifo_fd, pid_info, id, runtime, sandbox_container)) { ERROR("Failed to add exit monitor to supervisor"); ret = -1; } @@ -749,6 +750,7 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo oci_runtime_spec *oci_spec = NULL; rt_create_params_t create_params = { 0 }; rt_start_params_t start_params = { 0 }; + bool sandbox_container; nret = snprintf(bundle, sizeof(bundle), "%s/%s", cont->root_path, id); if (nret < 0 || (size_t)nret >= sizeof(bundle)) { @@ -897,6 +899,7 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo if (cont->common_config->sandbox_info != NULL) { create_params.task_addr = cont->common_config->sandbox_info->task_address; } + sandbox_container = is_sandbox_container(cont->common_config->sandbox_info); #endif if (runtime_create(id, runtime, &create_params) != 0) { @@ -921,7 +924,7 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo ret = runtime_start(id, runtime, &start_params, pid_info); if (ret == 0) { - if (do_post_start_on_success(id, runtime, pidfile, exit_fifo_fd, pid_info) != 0) { + if (do_post_start_on_success(id, runtime, sandbox_container, pidfile, exit_fifo_fd, pid_info) != 0) { ERROR("Failed to do post start on runtime start success"); ret = -1; goto clean_resources; diff --git a/src/daemon/sandbox/controller/shim/shim_controller.cc b/src/daemon/sandbox/controller/shim/shim_controller.cc index 39fcf8ea..593fade9 100644 --- a/src/daemon/sandbox/controller/shim/shim_controller.cc +++ b/src/daemon/sandbox/controller/shim/shim_controller.cc @@ -397,39 +397,8 @@ bool ShimController::Stop(const std::string &sandboxId, uint32_t timeoutSecs, Er bool ShimController::Wait(std::shared_ptr cb, const std::string &sandboxId, Errors &error) { - std::thread([this, cb, sandboxId]() { - if (m_cb == nullptr || m_cb->container.wait == nullptr) { - ERROR("Unimplemented callback"); - return; - } - - auto requestWrapper = makeUniquePtrCStructWrapper(free_container_wait_request); - if (requestWrapper == nullptr) { - ERROR("Out of memory"); - return; - } - auto request = requestWrapper->get(); - request->id = isula_strdup_s(sandboxId.c_str()); - request->condition = WAIT_CONDITION_STOPPED; - container_wait_response *response { nullptr }; - - int ret = m_cb->container.wait(request, &response); - auto responseWrapper = makeUniquePtrCStructWrapper(response, free_container_wait_response); - - if (ret != 0) { - std::string msg = (response != nullptr && response->errmsg != nullptr) ? response->errmsg : "internal"; - ERROR("Failed to wait sandbox %s: %s", sandboxId.c_str(), msg.c_str()); - return; - } - - ControllerExitInfo info; - auto currentTime = std::chrono::high_resolution_clock::now(); - auto duration = currentTime.time_since_epoch(); - info.exitedAt = std::chrono::duration_cast(duration).count(); - info.exitStatus = response->exit_code; - cb->OnSandboxExit(info); - }).detach(); - + // ShimController will use sandbox_on_exit callback of supervisor in lower container level + // to notify the sandbox exit event return true; } diff --git a/src/daemon/sandbox/sandbox_ops.cc b/src/daemon/sandbox/sandbox_ops.cc index 005063c0..b7fb40bf 100644 --- a/src/daemon/sandbox/sandbox_ops.cc +++ b/src/daemon/sandbox/sandbox_ops.cc @@ -18,6 +18,7 @@ #include #include "controller_manager.h" +#include "sandbox_manager.h" #include "namespace.h" #include "utils.h" @@ -175,3 +176,25 @@ int sandbox_purge_exec(const container_config_v2_common_config *config, const ch { return do_sandbox_purge(config, exec_id); } + +int sandbox_on_sandbox_exit(const char *sandbox_id, int exit_code) +{ + if (nullptr == sandbox_id) { + ERROR("Invalid parameter: sandbox_id"); + return -1; + } + + auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(sandbox_id); + if (nullptr == sandbox) { + ERROR("Sandbox %s not found", sandbox_id); + return -1; + } + + sandbox::ControllerExitInfo info; + auto currentTime = std::chrono::high_resolution_clock::now(); + auto duration = currentTime.time_since_epoch(); + info.exitedAt = std::chrono::duration_cast(duration).count(); + info.exitStatus = exit_code; + sandbox->OnSandboxExit(info); + return 0; +} diff --git a/src/daemon/sandbox/sandbox_ops.h b/src/daemon/sandbox/sandbox_ops.h index bef884fb..8189efd6 100644 --- a/src/daemon/sandbox/sandbox_ops.h +++ b/src/daemon/sandbox/sandbox_ops.h @@ -36,6 +36,8 @@ int sandbox_purge_container(const container_config_v2_common_config *config); int sandbox_purge_exec(const container_config_v2_common_config *config, const char *exec_id); +int sandbox_on_sandbox_exit(const char *sandbox_id, int exit_code); + #ifdef __cplusplus } #endif -- 2.34.1