From 32dbf764fd5b7f6941c49750b49dbba253bd3234 Mon Sep 17 00:00:00 2001 From: zhongtao Date: Mon, 13 Feb 2023 15:36:58 +0800 Subject: [PATCH 16/53] isulad-shim support execSync with timeout Signed-off-by: zhongtao --- .../executor/container_cb/execution_stream.c | 2 +- .../modules/runtime/isula/isula_rt_ops.c | 37 +++++++++++++++--- src/utils/cutils/utils.c | 39 +++++++++++++++++++ src/utils/cutils/utils.h | 5 +++ 4 files changed, 77 insertions(+), 6 deletions(-) diff --git a/src/daemon/executor/container_cb/execution_stream.c b/src/daemon/executor/container_cb/execution_stream.c index fde0335e..1a7353b5 100644 --- a/src/daemon/executor/container_cb/execution_stream.c +++ b/src/daemon/executor/container_cb/execution_stream.c @@ -161,7 +161,7 @@ static int container_exec_cb(const container_exec_request *request, container_ex if (exec_container(cont, request, *response, stdinfd, stdout_handler, stderr_handler) != 0) { ret = -1; - goto out; + goto pack_err_response; } goto out; diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c index 5463bb1b..6f2b4f7d 100644 --- a/src/daemon/modules/runtime/isula/isula_rt_ops.c +++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c @@ -205,6 +205,10 @@ static void show_shim_runtime_errlog(const char *workdir) char buf1[SHIM_LOG_SIZE] = { 0 }; char buf2[SHIM_LOG_SIZE] = { 0 }; + if (g_isulad_errmsg != NULL) { + return; + } + get_err_message(buf1, sizeof(buf1), workdir, "shim-log.json"); get_err_message(buf2, sizeof(buf2), workdir, "log.json"); ERROR("shim-log: %s", buf1); @@ -686,8 +690,29 @@ static int status_to_exit_code(int status) return exit_code; } +static int try_wait_pid(pid_t pid) +{ + if (waitpid(pid, NULL, WNOHANG) == pid) { + return 0; + } + + return 1; +} + +static void kill_and_show_err(pid_t pid) +{ + int nret = 0; + kill(pid, SIGKILL); + // wait atmost 0.5 seconds + DO_RETRY_CALL(5, 100000, nret, try_wait_pid, pid); + if (nret != 0) { + WARN("Fail to wait isulad-shim"); + } + isulad_set_error_message("Exec container error;exec timeout"); +} + static int shim_create(bool fg, const char *id, const char *workdir, const char *bundle, const char *runtime_cmd, - int *exit_code) + int *exit_code, const int64_t timeout) { pid_t pid = 0; int exec_fd[2] = { -1, -1 }; @@ -778,7 +803,7 @@ realexec: goto out; } - status = util_wait_for_pid_status(pid); + status = util_waitpid_with_timeout(pid, timeout, kill_and_show_err); if (status < 0) { ERROR("failed wait shim-parent %d exit %s", pid, strerror(errno)); ret = -1; @@ -792,7 +817,9 @@ realexec: out: if (ret != 0) { show_shim_runtime_errlog(workdir); - kill(pid, SIGKILL); /* can kill other process? */ + if (timeout <= 0) { + kill(pid, SIGKILL); /* can kill other process? */ + } } return ret; @@ -901,7 +928,7 @@ int rt_isula_create(const char *id, const char *runtime, const rt_create_params_ } get_runtime_cmd(runtime, &cmd); - ret = shim_create(false, id, workdir, params->bundle, cmd, NULL); + ret = shim_create(false, id, workdir, params->bundle, cmd, NULL, -1); if (ret != 0) { runtime_call_delete_force(workdir, runtime, id); ERROR("%s: failed create shim process", id); @@ -1173,7 +1200,7 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p } get_runtime_cmd(runtime, &cmd); - ret = shim_create(fg_exec(params), id, workdir, bundle, cmd, exit_code); + ret = shim_create(fg_exec(params), id, workdir, bundle, cmd, exit_code, params->timeout); if (ret != 0) { ERROR("%s: failed create shim process for exec %s", id, exec_id); goto errlog_out; diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c index f99b28e4..2c4c01e4 100644 --- a/src/utils/cutils/utils.c +++ b/src/utils/cutils/utils.c @@ -311,6 +311,45 @@ rep: return 0; } +/* + * If timeout <= 0, blocking wait pid. + * If timeout > 0, non-blocking wait pid with timeout. + * When waitpid timeout, calling handle_timeout_callback_t. + */ +int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_callback_t cb) +{ + int nret = 0; + time_t start_time = time(NULL); + time_t end_time; + double interval; + + if (timeout <= 0) { + return util_wait_for_pid_status(pid); + } + + for (;;) { + nret = waitpid(pid, NULL, WNOHANG); + if (nret == pid) { + break; + } + if (nret == -1 && errno != EINTR) { + return -1; + } + end_time = time(NULL); + interval = difftime(end_time, start_time); + if (nret == 0 && interval >= timeout) { + INFO("Wait %d timeout", pid); + if (cb != NULL) { + cb(pid); + } + return -1; + } + // sleep some time instead to avoid cpu full running and then retry. + sleep(0.1); + } + return 0; +} + int util_wait_for_pid_status(pid_t pid) { int st; diff --git a/src/utils/cutils/utils.h b/src/utils/cutils/utils.h index 6261dc05..01107605 100644 --- a/src/utils/cutils/utils.h +++ b/src/utils/cutils/utils.h @@ -302,6 +302,9 @@ typedef struct _proc_t { processor; /* current (or most recent?) CPU */ } proc_t; +// handle waitpid timeout. +typedef void(*handle_timeout_callback_t)(pid_t pid); + struct signame { int num; const char *name; @@ -329,6 +332,8 @@ char *util_strdup_s(const char *src); int util_wait_for_pid(pid_t pid); +int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_callback_t cb); + void util_contain_errmsg(const char *errmsg, int *exit_code); char *util_short_digest(const char *digest); -- 2.25.1