From aaf1b46c66aa596ec718c11c4f4270e41e7b570e Mon Sep 17 00:00:00 2001 From: zhongtao Date: Tue, 7 Nov 2023 16:39:35 +0800 Subject: [PATCH 11/14] add runc attach implement Signed-off-by: zhongtao --- .../connect/grpc/grpc_containers_client.cc | 9 + src/cmd/isula/stream/attach.c | 12 +- src/cmd/isulad-shim/common.c | 116 ++++- src/cmd/isulad-shim/common.h | 30 ++ src/cmd/isulad-shim/main.c | 14 + src/cmd/isulad-shim/process.c | 453 ++++++++++++++++-- src/cmd/isulad-shim/process.h | 7 +- src/cmd/isulad-shim/terminal.c | 2 +- .../executor/container_cb/execution_stream.c | 1 + src/daemon/modules/api/runtime_api.h | 1 + .../modules/runtime/isula/isula_rt_ops.c | 168 ++++++- src/utils/cutils/error.h | 4 +- 12 files changed, 755 insertions(+), 62 deletions(-) diff --git a/src/client/connect/grpc/grpc_containers_client.cc b/src/client/connect/grpc/grpc_containers_client.cc index 2dd73100..bcb1e8da 100644 --- a/src/client/connect/grpc/grpc_containers_client.cc +++ b/src/client/connect/grpc/grpc_containers_client.cc @@ -1394,6 +1394,8 @@ public: auto run(const struct isula_attach_request *request, struct isula_attach_response *response) -> int override { ClientContext context; + bool detach = false; + std::string attach_detach_msg = "read escape sequence"; if (set_custom_header_metadata(context, request) != 0) { ERROR("Failed to translate request to grpc"); @@ -1415,6 +1417,9 @@ public: break; } if (!stream_response.stdout().empty()) { + if (strcmp(stream_response.stdout().c_str(), attach_detach_msg.c_str()) == 0) { + detach = true; + } std::cout << stream_response.stdout() << std::flush; } if (!stream_response.stderr().empty()) { @@ -1437,6 +1442,10 @@ public: response->cc = ISULAD_ERR_EXEC; } + if (detach) { + response->server_errono = ISULAD_INFO_DETACH; + } + out: if (request->attach_stdin) { pthread_cancel(writer.native_handle()); diff --git a/src/cmd/isula/stream/attach.c b/src/cmd/isula/stream/attach.c index ff49af92..b61c9350 100644 --- a/src/cmd/isula/stream/attach.c +++ b/src/cmd/isula/stream/attach.c @@ -37,6 +37,7 @@ #include "connect.h" #include "constants.h" #include "client_helpers.h" +#include "error.h" #ifndef GRPC_CONNECTOR #include "client_console.h" #endif @@ -70,8 +71,10 @@ static int check_tty(bool tty, struct termios *oldtios, bool *reset_tty) } *reset_tty = true; } else { - INFO("the input device is not a TTY"); - return 0; + // if it is trying to attach to a container TTY + // from a non-TTY client input stream, returns -1. + COMMAND_ERROR("the input device is not a TTY"); + return -1; } return 0; @@ -353,6 +356,7 @@ static int client_attach(struct client_arguments *args, uint32_t *exit_code) #endif config = get_connect_config(args); + // Obtain the container's real exit code by waiting for the container to stop. container_wait_thread(args, exit_code, &sem_exited); ret = ops->container.attach(&request, response, &config); if (ret != 0) { @@ -374,7 +378,9 @@ static int client_attach(struct client_arguments *args, uint32_t *exit_code) if (sem_timedwait(&sem_exited, &ts) != 0) { if (errno == ETIMEDOUT) { - COMMAND_ERROR("Wait container status timeout."); + if (response->server_errono != ISULAD_INFO_DETACH) { + INFO("Wait container stopped status timeout."); + } } else { CMD_SYSERROR("Failed to wait sem"); } diff --git a/src/cmd/isulad-shim/common.c b/src/cmd/isulad-shim/common.c index 781dc004..48d266dc 100644 --- a/src/cmd/isulad-shim/common.c +++ b/src/cmd/isulad-shim/common.c @@ -33,16 +33,26 @@ #include int g_log_fd = -1; +int g_attach_log_fd = -1; int init_shim_log(void) { - g_log_fd = open_no_inherit(SHIM_LOG_NAME, O_CREAT | O_WRONLY | O_APPEND | O_SYNC, 0640); + g_log_fd = open_no_inherit(SHIM_LOG_NAME, O_CREAT | O_WRONLY | O_APPEND | O_SYNC, LOG_FILE_MODE); if (g_log_fd < 0) { return SHIM_ERR; } return SHIM_OK; } +int init_attach_log(void) +{ + g_attach_log_fd = open_no_inherit(ATTACH_LOG_NAME, O_CREAT | O_WRONLY | O_APPEND | O_SYNC, LOG_FILE_MODE); + if (g_attach_log_fd < 0) { + return SHIM_ERR; + } + return SHIM_OK; +} + void signal_routine(int sig) { switch (sig) { @@ -162,11 +172,24 @@ int generate_random_str(char *id, size_t len) return SHIM_OK; } -void write_message(const char *level, const char *fmt, ...) -{ #define MAX_MSG_JSON_TEMPLATE 32 #define MAX_MESSAGE_CONTENT_LEN 128 #define MAX_MESSAGE_LEN (MAX_MSG_JSON_TEMPLATE + MAX_MESSAGE_CONTENT_LEN) + +static void format_log_msg(const char *level, const char *buf, char *msg, int max_message_len) +{ + time_t current_time = time(NULL); + struct tm *local_time = localtime(¤t_time); + char time_str[20]; + + strftime(time_str, sizeof(time_str), "%Y-%m-%d %H:%M:%S", local_time); + + (void)snprintf(msg, max_message_len - 1, "{\"time\": \"%s\", \"level\": \"%s\", \"msg\": \"%s\"}\n", time_str, level, + buf); +} + +void write_message(const char *level, const char *fmt, ...) +{ if (g_log_fd < 0) { return; } @@ -183,15 +206,31 @@ void write_message(const char *level, const char *fmt, ...) return; } - nwrite = snprintf(msg, MAX_MESSAGE_LEN - 1, "{\"level\": \"%s\", \"msg\": \"%s\"}\n", level, buf); - if (nwrite < 0 || (size_t)nwrite >= (MAX_MESSAGE_LEN - 1)) { + format_log_msg(level, buf, msg, MAX_MESSAGE_CONTENT_LEN); + + (void)isula_file_total_write_nointr(g_log_fd, msg, strlen(msg)); +} + +void write_attach_message(const char *level, const char *fmt, ...) +{ + char buf[MAX_MESSAGE_CONTENT_LEN] = { 0 }; + char msg[MAX_MESSAGE_LEN] = { 0 }; + int nwrite = -1; + + if (g_attach_log_fd < 0) { return; } - - nwrite = isula_file_total_write_nointr(g_log_fd, msg, strlen(msg)); - if (nwrite < 0 || (size_t)nwrite != strlen(msg)) { + va_list arg_list; + va_start(arg_list, fmt); + nwrite = vsnprintf(buf, MAX_MESSAGE_CONTENT_LEN, fmt, arg_list); + va_end(arg_list); + if (nwrite < 0) { return; } + + format_log_msg(level, buf, msg, MAX_MESSAGE_CONTENT_LEN); + + (void)isula_file_total_write_nointr(g_attach_log_fd, msg, strlen(msg)); } /* note: This function can only read small text file. */ @@ -272,3 +311,64 @@ int open_no_inherit(const char *path, int flag, mode_t mode) return fd; } + +/* judge the fd whether is attach fifo */ +struct isula_linked_list *get_attach_fifo_item(int fd, struct isula_linked_list *list) +{ + struct isula_linked_list *it = NULL; + struct isula_linked_list *next = NULL; + + if (fd <= 0 || list == NULL || isula_linked_list_empty(list)) { + return it; + } + + isula_linked_list_for_each_safe(it, list, next) { + struct shim_fifos_fd *elem = (struct shim_fifos_fd *)it->elem; + if (elem == NULL) { + continue; + } + if (elem->in_fd == fd) { + return it; + } + if (elem->out_fd == fd) { + return it; + } + if (elem->err_fd == fd) { + return it; + } + } + + return it; +} + +void free_shim_fifos_fd(struct shim_fifos_fd *item) +{ + if (item == NULL) { + return; + } + if (item->in_fifo != NULL) { + free(item->in_fifo); + item->in_fifo = NULL; + } + if (item->out_fifo != NULL) { + free(item->out_fifo); + item->out_fifo = NULL; + } + if (item->err_fifo != NULL) { + free(item->err_fifo); + item->err_fifo = NULL; + } + if (item->in_fd >= 0) { + close(item->in_fd); + item->in_fd = -1; + } + if (item->out_fd >= 0) { + close(item->out_fd); + item->out_fd = -1; + } + if (item->err_fd >= 0) { + close(item->err_fd); + item->err_fd = -1; + } + free(item); +} \ No newline at end of file diff --git a/src/cmd/isulad-shim/common.h b/src/cmd/isulad-shim/common.h index 55efdc28..2020a799 100644 --- a/src/cmd/isulad-shim/common.h +++ b/src/cmd/isulad-shim/common.h @@ -21,6 +21,7 @@ #include #include #include +#include #ifdef __cplusplus extern "C" { @@ -59,8 +60,22 @@ extern "C" { #define CONTAINER_ACTION_REBOOT 129 #define CONTAINER_ACTION_SHUTDOWN 130 +#define ATTACH_SOCKET "attach_socket.sock" +#define ATTACH_LOG_NAME "attach-log.json" +#define ATTACH_DETACH_MSG "read escape sequence" +#define MAX_ATTACH_NUM 16 + +#define CTRL_Q 0x11 // ASCII code control character ctrl + Q + +#define LOG_FILE_MODE 0600 + +#define SOCKET_DIRECTORY_MODE 0600 +#define ATTACH_FIFOPATH_MODE 0600 + int init_shim_log(void); +int init_attach_log(void); + void signal_routine(int sig); /** @@ -90,18 +105,33 @@ void signal_routine(int sig); } \ } while (0) +struct shim_fifos_fd { + char *in_fifo; + char *out_fifo; + char *err_fifo; + int in_fd; + int out_fd; + int err_fd; +}; + char *read_text_file(const char *path); int cmd_combined_output(const char *binary, const char *params[], void *output, int *output_len); void write_message(const char *level, const char *fmt, ...); +void write_attach_message(const char *level, const char *fmt, ...); + int generate_random_str(char *id, size_t len); void close_fd(int *pfd); int open_no_inherit(const char *path, int flag, mode_t mode); +struct isula_linked_list *get_attach_fifo_item(int fd, struct isula_linked_list *list); + +void free_shim_fifos_fd(struct shim_fifos_fd *item); + #ifdef __cplusplus } #endif diff --git a/src/cmd/isulad-shim/main.c b/src/cmd/isulad-shim/main.c index 454011d0..deb07271 100644 --- a/src/cmd/isulad-shim/main.c +++ b/src/cmd/isulad-shim/main.c @@ -145,6 +145,20 @@ int main(int argc, char **argv) } } + if (p->state->attach_socket != NULL) { + ret = prepare_attach_socket(p); + if (ret != SHIM_OK) { + write_message(ERR_MSG, "failed to prepare attach socket:%d", ret); + exit(EXIT_FAILURE); + } + + ret = init_attach_log(); + if (ret != SHIM_OK) { + write_message(ERR_MSG, "failed to init shim attach log"); + exit(EXIT_FAILURE); + } + } + /* start epoll for io copy */ ret = process_io_start(p, &tid_epoll); if (ret != SHIM_OK) { diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c index 40908102..187067d2 100644 --- a/src/cmd/isulad-shim/process.c +++ b/src/cmd/isulad-shim/process.c @@ -19,20 +19,22 @@ #include #include #include +#include +#include +#include // IWYU pragma: keep +#include +#include #include #include #include -#include #include #include #include #include -#include -#include #include // IWYU pragma: keep -#include // IWYU pragma: keep #include #include +#include #include #include @@ -42,6 +44,8 @@ #include #include #include +#include +#include #include "common.h" #include "terminal.h" @@ -57,7 +61,7 @@ static shim_client_process_state *load_process() p_state = shim_client_process_state_parse_file("process.json", NULL, &err); if (p_state == NULL) { - write_message(ERR_MSG, "parse process state failed"); + write_message(ERR_MSG, "parse process state failed: %s", err); } /* "err" will definitely be allocated memory in the function above */ free(err); @@ -168,6 +172,99 @@ static int sync_exit_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr return EPOLL_LOOP_HANDLE_CLOSE; } +static bool fifo_exists(const char *path) +{ + struct stat sb; + int ret; + + ret = stat(path, &sb); + if (ret < 0) { + // could be something other than exist, just return false + return false; + } + + return S_ISFIFO(sb.st_mode); +} + +static int add_attach_terminal_fifos(const char *in, const char *out, const char *err, int *input_fd, process_t *p) +{ + __isula_auto_close int fifofd_in = -1; + __isula_auto_close int fifofd_out = -1; + __isula_auto_close int fifofd_err = -1; + struct shim_fifos_fd *fifos = NULL; + struct isula_linked_list *node = NULL; + + bool invalid = (in != NULL && !fifo_exists(in)) || (out != NULL && !fifo_exists(out)) || (err != NULL && + !fifo_exists(err)); + if (invalid) { + write_attach_message(ERR_MSG, "File %s or %s or %s does not refer to a FIFO", in, out, err); + return -1; + } + + if (in != NULL) { + fifofd_in = isula_file_open(in, O_RDONLY | O_NONBLOCK | O_CLOEXEC, 0); + if (fifofd_in < 0) { + write_attach_message(ERR_MSG, "Failed to open FIFO: %s", in); + return -1; + } + } + + if (out != NULL) { + fifofd_out = isula_file_open(out, O_WRONLY | O_NONBLOCK | O_CLOEXEC, 0); + if (fifofd_out < 0) { + write_attach_message(ERR_MSG, "Failed to open FIFO: %s", out); + return -1; + } + } + + if (err != NULL) { + fifofd_err = isula_file_open(err, O_WRONLY | O_NONBLOCK | O_CLOEXEC, 0); + if (fifofd_err < 0) { + write_attach_message(ERR_MSG, "Failed to open FIFO: %s", err); + return -1; + } + } + + fifos = isula_common_calloc_s(sizeof(*fifos)); + if (fifos == NULL) { + write_attach_message(ERR_MSG, "Out of memory"); + goto err_out; + } + + fifos->in_fifo = isula_strdup_s(in); + fifos->out_fifo = isula_strdup_s(out); + fifos->err_fifo = isula_strdup_s(err); + + fifos->in_fd = isula_transfer_fd(fifofd_in); + fifos->out_fd = isula_transfer_fd(fifofd_out); + fifos->err_fd = isula_transfer_fd(fifofd_err); + node = isula_common_calloc_s(sizeof(struct isula_linked_list)); + if (node == NULL) { + write_attach_message(ERR_MSG, "Out of memory"); + goto err_out; + } + + node->elem = fifos; + isula_linked_list_add(p->attach_fifos, node); + + if (input_fd != NULL) { + *input_fd = fifos->in_fd; + } + + return 0; +err_out: + free_shim_fifos_fd(fifos); + return -1; +} + +static void remove_attach_terminal_fifos(isula_epoll_descr_t *descr, struct isula_linked_list *item) +{ + struct shim_fifos_fd *elem = (struct shim_fifos_fd *)item->elem; + isula_epoll_remove_handler(descr, elem->in_fd); + isula_linked_list_del(item); + free_shim_fifos_fd(elem); +} + static int stdin_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t *descr) { process_t *p = (process_t *)cbdata; @@ -210,6 +307,57 @@ static int stdin_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t * return EPOLL_LOOP_HANDLE_CONTINUE; } +static int attach_stdin_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t *descr) +{ + process_t *p = (process_t *)cbdata; + int r_count = 0; + int w_count = 0; + int *fd_to = NULL; + struct isula_linked_list *item; + + if (events & EPOLLHUP) { + write_message(ERR_MSG, "attach stdin %d received the EPOLLHUP event", fd); + goto err_out; + } + + if (!(events & EPOLLIN)) { + return EPOLL_LOOP_HANDLE_CONTINUE; + } + + (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF); + + r_count = isula_file_read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF); + if (r_count <= 0) { + write_message(ERR_MSG, "failed to read from attach stdin %d, error:%d", fd, SHIM_SYS_ERR(errno)); + goto err_out; + } + + if (p->state->terminal) { + fd_to = &(p->recv_fd); + } else { + fd_to = &(p->shim_io->in); + } + + if (fd_to == NULL || *fd_to == -1) { + return EPOLL_LOOP_HANDLE_CONTINUE; + } + w_count = isula_file_total_write_nointr(*fd_to, p->buf, r_count); + if (w_count < 0) { + /* When any error occurs, set the write fd -1 */ + write_message(WARN_MSG, "write in_fd %d error:%d", *fd_to, SHIM_SYS_ERR(errno)); + close(*fd_to); + *fd_to = -1; + } + + return EPOLL_LOOP_HANDLE_CONTINUE; +err_out: + item = get_attach_fifo_item(fd, p->attach_fifos); + if (item != NULL && item->elem != NULL) { + remove_attach_terminal_fifos(descr, item); + } + return EPOLL_LOOP_HANDLE_CONTINUE; +} + static int stdout_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t *descr) { process_t *p = (process_t *)cbdata; @@ -237,16 +385,29 @@ static int stdout_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t shim_write_container_log_file(p->terminal, STDID_OUT, p->buf, r_count); - if (p->isulad_io->out == -1) { + if (p->isulad_io->out != -1) { + w_count = isula_file_total_write_nointr(p->isulad_io->out, p->buf, r_count); + if (w_count < 0) { + /* When any error occurs, set the write fd -1 */ + write_message(WARN_MSG, "write out_fd %d error:%d", p->isulad_io->out, SHIM_SYS_ERR(errno)); + close(p->isulad_io->out); + p->isulad_io->out = -1; + } + } + + if (isula_linked_list_empty(p->attach_fifos)) { return EPOLL_LOOP_HANDLE_CONTINUE; } - w_count = isula_file_total_write_nointr(p->isulad_io->out, p->buf, r_count); - if (w_count < 0) { - /* When any error occurs, set the write fd -1 */ - write_message(WARN_MSG, "write out_fd %d error:%d", p->isulad_io->out, SHIM_SYS_ERR(errno)); - close(p->isulad_io->out); - p->isulad_io->out = -1; + struct isula_linked_list *it = NULL; + struct isula_linked_list *next = NULL; + + isula_linked_list_for_each_safe(it, p->attach_fifos, next) { + struct shim_fifos_fd *elem = (struct shim_fifos_fd *)it->elem; + w_count = isula_file_total_write_nointr(elem->out_fd, p->buf, r_count); + if (w_count < 0) { + remove_attach_terminal_fifos(descr, it); + } } return EPOLL_LOOP_HANDLE_CONTINUE; @@ -279,16 +440,29 @@ static int stderr_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t shim_write_container_log_file(p->terminal, STDID_ERR, p->buf, r_count); - if (p->isulad_io->err == -1) { + if (p->isulad_io->err != -1) { + w_count = isula_file_total_write_nointr(p->isulad_io->err, p->buf, r_count); + if (w_count < 0) { + /* When any error occurs, set the write fd -1 */ + write_message(WARN_MSG, "write err_fd %d error:%d", p->isulad_io->err, SHIM_SYS_ERR(errno)); + close(p->isulad_io->err); + p->isulad_io->err = -1; + } + } + + if (isula_linked_list_empty(p->attach_fifos)) { return EPOLL_LOOP_HANDLE_CONTINUE; } - w_count = isula_file_total_write_nointr(p->isulad_io->err, p->buf, r_count); - if (w_count < 0) { - /* When any error occurs, set the write fd -1 */ - write_message(WARN_MSG, "write err_fd %d error:%d", p->isulad_io->err, SHIM_SYS_ERR(errno)); - close(p->isulad_io->err); - p->isulad_io->err = -1; + struct isula_linked_list *it = NULL; + struct isula_linked_list *next = NULL; + + isula_linked_list_for_each_safe(it, p->attach_fifos, next) { + struct shim_fifos_fd *elem = (struct shim_fifos_fd *)it->elem; + w_count = isula_file_total_write_nointr(elem->out_fd, p->buf, r_count); + if (w_count < 0) { + remove_attach_terminal_fifos(descr, it); + } } return EPOLL_LOOP_HANDLE_CONTINUE; @@ -326,6 +500,159 @@ static int resize_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t return EPOLL_LOOP_HANDLE_CONTINUE; } +static bool attach_fifopath_security_check(process_t *p, const char *fifopath) +{ + struct stat st = { 0 }; + char real_path[PATH_MAX] = { 0 }; + + if (isula_validate_absolute_path(fifopath) != 0) { + write_attach_message(ERR_MSG, "attach fifo path \"%s\" must be an valid absolute path", fifopath); + return false; + } + + if (realpath(fifopath, real_path) == NULL) { + write_attach_message(ERR_MSG, "Failed to get realpath for '%s': %s.", real_path, SHIM_SYS_ERR(errno)); + return false; + } + + if (!isula_has_prefix(real_path, p->workdir)) { + write_attach_message(ERR_MSG, "attach fifo path \"%s\" must be under the state path", real_path, p->workdir); + return false; + } + + if (lstat(real_path, &st) != 0) { + write_attach_message(ERR_MSG, "Failed to lstat %s : %s", real_path, SHIM_SYS_ERR(errno)); + return false; + } + + if (!S_ISFIFO(st.st_mode)) { + write_attach_message(ERR_MSG, "attach fifo path \"%s\" must be an FIFO", real_path); + return false; + } + + if ((st.st_mode & 0777) != ATTACH_FIFOPATH_MODE) { + write_attach_message(ERR_MSG, "attach fifo path \"%s\" permission invalid", real_path); + return false; + } + + if (st.st_uid != 0) { + write_attach_message(ERR_MSG, "attach fifo path \"%s\" uid invalid", real_path); + return false; + } + + return true; +} + +// attach_cb needs to read the content from communication fd and parse it. +// at the same time, it also needs to establish a connection between the attach fd and the container fd. +// 1. if it fails, it needs to write an error message to attach log file, +// and write -1 to connection fd to let isulad know that it has failed. +// 2. if it succeeds, write 0 to let isulad know that it is ready. +// attach_cb returns EPOLL_LOOP_HANDLE_CONTINUE regardless of success or failure, +// because whether the attach operation is successful or not does not affect the first process of the container. +static int attach_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t *descr) +{ + process_t *p = (process_t *)cbdata; + int r_count = 0; + char tmp_buf[BUFSIZ + 1] = { 0 }; + char *in = NULL, *out = NULL, *err = NULL; + int fifofd_in = -1; + isula_string_array *tmp_str_array = NULL; + int ret = 0; + // attach execution return value + int status = -1; + bool valid = true; + + // after receiving the event that isulad closes the connection, + // close the communication fd and remove it from epoll. + if (events & EPOLLHUP) { + close(fd); + isula_epoll_remove_handler(descr, fd); + return EPOLL_LOOP_HANDLE_CONTINUE; + } + + if (!(events & EPOLLIN)) { + return EPOLL_LOOP_HANDLE_CONTINUE; + } + + r_count = isula_file_read_nointr(fd, tmp_buf, sizeof(tmp_buf) - 1); + if (r_count <= 0) { + write_attach_message(ERR_MSG, "Failed to read msg from attach conn fd"); + goto out; + } + + // limit the number of attach connections to MAX_ATTACH_NUM + if (isula_linked_list_len(p->attach_fifos) >= MAX_ATTACH_NUM) { + write_attach_message(ERR_MSG, "The number of attach connections exceeds the limit:%d, and this connection is rejected.", + MAX_ATTACH_NUM); + goto out; + } + + tmp_str_array = isula_string_split_to_multi(tmp_buf, ' '); + if (tmp_str_array->len != 3) { + write_attach_message(ERR_MSG, "Invalid attach msg from isulad"); + goto out; + } + + for (int i = 0; i < tmp_str_array->len; i++) { + valid = valid && attach_fifopath_security_check(p, tmp_str_array->items[i]); + } + + if (!valid) { + write_attach_message(ERR_MSG, "Invalid attach fifo path from isulad"); + goto out; + } + + in = tmp_str_array->items[0]; + out = tmp_str_array->items[1]; + err = tmp_str_array->items[2]; + + if (add_attach_terminal_fifos(in, out, err, &fifofd_in, p) < 0) { + write_attach_message(ERR_MSG, "Failed to add attach terminal fifos"); + goto out; + } + + // attach stdin --> container stdin + ret = isula_epoll_add_handler(descr, fifofd_in, attach_stdin_cb, p); + if (ret != SHIM_OK) { + write_attach_message(ERR_MSG, "add fifofd_in fd %d to epoll loop failed:%d", fifofd_in, SHIM_SYS_ERR(errno)); + struct isula_linked_list *item = get_attach_fifo_item(fd, p->attach_fifos); + if (item != NULL && item->elem != NULL) { + remove_attach_terminal_fifos(descr, item); + } + goto out; + } + + status = 0; +out: + isula_string_array_free(tmp_str_array); + (void)isula_file_write_nointr(fd, &status, sizeof(int)); + return EPOLL_LOOP_HANDLE_CONTINUE; +} + +// do_attach_socket_accept returns EPOLL_LOOP_HANDLE_CONTINUE regardless of success or failure, +// because whether the attach operation is successful or not does not affect the first process of the container. +static int do_attach_socket_accept(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t *descr) +{ + process_t *p = (process_t *)cbdata; + int conn_fd = -1; + int ret = SHIM_ERR; + + conn_fd = accept(p->attach_socket_fd, NULL, NULL); + if (conn_fd < 0) { + write_attach_message(ERR_MSG, "accept from fd %d failed:%d", p->attach_socket_fd, SHIM_SYS_ERR(errno)); + return EPOLL_LOOP_HANDLE_CONTINUE; + } + + ret = isula_epoll_add_handler(descr, conn_fd, attach_cb, p); + if (ret != SHIM_OK) { + write_attach_message(ERR_MSG, "add recv_fd %d to epoll loop failed:%d", conn_fd, SHIM_SYS_ERR(errno)); + close(conn_fd); + return EPOLL_LOOP_HANDLE_CONTINUE; + } + return EPOLL_LOOP_HANDLE_CONTINUE; +} + static int task_console_accept(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t *descr) { process_t *p = (process_t *)cbdata; @@ -399,6 +726,7 @@ static void stdio_release(int (*stdio_fd)[2]) for (j = 0; j < 2; j++) { if (stdio_fd[i][j] > 0) { close(stdio_fd[i][j]); + stdio_fd[i][j] = -1; } } } @@ -568,24 +896,6 @@ static int open_generic_io(process_t *p, isula_epoll_descr_t *descr) return SHIM_OK; } -static int set_non_block(int fd) -{ - int flag = -1; - int ret = SHIM_ERR; - - flag = fcntl(fd, F_GETFL, 0); - if (flag < 0) { - return SHIM_ERR; - } - - ret = fcntl(fd, F_SETFL, flag | O_NONBLOCK); - if (ret != 0) { - return SHIM_ERR; - } - - return SHIM_OK; -} - /* std_id: channel type isulad_stdio: one side of the isulad fifo file @@ -623,6 +933,14 @@ static void *io_epoll_loop(void *data) exit(EXIT_FAILURE); } + if (p->state->attach_socket != NULL) { + ret = isula_epoll_add_handler(&descr, p->attach_socket_fd, do_attach_socket_accept, p); + if (ret != SHIM_OK) { + write_message(ERR_MSG, "add attach_socket_fd %d to epoll loop failed:%d", p->attach_socket_fd, SHIM_SYS_ERR(errno)); + exit(EXIT_FAILURE); + } + } + if (p->state->terminal) { ret = open_terminal_io(p, &descr); } else { @@ -651,7 +969,7 @@ static void *io_epoll_loop(void *data) } if (fd_out > 0) { - ret = set_non_block(fd_out); + ret = isula_set_non_block(fd_out); if (ret != SHIM_OK) { write_message(ERR_MSG, "set fd %d non_block failed:%d", fd_out, SHIM_SYS_ERR(errno)); exit(EXIT_FAILURE); @@ -666,7 +984,7 @@ static void *io_epoll_loop(void *data) } if (fd_err > 0) { - ret = set_non_block(fd_err); + ret = isula_set_non_block(fd_err); if (ret != SHIM_OK) { write_message(ERR_MSG, "set fd %d non_block failed:%d", fd_err, SHIM_SYS_ERR(errno)); exit(EXIT_FAILURE); @@ -807,15 +1125,19 @@ failure: if (p->isulad_io != NULL) { if (p->isulad_io->in > 0) { close(p->isulad_io->in); + p->isulad_io->in = -1; } if (p->isulad_io->out > 0) { close(p->isulad_io->out); + p->isulad_io->out = -1; } if (p->isulad_io->err > 0) { close(p->isulad_io->err); + p->isulad_io->err = -1; } if (p->isulad_io->resize > 0) { close(p->isulad_io->resize); + p->isulad_io->resize = -1; } free(p->isulad_io); p->isulad_io = NULL; @@ -937,6 +1259,13 @@ process_t *new_process(char *id, char *bundle, char *runtime) goto failure; } + p->attach_fifos = isula_common_calloc_s(sizeof(struct isula_linked_list)); + if (p->attach_fifos == NULL) { + goto failure; + } + + isula_linked_list_init(p->attach_fifos); + return p; failure: @@ -1368,3 +1697,49 @@ int process_signal_handle_routine(process_t *p, const pthread_t tid_epoll, const (void)isula_file_write_nointr(STDOUT_FILENO, &status, sizeof(int)); return SHIM_OK; } + +int prepare_attach_socket(process_t *p) +{ + struct sockaddr_un addr; + int ret = -1; + + if (strlen(p->state->attach_socket) >= sizeof(addr.sun_path)) { + write_message(ERR_MSG, "Invalid attach socket path: %s", p->state->attach_socket); + return SHIM_ERR; + } + + p->attach_socket_fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (p->attach_socket_fd < 0) { + write_message(ERR_MSG, "Failed to create socket:%d", SHIM_SYS_ERR(errno)); + return SHIM_ERR; + } + + (void)memset(&addr, 0, sizeof(addr)); + addr.sun_family = AF_UNIX; + (void)strncpy(addr.sun_path, p->state->attach_socket, sizeof(addr.sun_path) - 1); + + ret = bind(p->attach_socket_fd, (struct sockaddr *)&addr, sizeof(addr)); + if (ret < 0) { + write_message(ERR_MSG, "bind console fd failed:%d", SHIM_SYS_ERR(errno)); + return SHIM_ERR; + } + + ret = chmod(p->state->attach_socket, SOCKET_DIRECTORY_MODE); + if (ret != 0) { + write_message(ERR_MSG, "Failed to chmod for socket: %s", p->state->attach_socket); + return SHIM_ERR; + } + + //If the backlog argument is greater than the value in + // /proc/sys/net/core/somaxconn, then it is silently capped to that + // value. Since Linux 5.4, the default in this file is 4096; in + // earlier kernels, the default value is 128. Before Linux 2.4.25, + // this limit was a hard coded value, SOMAXCONN, with the value 128. + // The maximum number of attach we allow here is MAX_ATTACH_NUM, so just use it directly + ret = listen(p->attach_socket_fd, MAX_ATTACH_NUM); + if (ret < 0) { + write_message(ERR_MSG, "listen console fd failed:%d", SHIM_SYS_ERR(errno)); + return SHIM_ERR; + } + return SHIM_OK; +} \ No newline at end of file diff --git a/src/cmd/isulad-shim/process.h b/src/cmd/isulad-shim/process.h index 280d9874..5607316c 100644 --- a/src/cmd/isulad-shim/process.h +++ b/src/cmd/isulad-shim/process.h @@ -19,7 +19,8 @@ #include #include #include -#include "isula_libutils/shim_client_process_state.h" +#include +#include "isula_libutils/utils_linked_list.h" #include "terminal.h" #ifdef __cplusplus @@ -49,6 +50,7 @@ typedef struct process { char *root_path; int io_loop_fd; int exit_fd; + int attach_socket_fd; // the server socket fd that establishes a connection with isulad int ctr_pid; int sync_fd; int listen_fd; @@ -58,6 +60,7 @@ typedef struct process { stdio_t *stdio; // shim to on runtime side, in:r out/err: w stdio_t *shim_io; // shim io on isulad side, in: w out/err: r stdio_t *isulad_io; // isulad io, in:r out/err: w + struct isula_linked_list *attach_fifos; /* isulad: fifos used to attach teminal */ shim_client_process_state *state; sem_t sem_mainloop; char *buf; @@ -70,6 +73,8 @@ typedef struct { process_t* new_process(char *id, char *bundle, char *runtime); +int prepare_attach_socket(process_t *p); + int process_io_start(process_t *p, pthread_t *tid_epoll); int create_process(process_t *p); int process_signal_handle_routine(process_t *p, const pthread_t tid_epoll, const uint64_t timeout); diff --git a/src/cmd/isulad-shim/terminal.c b/src/cmd/isulad-shim/terminal.c index 0653dc45..1c063300 100644 --- a/src/cmd/isulad-shim/terminal.c +++ b/src/cmd/isulad-shim/terminal.c @@ -162,7 +162,7 @@ static int shim_json_data_write(log_terminal *terminal, const char *buf, int rea * shouldn't happen, otherwise, discard some last bytes. */ nret = isula_file_total_write_nointr(terminal->fd, buf, - terminal->log_maxsize < read_count ? terminal->log_maxsize : read_count); + terminal->log_maxsize < read_count ? terminal->log_maxsize : read_count); if (nret < 0) { ret = -1; goto out; diff --git a/src/daemon/executor/container_cb/execution_stream.c b/src/daemon/executor/container_cb/execution_stream.c index 7db96b19..124dcfe2 100644 --- a/src/daemon/executor/container_cb/execution_stream.c +++ b/src/daemon/executor/container_cb/execution_stream.c @@ -346,6 +346,7 @@ static int container_attach_cb(const container_attach_request *request, containe } params.rootpath = cont->root_path; + params.state = cont->state_path; params.stdin = fifos[0]; params.stdout = fifos[1]; params.stderr = fifos[2]; diff --git a/src/daemon/modules/api/runtime_api.h b/src/daemon/modules/api/runtime_api.h index edb33d02..3c2100f5 100644 --- a/src/daemon/modules/api/runtime_api.h +++ b/src/daemon/modules/api/runtime_api.h @@ -161,6 +161,7 @@ typedef struct _rt_resume_params_t { typedef struct _rt_attach_params_t { const char *rootpath; + const char *state; const char *stdin; const char *stdout; const char *stderr; diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c index cb1ee26f..1787170b 100644 --- a/src/daemon/modules/runtime/isula/isula_rt_ops.c +++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c @@ -18,6 +18,10 @@ #include "isula_rt_ops.h" #include #include +#include +#include +#include +#include #include #include #include @@ -26,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -52,9 +57,11 @@ #define SHIM_BINARY "isulad-shim" #define RESIZE_FIFO_NAME "resize_fifo" +#define ATTACH_SOCKET "attach_socket.sock" #define SHIM_LOG_SIZE ((BUFSIZ - 100) / 2) #define RESIZE_DATA_SIZE 100 #define PID_WAIT_TIME 120 +#define ATTACH_WAIT_TIME 120 #define SHIM_EXIT_TIMEOUT 2 // file name formats of cgroup resources json @@ -223,6 +230,19 @@ static void show_shim_runtime_errlog(const char *workdir) isulad_set_error_message(buf); } +static void show_shim_attach_errlog(const char *workdir) +{ + char buf[SHIM_LOG_SIZE] = { 0 }; + + if (g_isulad_errmsg != NULL) { + return; + } + + get_err_message(buf, sizeof(buf), workdir, "attach-log.json"); + ERROR("shim-log: %s", buf); + isulad_set_error_message("shim-log error:\n%s\n", buf); +} + bool rt_isula_detect(const char *runtime) { if (runtime != NULL && (strcasecmp(runtime, "lcr") != 0)) { @@ -463,8 +483,9 @@ static void runtime_exec_param_init(runtime_exec_info *rei) } } -static int runtime_exec_info_init(runtime_exec_info *rei, const char *workdir, const char *root_path, const char *runtime, const char *subcmd, - const char **opts, size_t opts_len, const char *id, char **params, size_t params_num) +static int runtime_exec_info_init(runtime_exec_info *rei, const char *workdir, const char *root_path, + const char *runtime, const char *subcmd, const char **opts, size_t opts_len, const char *id, char **params, + size_t params_num) { int ret = 0; rei->workdir = workdir; @@ -1012,6 +1033,7 @@ int rt_isula_create(const char *id, const char *runtime, const rt_create_params_ size_t runtime_args_len = 0; int ret = 0; char workdir[PATH_MAX] = { 0 }; + char attach_socket[PATH_MAX] = { 0 }; shim_client_process_state p = { 0 }; int shim_exit_code = 0; int nret = 0; @@ -1034,6 +1056,13 @@ int rt_isula_create(const char *id, const char *runtime, const rt_create_params_ goto out; } + nret = snprintf(attach_socket, sizeof(attach_socket), "%s/%s", workdir, ATTACH_SOCKET); + if (nret < 0 || (size_t)nret >= sizeof(attach_socket)) { + INFO("Failed to get full attach socket path"); + ret = -1; + goto out; + } + p.exit_fifo = (char *)params->exit_fifo; p.open_tty = params->tty; p.open_stdin = params->open_stdin; @@ -1042,6 +1071,7 @@ int rt_isula_create(const char *id, const char *runtime, const rt_create_params_ p.isulad_stderr = (char *)params->stderr; p.runtime_args = (char **)runtime_args; p.runtime_args_len = runtime_args_len; + p.attach_socket = attach_socket; copy_process(&p, config->process); copy_annotations(&p, config->annotations); @@ -1224,7 +1254,7 @@ static bool fg_exec(const rt_exec_params_t *params) return false; } -static char *try_generate_exec_id() +static char *try_generate_random_id() { char *id = NULL; @@ -1324,7 +1354,7 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p if (params->suffix != NULL) { exec_id = util_strdup_s(params->suffix); } else { - exec_id = try_generate_exec_id(); + exec_id = try_generate_random_id(); } if (exec_id == NULL) { ERROR("Out of memory or generate exec id failed"); @@ -1423,13 +1453,133 @@ out: return ret; } -int rt_isula_attach(const char *id, const char *runtime, const rt_attach_params_t *params) +static int get_container_attach_statuscode(const char *workdir, int attach_shim_fd) { - ERROR("isula attach not support on isulad-shim"); - isulad_set_error_message("isula attach not support on isulad-shim"); + int status_code = 0; + int ret = -1; + struct timespec beg = { 0 }; + struct timespec end = { 0 }; + + if (clock_gettime(CLOCK_MONOTONIC, &beg) != 0) { + ERROR("Failed get time"); + return -1; + } + + while (true) { + if (clock_gettime(CLOCK_MONOTONIC, &end) != 0) { + ERROR("Failed get time"); + return -1; + } + if (end.tv_sec - beg.tv_sec > ATTACH_WAIT_TIME) { + ERROR("Wait container attach exitcode timeout"); + return -1; + } + ret = util_read_nointr(attach_shim_fd, &status_code, sizeof(int)); + if (ret <= 0) { + if (shim_alive(workdir)) { + // wait 100 millisecond to read exit code + util_usleep_nointerupt(100000); + continue; + } + ERROR("Failed read pid from dead shim %s", workdir); + return -1; + } + return status_code; /* success */ + } return -1; } +static int get_attach_socketfd(const char *attach_socket, int *socket_fd) +{ + struct sockaddr_un addr = { 0 }; + __isula_auto_close int tmp_socket = -1; + + if (strlen(attach_socket) >= sizeof(addr.sun_path)) { + SYSERROR("Invalid attach socket path: %s", attach_socket); + return -1; + } + + tmp_socket = socket(AF_UNIX, SOCK_STREAM, 0); + if (tmp_socket < 0) { + SYSERROR("Failed to create attach socket"); + return -1; + } + + if (isula_set_non_block(tmp_socket) < 0) { + SYSERROR("Failed to set socket non block"); + return -1; + } + + (void)memset(&addr, 0, sizeof(addr)); + addr.sun_family = AF_UNIX; + (void)strcpy(addr.sun_path, attach_socket); + + if (connect(tmp_socket, (void *)&addr, sizeof(addr)) < 0) { + SYSERROR("Failed to connect attach socket: %s", attach_socket); + return -1; + } + *socket_fd = isula_transfer_fd(tmp_socket); + return 0; +} + +int rt_isula_attach(const char *id, const char *runtime, const rt_attach_params_t *params) +{ + int ret = 0; + int len = 0; + int status_code = 0; + __isula_auto_close int socket_fd = -1; + char buf[BUFSIZ] = { 0 }; + char workdir[PATH_MAX] = { 0 }; + char attach_socket[PATH_MAX] = { 0 }; + + if (id == NULL || runtime == NULL || params == NULL) { + ERROR("Null argument"); + return -1; + } + + ret = snprintf(workdir, sizeof(workdir), "%s/%s", params->state, id); + if (ret < 0 || (size_t)ret >= sizeof(workdir)) { + ERROR("Failed join exec full path"); + return -1; + } + + // the communication format between isulad and isulad-shim attach is: + // stdin-path stdout-path stderr-path + len = snprintf(buf, sizeof(buf), "%s %s %s", params->stdin, params->stdout, params->stderr); + if (len < 0 || (size_t)len >= sizeof(buf)) { + ERROR("Failed to snprintf string"); + return -1; + } + + ret = snprintf(attach_socket, sizeof(attach_socket), "%s/%s", workdir, ATTACH_SOCKET); + if (ret < 0 || (size_t)ret >= sizeof(attach_socket)) { + ERROR("Failed to get full attach socket path"); + return -1; + } + + ret = get_attach_socketfd(attach_socket, &socket_fd); + if (ret < 0) { + ERROR("Failed to get attach socketfd"); + return -1; + } + + DEBUG("write %s to attach fd", buf); + + ret = isula_file_write_nointr(socket_fd, buf, len); + if (ret < 0) { + SYSERROR("Failed to write attach isulad fd"); + return -1; + } + + status_code = get_container_attach_statuscode(workdir, socket_fd); + if (status_code < 0) { + show_shim_attach_errlog(workdir); + return -1; + } + + return 0; +} + static int to_engine_resources_unified(const host_config *hostconfig, shim_client_cgroup_resources *cr) { int i; @@ -1673,7 +1823,7 @@ static int parse_ps_data(char *stdout_msg, rt_listpids_out_t *out) } static int runtime_call_ps(const char *workdir, const char *runtime, const char *id, - rt_listpids_out_t *out) + rt_listpids_out_t *out) { __isula_auto_free char *stdout_msg = NULL; __isula_auto_free char *stderr_msg = NULL; @@ -1681,7 +1831,7 @@ static int runtime_call_ps(const char *workdir, const char *runtime, const char int ret = 0; int nret = 0; char *params[PARAM_NUM] = { 0 }; - const char *opts[2] = { "--format" , "json" }; + const char *opts[2] = { "--format", "json" }; char root_path[PATH_MAX] = { 0 }; nret = snprintf(root_path, PATH_MAX, "%s/%s", workdir, runtime); diff --git a/src/utils/cutils/error.h b/src/utils/cutils/error.h index 088ed261..75eae760 100644 --- a/src/utils/cutils/error.h +++ b/src/utils/cutils/error.h @@ -44,8 +44,10 @@ extern "C" { /* err in runtime module */ \ XX(ERR_RUNTIME, DEF_ERR_RUNTIME_STR) \ \ + /* info for detach */ \ + XX(INFO_DETACH, "Attach detach") \ /* err max */ \ - XX(ERR_UNKNOWN, "Unknown error") + XX(ERR_UNKNOWN, "Unknown error") #define ISULAD_ERRNO_GEN(n, s) ISULAD_##n, typedef enum { ISULAD_ERRNO_MAP(ISULAD_ERRNO_GEN) } isulad_errno_t; -- 2.42.0