From a6f1ff360dded79ce5139a8b97a51c37d2fbd403 Mon Sep 17 00:00:00 2001 From: xuxuepeng Date: Thu, 23 Nov 2023 13:18:13 +0000 Subject: [PATCH 29/64] !2255 Fix cpusets offline issue * Fix cpusets offline issue --- src/daemon/common/sysinfo.c | 1 + src/daemon/common/sysinfo.h | 3 + src/daemon/modules/spec/verify.c | 24 ++--- test/mocks/image_mock.cc | 16 +++ test/mocks/image_mock.h | 2 + test/specs/CMakeLists.txt | 1 + test/specs/verify/CMakeLists.txt | 85 +++++++++++++++ test/specs/verify/verify_ut.cc | 173 +++++++++++++++++++++++++++++++ 8 files changed, 289 insertions(+), 16 deletions(-) create mode 100644 test/specs/verify/CMakeLists.txt create mode 100644 test/specs/verify/verify_ut.cc diff --git a/src/daemon/common/sysinfo.c b/src/daemon/common/sysinfo.c index 957b370b..39338925 100644 --- a/src/daemon/common/sysinfo.c +++ b/src/daemon/common/sysinfo.c @@ -393,6 +393,7 @@ sysinfo_t *get_sys_info(bool quiet) } sysinfo->ncpus = get_nprocs(); + sysinfo->ncpus_conf = get_nprocs_conf(); cgroup_version = common_get_cgroup_version(); if (cgroup_version < 0) { diff --git a/src/daemon/common/sysinfo.h b/src/daemon/common/sysinfo.h index 4ac65df6..363576a9 100644 --- a/src/daemon/common/sysinfo.h +++ b/src/daemon/common/sysinfo.h @@ -25,7 +25,10 @@ extern "C" { #include "cgroup.h" typedef struct { + // Number of processors currently online (i.e., available). int ncpus; + // Number of processors configured. + int ncpus_conf; cgroup_mem_info_t cgmeminfo; cgroup_cpu_info_t cgcpuinfo; cgroup_hugetlb_info_t hugetlbinfo; diff --git a/src/daemon/modules/spec/verify.c b/src/daemon/modules/spec/verify.c index 850595ed..2a8b3259 100644 --- a/src/daemon/modules/spec/verify.c +++ b/src/daemon/modules/spec/verify.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -560,7 +561,7 @@ static bool check_cpu(const char *provided, const char *available) } /* parse unit list */ -int parse_unit_list(const char *val, bool *available_list, int cpu_num) +STATIC int parse_unit_list(const char *val, bool *available_list, int cpu_num) { int ret = -1; char *str = NULL; @@ -612,22 +613,13 @@ out: } /* is cpuset list available */ -static bool is_cpuset_list_available(const char *provided, const char *available) +STATIC bool is_cpuset_list_available(const char *provided, const char *available, int cpu_num) { - int cpu_num = 0; int i = 0; bool ret = false; bool *parsed_provided = NULL; bool *parsed_available = NULL; - sysinfo_t *sysinfo = NULL; - - sysinfo = get_sys_info(true); - if (sysinfo == NULL) { - ERROR("get sysinfo failed"); - return false; - } - cpu_num = sysinfo->ncpus; parsed_provided = util_smart_calloc_s(sizeof(bool), (unsigned int)cpu_num); if (parsed_provided == NULL) { ERROR("memory alloc failed!"); @@ -661,10 +653,10 @@ out: } /* is cpuset cpus available */ -bool is_cpuset_cpus_available(const sysinfo_t *sysinfo, const char *cpus) +STATIC bool is_cpuset_cpus_available(const sysinfo_t *sysinfo, const char *cpus) { bool ret = false; - ret = is_cpuset_list_available(cpus, sysinfo->cpusetinfo.cpus); + ret = is_cpuset_list_available(cpus, sysinfo->cpusetinfo.cpus, sysinfo->ncpus_conf); if (!ret) { ERROR("Checking cpuset.cpus got invalid format: %s.", cpus); isulad_set_error_message("Checking cpuset.cpus got invalid format: %s.", cpus); @@ -673,10 +665,10 @@ bool is_cpuset_cpus_available(const sysinfo_t *sysinfo, const char *cpus) } /* is cpuset mems available */ -bool is_cpuset_mems_available(const sysinfo_t *sysinfo, const char *mems) +STATIC bool is_cpuset_mems_available(const sysinfo_t *sysinfo, const char *mems) { bool ret = false; - ret = is_cpuset_list_available(mems, sysinfo->cpusetinfo.mems); + ret = is_cpuset_list_available(mems, sysinfo->cpusetinfo.mems, sysinfo->ncpus_conf); if (!ret) { ERROR("Checking cpuset.mems got invalid format: %s.", mems); isulad_set_error_message("Checking cpuset.mems got invalid format: %s.", mems); @@ -685,7 +677,7 @@ bool is_cpuset_mems_available(const sysinfo_t *sysinfo, const char *mems) } // cpuset subsystem checks and adjustments -static int verify_resources_cpuset(const sysinfo_t *sysinfo, const char *cpus, const char *mems) +STATIC int verify_resources_cpuset(const sysinfo_t *sysinfo, const char *cpus, const char *mems) { int ret = 0; bool cpus_available = false; diff --git a/test/mocks/image_mock.cc b/test/mocks/image_mock.cc index 7114080c..cebe418d 100644 --- a/test/mocks/image_mock.cc +++ b/test/mocks/image_mock.cc @@ -54,3 +54,19 @@ int im_umount_container_rootfs(const char *image_type, const char *image_name, c } return 0; } + +struct graphdriver_status *im_graphdriver_get_status(void) +{ + if (g_image_mock != nullptr) { + return g_image_mock->ImGraphdriverGetStatus(); + } + return nullptr; +} + +void im_free_graphdriver_status(struct graphdriver_status *status) +{ + if (g_image_mock != nullptr) { + g_image_mock->ImFreeGraphdriverStatus(status); + } + return; +} diff --git a/test/mocks/image_mock.h b/test/mocks/image_mock.h index 0c7c1e51..f05be516 100644 --- a/test/mocks/image_mock.h +++ b/test/mocks/image_mock.h @@ -28,6 +28,8 @@ public: const char *container_id)); MOCK_METHOD3(ImUmountContainerRootfs, int(const char *image_type, const char *image_name, const char *container_id)); + MOCK_METHOD0(ImGraphdriverGetStatus, struct graphdriver_status *()); + MOCK_METHOD1(ImFreeGraphdriverStatus, void(struct graphdriver_status *status)); }; void MockImage_SetMock(MockImage *mock); diff --git a/test/specs/CMakeLists.txt b/test/specs/CMakeLists.txt index 7acd68a1..bf5ed535 100644 --- a/test/specs/CMakeLists.txt +++ b/test/specs/CMakeLists.txt @@ -2,3 +2,4 @@ project(iSulad_UT) add_subdirectory(specs) add_subdirectory(specs_extend) +add_subdirectory(verify) diff --git a/test/specs/verify/CMakeLists.txt b/test/specs/verify/CMakeLists.txt new file mode 100644 index 00000000..0e60a39e --- /dev/null +++ b/test/specs/verify/CMakeLists.txt @@ -0,0 +1,85 @@ +project(iSulad_UT) + +SET(EXE specs_verify_ut) + +add_definitions(-DUNIT_TEST=ON) + +add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_regex.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_verify.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_array.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_string.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_convert.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_file.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_timestamp.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/util_atomic.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_mount_spec.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_fs.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_cap.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/sha256/sha256.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/path.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map/map.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map/rb_tree.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/sysinfo.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup_v1.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/spec/verify.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/image/oci/oci_ut_common.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/containers_store_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/namespace_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/container_unix_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/engine_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/selinux_label_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/isulad_config_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/image_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc + verify_ut.cc) + +target_include_directories(${EXE} PUBLIC + ${GTEST_INCLUDE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/../../include + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/image/oci + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/image + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/image/external + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/common + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/api + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/volume + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/runtime + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/container + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/container/restart_manager + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/container/health_check + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/image/oci/storage + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/services + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/services/execution + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/spec/ + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/services/execution/manager + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/events + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/services/execution/execute + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/tar + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/plugin + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/http + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/runtime/engines + ${ENGINES_INCS} + ${RUNTIME_INCS} + ${IMAGE_INCS} + ${CMAKE_BINARY_DIR}/conf + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/sha256 + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/services/graphdriver + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/console + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/image/oci + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks + ) + +target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) +add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) diff --git a/test/specs/verify/verify_ut.cc b/test/specs/verify/verify_ut.cc new file mode 100644 index 00000000..e764e476 --- /dev/null +++ b/test/specs/verify/verify_ut.cc @@ -0,0 +1,173 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * iSulad licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Description: specs verify ut + * Author: xuxuepeng + * Create: 2023-11-16 + */ + +#include +#include +#include +#include "mock.h" +#include +#include +#include "sysinfo.h" +#include "utils.h" + +using namespace std; + +#define HOST_CONFIG_FILE "../../../../test/specs/verify/hostconfig.json" +#define OCI_RUNTIME_SPEC_FILE "../../../../test/specs/verify/oci_runtime_spec.json" + +extern "C" { + int verify_resources_cpuset(const sysinfo_t *sysinfo, const char *cpus, const char *mems); +} + +/* get sys info */ +sysinfo_t *create_sys_info_for_cpuset_test(const char *cpus, const char *mems, int ncpus_conf, int ncpus) +{ + sysinfo_t *sysinfo = NULL; + + sysinfo = (sysinfo_t *)util_common_calloc_s(sizeof(sysinfo_t)); + if (sysinfo == NULL) { + ERROR("Out of memory"); + return NULL; + } + + sysinfo->ncpus = ncpus; + sysinfo->ncpus_conf = ncpus_conf; + + sysinfo->cpusetinfo.cpuset = true; + sysinfo->cpusetinfo.cpus = util_strdup_s(cpus); + sysinfo->cpusetinfo.mems = util_strdup_s(mems); + + return sysinfo; +} + +void test_different_provided_cpus_mems(sysinfo_t *sysinfo, const char *provided_cpus, const char *provided_mems, + int expected) +{ + int ret = 0; + ret = verify_resources_cpuset(sysinfo, provided_cpus, provided_mems); + ASSERT_EQ(ret, expected); +} + +// Test the case when provided is null, and available is 0-7 +TEST(test_verify_resources_cpuset, test_0_7) +{ + sysinfo_t *sysinfo = create_sys_info_for_cpuset_test("0-7", "0-7", 8, 8); + test_different_provided_cpus_mems(sysinfo, nullptr, nullptr, 0); + + test_different_provided_cpus_mems(sysinfo, "0", "0", 0); + test_different_provided_cpus_mems(sysinfo, "2", "2", 0); + test_different_provided_cpus_mems(sysinfo, "7", "7", 0); + test_different_provided_cpus_mems(sysinfo, "8", "8", -1); + + test_different_provided_cpus_mems(sysinfo, "1,2", "1,2", 0); + test_different_provided_cpus_mems(sysinfo, "1,3,5", "1,3,5", 0); + + test_different_provided_cpus_mems(sysinfo, "0-7", "0-7", 0); + test_different_provided_cpus_mems(sysinfo, "0-8", "0-8", -1); + test_different_provided_cpus_mems(sysinfo, "0-1,3-7", "0-1,3-7", 0); + test_different_provided_cpus_mems(sysinfo, "0-1,3,5-7", "0-1,3,5-7", 0); + + free_sysinfo(sysinfo); +} + +// Test the case when provided is null, and available is 0-1,3-7 +TEST(test_verify_resources_cpuset, test_0_1_3_7) +{ + sysinfo_t *sysinfo = create_sys_info_for_cpuset_test("0-1,3-7", "0-1,3-7", 8, 7); + test_different_provided_cpus_mems(sysinfo, nullptr, nullptr, 0); + + test_different_provided_cpus_mems(sysinfo, "0", "0", 0); + test_different_provided_cpus_mems(sysinfo, "2", "2", -1); + test_different_provided_cpus_mems(sysinfo, "7", "7", 0); + test_different_provided_cpus_mems(sysinfo, "8", "8", -1); + + test_different_provided_cpus_mems(sysinfo, "1,2", "1,2", -1); + test_different_provided_cpus_mems(sysinfo, "1,3,5", "1,3,5", 0); + + test_different_provided_cpus_mems(sysinfo, "0-7", "0-7", -1); + test_different_provided_cpus_mems(sysinfo, "0-8", "0-8", -1); + test_different_provided_cpus_mems(sysinfo, "0-1,3-7", "0-1,3-7", 0); + test_different_provided_cpus_mems(sysinfo, "0-1,3,5-7", "0-1,3,5-7", 0); + + free_sysinfo(sysinfo); +} + +// Test the case when provided is null, and available is 0-6 +TEST(test_verify_resources_cpuset, test_0_6) +{ + sysinfo_t *sysinfo = create_sys_info_for_cpuset_test("0-6", "0-6", 8, 7); + + test_different_provided_cpus_mems(sysinfo, nullptr, nullptr, 0); + + test_different_provided_cpus_mems(sysinfo, "0", "0", 0); + test_different_provided_cpus_mems(sysinfo, "2", "2", 0); + test_different_provided_cpus_mems(sysinfo, "7", "7", -1); + test_different_provided_cpus_mems(sysinfo, "8", "8", -1); + + test_different_provided_cpus_mems(sysinfo, "1,2", "1,2", 0); + test_different_provided_cpus_mems(sysinfo, "1,3,5", "1,3,5", 0); + + test_different_provided_cpus_mems(sysinfo, "0-7", "0-7", -1); + test_different_provided_cpus_mems(sysinfo, "0-8", "0-8", -1); + test_different_provided_cpus_mems(sysinfo, "0-1,3-7", "0-1,3-7", -1); + test_different_provided_cpus_mems(sysinfo, "0-1,3,5-7", "0-1,3,5-7", -1); + + free_sysinfo(sysinfo); +} + +// Test the case when provided is null, and available is 1-7 +TEST(test_verify_resources_cpuset, test_1_7) +{ + sysinfo_t *sysinfo = create_sys_info_for_cpuset_test("1-7", "1-7", 8, 7); + + test_different_provided_cpus_mems(sysinfo, nullptr, nullptr, 0); + + test_different_provided_cpus_mems(sysinfo, "0", "0", -1); + test_different_provided_cpus_mems(sysinfo, "2", "2", 0); + test_different_provided_cpus_mems(sysinfo, "7", "7", 0); + test_different_provided_cpus_mems(sysinfo, "8", "8", -1); + + test_different_provided_cpus_mems(sysinfo, "1,2", "1,2", 0); + test_different_provided_cpus_mems(sysinfo, "1,3,5", "1,3,5", 0); + + test_different_provided_cpus_mems(sysinfo, "0-7", "0-7", -1); + test_different_provided_cpus_mems(sysinfo, "0-8", "0-8", -1); + test_different_provided_cpus_mems(sysinfo, "0-1,3-7", "0-1,3-7", -1); + test_different_provided_cpus_mems(sysinfo, "0-1,3,5-7", "0-1,3,5-7", -1); + + free_sysinfo(sysinfo); +} + +// Test the case when provided is null, and available is 0,3 +TEST(test_verify_resources_cpuset, test_null_03) +{ + sysinfo_t *sysinfo = create_sys_info_for_cpuset_test("0,3", "0,3", 8, 2); + test_different_provided_cpus_mems(sysinfo, nullptr, nullptr, 0); + + test_different_provided_cpus_mems(sysinfo, "0", "0", 0); + test_different_provided_cpus_mems(sysinfo, "2", "2", -1); + test_different_provided_cpus_mems(sysinfo, "7", "7", -1); + test_different_provided_cpus_mems(sysinfo, "8", "8", -1); + + test_different_provided_cpus_mems(sysinfo, "1,2", "1,2", -1); + test_different_provided_cpus_mems(sysinfo, "1,3,5", "1,3,5", -1); + + test_different_provided_cpus_mems(sysinfo, "0-7", "0-7", -1); + test_different_provided_cpus_mems(sysinfo, "0-8", "0-8", -1); + test_different_provided_cpus_mems(sysinfo, "0-1,3-7", "0-1,3-7", -1); + test_different_provided_cpus_mems(sysinfo, "0-1,3,5-7", "0-1,3,5-7", -1); + + free_sysinfo(sysinfo); +} -- 2.42.0