diff options
Diffstat (limited to '0001-elf-dynamic-linker-load-shared-object-use-hugepage-a.patch')
| -rw-r--r-- | 0001-elf-dynamic-linker-load-shared-object-use-hugepage-a.patch | 1206 |
1 files changed, 1206 insertions, 0 deletions
diff --git a/0001-elf-dynamic-linker-load-shared-object-use-hugepage-a.patch b/0001-elf-dynamic-linker-load-shared-object-use-hugepage-a.patch new file mode 100644 index 0000000..fdbdead --- /dev/null +++ b/0001-elf-dynamic-linker-load-shared-object-use-hugepage-a.patch @@ -0,0 +1,1206 @@ +From d3a58c218f2c1605499be11a4b3a21f8ef133180 Mon Sep 17 00:00:00 2001 +From: Lv Ying <lvying6@huawei.com> +Date: Mon, 7 Mar 2022 03:28:33 +0000 +Subject: [PATCH] elf: dynamic linker load shared object use hugepage as much + as possible + +This patch provides environment variables HUGEPAGE_PROBE and LD_HUGEPAGE_LIB +to enable load shared object use hugepage. + +When load shared object, ld.so first to map text PT_LOAD segment into 2MB +hugepage area. And then, load the neighbor PT_LOAD segment use 2MB hugepage +as much as possible. This means: +* PT_LOAD segment's mapstart_va is 2MB aligned, howerver its maplenth is + less than 2MB, fallback to 4KB page +* PT_LOAD segment's mapstart_va is 2MB aligned, and its maplenth is larger + than 2MB, the first 2MB aligned area use 2MB hugepage, the end area (if it exists) use 4KB area +* PT_LOAD segment's mapstart_va is not 2MB aligned, alignup this address + to 2MB aligned address mapstart_align, if its maplenth is less than + mapstart_align - mapstart_va, or maplenth - (mapstart_align - mapstart_va), fallback to 4KB hugepage +* PT_LOAD segment's mapstart_va is not 2MB aligned, maplenth - (mapstart_align - mapstart_va) + is still larger than 2MB, first map (mapstart_align - mapstart_va) as 4KB page + then map 2MB aligned area as hugepage, the end area (if it exists) use 4KB area +--- + config.h.in | 2 + + configure | 20 ++ + configure.ac | 11 + + elf/Makefile | 8 + + elf/dl-environ.c | 11 + + elf/dl-load.c | 38 +++ + elf/dl-load.h | 16 + + elf/dl-map-segments-hugepage.h | 593 +++++++++++++++++++++++++++++++++ + elf/dl-support.c | 4 + + elf/elf.h | 2 + + elf/hugepageedit.c | 169 ++++++++++ + elf/rtld.c | 63 ++++ + sysdeps/generic/ldsodefs.h | 11 +- + 13 files changed, 947 insertions(+), 1 deletion(-) + create mode 100644 elf/dl-map-segments-hugepage.h + create mode 100644 elf/hugepageedit.c + +diff --git a/config.h.in b/config.h.in +index db6402cd..13101496 100644 +--- a/config.h.in ++++ b/config.h.in +@@ -277,6 +277,8 @@ + /* Define if static PIE is supported. */ + #undef SUPPORT_STATIC_PIE + ++#undef HUGEPAGE_SHARED_LIB ++ + /* Define if static PIE is enabled. */ + #define ENABLE_STATIC_PIE 0 + +diff --git a/configure b/configure +index 7272fbf6..43993923 100755 +--- a/configure ++++ b/configure +@@ -670,6 +670,7 @@ stack_protector + libc_cv_ssp + libc_cv_with_fp + base_machine ++enable_hugepage_shared_library + have_tunables + build_pt_chown + build_nscd +@@ -791,6 +792,7 @@ enable_pt_chown + enable_tunables + enable_mathvec + enable_cet ++enable_hugepage_shared_library + enable_scv + with_cpu + ' +@@ -1464,6 +1466,9 @@ Optional Features: + depends on architecture] + --enable-cet enable Intel Control-flow Enforcement Technology + (CET), x86 only ++ --enable-hugepage-shared-library ++ enable shared library use huge page to decrease TLB ++ miss, x86_64 aarch64 only + --disable-scv syscalls will not use scv instruction, even if the + kernel supports it, powerpc only + +@@ -3830,6 +3835,21 @@ if test "$use_scv" != "no"; then : + + fi + ++# Check whether --enable-hugepage-shared-library was given. ++if test "${enable_hugepage_shared_library+set}" = set; then : ++ enableval=$enable_hugepage_shared_library; enable_hugepage_shared_library=$enableval ++else ++ enable_hugepage_shared_library=no ++fi ++ ++ ++config_vars="$config_vars ++enable-hugepage-shared-library = $enable_hugepage_shared_library" ++if test "$enable_hugepage_shared_library" = yes; then ++ $as_echo "#define HUGEPAGE_SHARED_LIB 1" >>confdefs.h ++ ++fi ++ + # We keep the original values in `$config_*' and never modify them, so we + # can write them unchanged into config.make. Everything else uses + # $machine, $vendor, and $os, and changes them whenever convenient. +diff --git a/configure.ac b/configure.ac +index af47cd51..27a48338 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -478,6 +478,17 @@ AC_ARG_ENABLE([scv], + + AS_IF([[test "$use_scv" != "no"]],[AC_DEFINE(USE_PPC_SCV)]) + ++AC_ARG_ENABLE([hugepage-shared-library], ++ AC_HELP_STRING([--enable-hugepage-shared-library], ++ [enable shared library use huge page to decrease TLB miss, x86_64 aarch64 only]), ++ [enable_hugepage_shared_library=$enableval], ++ [enable_hugepage_shared_library=no]) ++ ++LIBC_CONFIG_VAR([enable-hugepage-shared-library], [$enable_hugepage_shared_library]) ++if test "$enable_hugepage_shared_library" = yes; then ++ AC_DEFINE(HUGEPAGE_SHARED_LIB) ++fi ++ + # We keep the original values in `$config_*' and never modify them, so we + # can write them unchanged into config.make. Everything else uses + # $machine, $vendor, and $os, and changes them whenever convenient. +diff --git a/elf/Makefile b/elf/Makefile +index 6fd515ba..32aba7ac 100644 +--- a/elf/Makefile ++++ b/elf/Makefile +@@ -207,6 +207,14 @@ others-extras = $(ldconfig-modules) + endif + endif + ++ifeq (yes,$(enable-hugepage-shared-library)) ++others += hugepageedit ++others-pie += hugepageedit ++install-bin += hugepageedit ++ ++$(objpfx)hugepageedit: $(objpfx)hugepageedit.o ++endif ++ + # To find xmalloc.c and xstrdup.c + vpath %.c ../locale/programs + +diff --git a/elf/dl-environ.c b/elf/dl-environ.c +index 31c1c09f..ac70c9ab 100644 +--- a/elf/dl-environ.c ++++ b/elf/dl-environ.c +@@ -31,6 +31,17 @@ _dl_next_ld_env_entry (char ***position) + + while (*current != NULL) + { ++#ifdef HUGEPAGE_SHARED_LIB ++ #define LEN_HUGEPAGE_PROBE (sizeof("HUGEPAGE_PROBE") - 1) ++ if (memcmp (*current, "HUGEPAGE_PROBE", LEN_HUGEPAGE_PROBE) == 0) ++ { ++ result = *current; ++ ++ /* Save current position for next visit. */ ++ *position = ++current; ++ break; ++ } ++#endif + if (__builtin_expect ((*current)[0] == 'L', 0) + && (*current)[1] == 'D' && (*current)[2] == '_') + { +diff --git a/elf/dl-load.c b/elf/dl-load.c +index 0976977f..f4b5c4a7 100644 +--- a/elf/dl-load.c ++++ b/elf/dl-load.c +@@ -73,6 +73,9 @@ struct filebuf + #include <dl-sysdep-open.h> + #include <dl-prop.h> + #include <not-cancel.h> ++#ifdef HUGEPAGE_SHARED_LIB ++#include <dl-map-segments-hugepage.h> ++#endif + + #include <endian.h> + #if BYTE_ORDER == BIG_ENDIAN +@@ -1131,6 +1134,9 @@ _dl_map_object_from_fd (const char *name, const char *origname, int fd, + size_t nloadcmds = 0; + bool has_holes = false; + bool empty_dynamic = false; ++#ifdef HUGEPAGE_SHARED_LIB ++ bool use_hugepage = false; ++#endif + + /* The struct is initialized to zero so this is not necessary: + l->l_ld = 0; +@@ -1188,6 +1194,11 @@ _dl_map_object_from_fd (const char *name, const char *origname, int fd, + if (nloadcmds > 1 && c[-1].mapend != c->mapstart) + has_holes = true; + ++#ifdef HUGEPAGE_SHARED_LIB ++ if (ph->p_flags & PF_HUGEPAGE) ++ use_hugepage = true; ++#endif ++ + /* Optimize a common case. */ + #if (PF_R | PF_W | PF_X) == 7 && (PROT_READ | PROT_WRITE | PROT_EXEC) == 7 + c->prot = (PF_TO_PROT +@@ -1278,12 +1289,39 @@ _dl_map_object_from_fd (const char *name, const char *origname, int fd, + /* Length of the sections to be loaded. */ + maplength = loadcmds[nloadcmds - 1].allocend - loadcmds[0].mapstart; + ++#ifdef HUGEPAGE_SHARED_LIB ++#define ERRSTRING_BUF_LEN 1024 ++ int hp_errcode = 0; ++ char hp_buf[ERRSTRING_BUF_LEN]; ++ if ((GLRO(dl_hugepage_mask) & DL_HUGEPAGE_LIB_LARGE_IN_FLAG) || ++ ((GLRO(dl_hugepage_mask) & DL_HUGEPAGE_PROBE_FLAG) && use_hugepage)) ++ { ++ errstring = _dl_map_segments_largein (l, fd, header, type, loadcmds, nloadcmds, ++ maplength, has_holes); ++ if (__glibc_unlikely (errstring != NULL)) ++ { ++ hp_errcode = errno; ++ /* __strerror_r will set hp_buf last character '\0', hp_buf will not overflow */ ++ if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_FILES)) ++ _dl_debug_printf ("_dl_map_segments_largein: %s, %s\n", errstring, ++ hp_errcode ? __strerror_r (hp_errcode, hp_buf, sizeof hp_buf) : ""); ++ goto fallback; ++ } ++ } ++ else ++ { ++ fallback: ++ errstring = _dl_map_segments (l, fd, header, type, loadcmds, nloadcmds, ++ maplength, has_holes, loader); ++ } ++#else + /* Now process the load commands and map segments into memory. + This is responsible for filling in: + l_map_start, l_map_end, l_addr, l_contiguous, l_text_end, l_phdr + */ + errstring = _dl_map_segments (l, fd, header, type, loadcmds, nloadcmds, + maplength, has_holes, loader); ++#endif + if (__glibc_unlikely (errstring != NULL)) + { + /* Mappings can be in an inconsistent state: avoid unmap. */ +diff --git a/elf/dl-load.h b/elf/dl-load.h +index e329d49a..d3f69466 100644 +--- a/elf/dl-load.h ++++ b/elf/dl-load.h +@@ -131,5 +131,21 @@ static const char *_dl_map_segments (struct link_map *l, int fd, + #define DL_MAP_SEGMENTS_ERROR_MAP_ZERO_FILL \ + N_("cannot map zero-fill pages") + ++#ifdef HUGEPAGE_SHARED_LIB ++#define DL_MAP_SEGMENTS_ERROR_TYPE \ ++ N_("cannot map Non shared object file in hugepage") ++#define DL_MAP_SEGMENTS_ERROR_READ_SEGMENT \ ++ N_("failed to read shared object file") ++#define DL_MAP_SEGMENTS_ERROR_ARRANGE \ ++ N_("shared object's PT_LOAD segment in wrong arrange") ++#define DL_MAP_SEGMENTS_ERROR_MAP_HOLE_FILL \ ++ N_("failed to mmap shared object's hole part of PT_LOAD") ++#define DL_MAP_RESERVED_HUGEPAGE_AREA_ERROR \ ++ N_("failed to map reserved 2MB contiguous hugepage va space") ++#define DL_FIND_EXEC_SEGMENT_ERROR \ ++ N_("fail to find exec prot segment") ++#define DL_MAP_SEGMENT_ERROR_EXTRA_SIZE \ ++ N_("wrong segment extra size") ++#endif + + #endif /* dl-load.h */ +diff --git a/elf/dl-map-segments-hugepage.h b/elf/dl-map-segments-hugepage.h +new file mode 100644 +index 00000000..cd7b6d79 +--- /dev/null ++++ b/elf/dl-map-segments-hugepage.h +@@ -0,0 +1,593 @@ ++/* Map a shared object's segments into hugepage. Generic version. ++ Copyright (C) 1995-2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <http://www.gnu.org/licenses/>. */ ++ ++#include <dl-load.h> ++ ++#define SHFIT_2MB 21 ++#define SIZE_2MB 0x200000 ++#define MASK_2MB 0x1FFFFF ++#define THRESHOLD 16 ++ ++/* ++ * Find the first PT_LOAD segment with execute permission ++ */ ++static __always_inline const struct loadcmd * ++_find_exec_segment(const struct loadcmd loadcmds[], size_t nloadcmds) ++{ ++ const struct loadcmd *c = loadcmds; ++ ++ while (c < &loadcmds[nloadcmds]) ++ { ++ if (c->prot & PROT_EXEC) ++ return c; ++ c++; ++ } ++ return NULL; ++} ++ ++static __always_inline void * ++__mmap_reserved_area(const struct loadcmd loadcmds[], size_t nloadcmds, ++ size_t *maparealen) ++{ ++ const struct loadcmd * c = loadcmds; ++ *maparealen = 0; ++ ++ while (c < &loadcmds[nloadcmds]) ++ { ++ *maparealen += ALIGN_UP((c->mapend > c->allocend ? c->mapend : c->allocend), SIZE_2MB) - ++ ALIGN_DOWN(c->mapstart, SIZE_2MB); ++ c++; ++ } ++ ++ /* ++ * Get 2MB aligned contiguous va space ++ * This va space can not be munmap in case of multi thread dlopen concurrently ++ */ ++ void *map_area_start = __mmap(0, *maparealen, PROT_NONE, ++ MAP_PRIVATE|MAP_ANONYMOUS|MAP_HUGETLB|(SHFIT_2MB << MAP_HUGE_SHIFT), -1, 0); ++ if (__glibc_unlikely (map_area_start == MAP_FAILED)) ++ return MAP_FAILED; ++ ++ /* ++ * Remap 2MB aligned contiguous va space into 4KB contiguous va space ++ * to avoid the tedious work of splitting hugepage into 4KB page ++ */ ++ if (__glibc_unlikely(__mmap(map_area_start, *maparealen, PROT_NONE, ++ MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, -1, 0) ++ == MAP_FAILED)) ++ { ++ goto unmap_reserved_area; ++ } ++ return map_area_start; ++ ++unmap_reserved_area: ++ __munmap(map_area_start, *maparealen); ++ return MAP_FAILED; ++} ++ ++static __always_inline size_t ++_get_relro_len(struct link_map *l, const struct loadcmd *c) ++{ ++ size_t relro_len = 0; ++ if (c->mapstart == ALIGN_DOWN (l->l_relro_addr, GLRO(dl_pagesize))) ++ { ++ relro_len = ALIGN_DOWN(l->l_relro_addr + l->l_relro_size, GLRO(dl_pagesize)) - ++ ALIGN_DOWN(l->l_relro_addr, GLRO(dl_pagesize)); ++ } ++ return relro_len; ++} ++ ++/* ++ * the alignment stands for the size of page which is to be cleared to zero ++ */ ++static __always_inline const char * ++_zero_tail_page(const struct loadcmd *c, ElfW(Addr) zero, ElfW(Addr) zeropage, ++ size_t alignment) ++{ ++ if (__glibc_unlikely ((c->prot & PROT_WRITE) == 0)) ++ { ++ /* Dag nab it. */ ++ if (__mprotect ((caddr_t) ALIGN_DOWN(zero, alignment), alignment, ++ c->prot|PROT_WRITE) < 0) ++ return DL_MAP_SEGMENTS_ERROR_MPROTECT; ++ } ++ memset ((void *) zero, '\0', zeropage - zero); ++ if (__glibc_unlikely ((c->prot & PROT_WRITE) == 0)) ++ __mprotect ((caddr_t) ALIGN_DOWN(zero, alignment), alignment, c->prot); ++ return NULL; ++} ++ ++static __always_inline const char * ++_mmap_remain_zero_page(ElfW(Addr) zeropage, ElfW(Addr) zeroend, int prot) ++{ ++ ElfW(Addr) hp_start = ALIGN_UP(zeropage, SIZE_2MB); ++ size_t len = 0, mod = 0; ++ caddr_t mapat; ++ ++ if (zeroend > hp_start && zeroend - hp_start >= SIZE_2MB) ++ { ++ len = zeroend - hp_start; ++ mod = len % SIZE_2MB; ++ } ++ else ++ hp_start = 0; ++ ++ if (hp_start == 0) ++ { ++ mapat = __mmap((caddr_t) zeropage, zeroend - zeropage, prot, ++ MAP_ANON|MAP_PRIVATE|MAP_FIXED, -1, 0); ++ if (__glibc_unlikely (mapat == MAP_FAILED)) ++ return DL_MAP_SEGMENTS_ERROR_MAP_ZERO_FILL; ++ return NULL; ++ } ++ ++ if (hp_start - zeropage > 0) ++ { ++ mapat = __mmap((caddr_t) zeropage, hp_start - zeropage, ++ prot, MAP_ANON|MAP_PRIVATE|MAP_FIXED, -1, 0); ++ if (__glibc_unlikely (mapat == MAP_FAILED)) ++ return DL_MAP_SEGMENTS_ERROR_MAP_ZERO_FILL; ++ } ++ ++ if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_FILES)) ++ _dl_debug_printf("\t\t\t=> mmap anonymous hugepage: [%lx-%lx)\n", hp_start, hp_start + len - mod); ++ mapat = __mmap((caddr_t) hp_start, len - mod, prot, ++ MAP_FIXED|MAP_PRIVATE|MAP_ANONYMOUS|MAP_HUGETLB|(SHFIT_2MB << MAP_HUGE_SHIFT), ++ -1, 0); ++ if (__glibc_unlikely (mapat == MAP_FAILED)) ++ return DL_MAP_SEGMENTS_ERROR_MAP_ZERO_FILL; ++ ++ if (mod > 0) ++ { ++ mapat =__mmap((caddr_t)(hp_start + len - mod), mod, prot, ++ MAP_ANON|MAP_PRIVATE|MAP_FIXED, -1, 0); ++ if (__glibc_unlikely (mapat == MAP_FAILED)) ++ return DL_MAP_SEGMENTS_ERROR_MAP_ZERO_FILL; ++ } ++ ++ return NULL; ++} ++ ++/* ++ * memsz_len records the remain memsiz part ++ */ ++static __always_inline const char * ++_mmap_segment_memsz(struct link_map *l, const struct loadcmd * c, ++ ElfW(Addr) mapstart, size_t extra_len, size_t *memsz_len) ++{ ++ const char * errstring = NULL; ++ ++ /* Extra zero pages should appear at the end of this segment, ++ after the data mapped from the file. */ ++ ElfW(Addr) zero, zeroend, zeropage; ++ ++ zero = mapstart + c->dataend - c->mapstart; ++ zeroend = mapstart + c->allocend - c->mapstart; ++ zeropage = ALIGN_UP(zero, GLRO(dl_pagesize)); ++ size_t alignment = GLRO(dl_pagesize); ++ *memsz_len = 0; ++ ++ /* ++ * no matter what the extra space consists of: ++ * 1. all the extra space is initialized data area (MemSiz > FileSiz) ++ * 2. initialized data area and hole ++ * 3. all the extra space is hole (MemSiz == FileSiz) ++ * ++ * the extra space just needs to be set zero, for the initialized data area, it's ++ * initialized to zero; for the hole area, it's initialized to invalid instruction ++ */ ++ if (extra_len > 0) ++ { ++ if (__glibc_unlikely(zeropage == ALIGN_UP(zero, SIZE_2MB) || ++ zeropage + extra_len != ALIGN_UP(zero, SIZE_2MB))) ++ return DL_MAP_SEGMENT_ERROR_EXTRA_SIZE; ++ ++ zeropage = ALIGN_UP(zero, SIZE_2MB); ++ alignment = SIZE_2MB; ++ } ++ else ++ { ++ /* ++ * extra_len = 0, _mmap_segment_filesz just mmap segment's FileSiz part, ++ * here, it needs to zero tail page [FileSiz end, tail page end) part ++ */ ++ if (c->allocend <= c->dataend) ++ return NULL; ++ ++ if (ALIGN_UP(zero, GLRO(dl_pagesize)) == ALIGN_UP(zero, SIZE_2MB) && ++ (zeropage - (mapstart + _get_relro_len(l, c)) >= SIZE_2MB)) ++ { ++ alignment = SIZE_2MB; ++ } ++ ++ if (zeroend < zeropage) ++ zeropage = zeroend; ++ } ++ ++ if (zeropage > zero) ++ { ++ if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_FILES)) ++ _dl_debug_printf("\t\tzero tail page [%lx-%lx) which contains hole area length: 0x%lx\n", ++ zero, zeropage, zeropage > ALIGN_UP(zero, GLRO(dl_pagesize)) ? ++ zeropage - ALIGN_UP(zero, GLRO(dl_pagesize)) : 0); ++ errstring = _zero_tail_page(c, zero, zeropage, alignment); ++ if (errstring != NULL) ++ return errstring; ++ } ++ ++ if (zeroend > zeropage) ++ { ++ *memsz_len = ALIGN_UP(zeroend, GLRO(dl_pagesize)) - zeropage; ++ if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_FILES)) ++ _dl_debug_printf("\t\tzero remain page [%lx-%lx)\n", zeropage, zeroend); ++ errstring = _mmap_remain_zero_page(zeropage, zeroend, c->prot); ++ } ++ return errstring; ++} ++ ++/* ++ * mmap as fixed addr, if the middle part is 2MB aligned, ++ * this part should be mmaped in 2MB aligned, else in 4KB aligned ++ * 2MB hugepage area should be set with correct permissions, no need to remap ++ */ ++static __always_inline const char * ++_mmap_segment_filesz(struct link_map *l, const struct loadcmd *c, ElfW(Addr) mapstart, ++ size_t extra_len, int fd) ++{ ++ void *map_addr = 0; ++ ++ size_t relro_len = _get_relro_len(l, c); ++ if (relro_len > 0) ++ { ++ if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_FILES)) ++ _dl_debug_printf("\t\tmmap relro: [%lx-%lx)\n", mapstart, mapstart + relro_len); ++ /* ++ * relro part must be mapped as normal page size to avoid ++ * _dl_protect_relro failure ++ */ ++ map_addr = __mmap((void *)mapstart, relro_len, c->prot, ++ MAP_PRIVATE|MAP_FIXED|MAP_FILE, ++ fd, c->mapoff); ++ if (__glibc_unlikely (map_addr == MAP_FAILED)) ++ return DL_MAP_SEGMENTS_ERROR_MAP_SEGMENT; ++ ++ mapstart += relro_len; ++ } ++ ++ size_t prev_map_len = ALIGN_UP(mapstart, SIZE_2MB) - mapstart; ++ size_t len = (c->mapend + extra_len) - (c->mapstart + relro_len); ++ if (len <= prev_map_len || len - prev_map_len < SIZE_2MB) ++ { ++ if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_FILES)) ++ _dl_debug_printf("\t\tmmap all: [%lx-%lx), which includes prev_map_len(0x%lx)\n", ++ mapstart, mapstart + len, prev_map_len); ++ mapstart = (ElfW(Addr))__mmap((void *)mapstart, len, c->prot, ++ MAP_PRIVATE|MAP_FIXED|MAP_FILE, ++ fd, c->mapoff + relro_len); ++ if (__glibc_unlikely ((void *)mapstart == MAP_FAILED)) ++ return DL_MAP_SEGMENTS_ERROR_MAP_SEGMENT; ++ return NULL; ++ } ++ ++ if (prev_map_len > 0) ++ { ++ if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_FILES)) ++ _dl_debug_printf("\t\tmmap prev_map_len: [%lx-%lx)\n", ++ mapstart, mapstart + prev_map_len); ++ mapstart = (ElfW(Addr))__mmap((void *)mapstart, prev_map_len, c->prot, ++ MAP_PRIVATE|MAP_FIXED|MAP_FILE, ++ fd, c->mapoff + relro_len); ++ if (__glibc_unlikely ((void *)mapstart == MAP_FAILED)) ++ return DL_MAP_SEGMENTS_ERROR_MAP_SEGMENT; ++ ++ map_addr = map_addr == 0 ? (void *)mapstart : map_addr; ++ mapstart += prev_map_len; ++ len -= prev_map_len; ++ } ++ ++ size_t mod = len % SIZE_2MB; ++ if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_FILES)) ++ _dl_debug_printf("\t\tmmap hugepage: [%lx-%lx)\n", mapstart, mapstart + len - mod); ++ mapstart = (ElfW(Addr))__mmap((void *)mapstart, len - mod, c->prot, ++ MAP_FIXED|MAP_PRIVATE|MAP_ANONYMOUS|MAP_HUGETLB|(SHFIT_2MB << MAP_HUGE_SHIFT), ++ -1, 0); ++ if (__glibc_unlikely ((void *)mapstart == MAP_FAILED)) ++ return DL_MAP_SEGMENTS_ERROR_MAP_SEGMENT; ++ ++ if ((c->prot & PROT_WRITE) == 0 && __mprotect((void *)mapstart, len - mod, c->prot | PROT_WRITE) < 0) ++ { ++ return DL_MAP_SEGMENTS_ERROR_MPROTECT; ++ } ++ ++ /* Read the segment contents from the file. */ ++ size_t file_len = (size_t)(c->dataend - c->mapstart) <= prev_map_len + relro_len ? 0 : ++ (size_t)(c->dataend - c->mapstart) - prev_map_len - relro_len; ++ if (file_len > 0) ++ { ++ lseek(fd, c->mapoff + relro_len + prev_map_len, SEEK_SET); ++ if ( __read(fd, (void *)mapstart, file_len < len - mod ? file_len : len - mod) < 0) ++ return DL_MAP_SEGMENTS_ERROR_READ_SEGMENT; ++ } ++ ++ if ((c->prot & PROT_WRITE) == 0 && __mprotect((void *)mapstart, len - mod, c->prot) < 0) ++ { ++ return DL_MAP_SEGMENTS_ERROR_MPROTECT; ++ } ++ ++ map_addr = map_addr == 0 ? (void *)mapstart : map_addr; ++ mapstart += len - mod; ++ ++ if (__glibc_unlikely (extra_len > 0 && mod > 0)) ++ return DL_MAP_SEGMENT_ERROR_EXTRA_SIZE; ++ ++ if (mod > 0 && __glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_FILES)) ++ _dl_debug_printf("\t\tmmap tail part: [%lx-%lx)\n", mapstart, mapstart + mod); ++ if (mod > 0 && __mmap((void *)mapstart, mod, c->prot, ++ MAP_PRIVATE|MAP_FIXED|MAP_FILE, ++ fd, c->mapoff + relro_len + prev_map_len + len - mod) ++ == MAP_FAILED) ++ { ++ return DL_MAP_SEGMENTS_ERROR_MAP_SEGMENT; ++ } ++ return NULL; ++} ++ ++/* ++ * mmap segment filesz tail part only covers the very first part of hugepage, ++ * if the size of this tail part reach the threshold, map the tail part in hugepage ++ * ++ * The tail part must be calculated by mapend, because this is file mmaping, ++ * if tail part is calculated by allocend, it will mmap invalid data in file ++ * s: mapstart mp: mapend ac: allocend ++ * 1. [s, mp) can not cover the tail hugepage start, mp, s, ac are all in same hugepage, no extra space ++ * s mp ac ++ * | | | ++ * |--------|--------| ++ * ++ * 2. [s, mp) can not cover the tail hugepage start, ac is in the behind hugepage, no extra space ++ * s mp ac ++ * | | | ++ * |--------|--------|--------| ++ * ++ * 3. [s, mp) covers the tail hugepage start, mp and the ac in the same hugepage, ++ * if (ac - ALIGN_DOWN(mp, SIZE_2MB) < threshold, no extra space; else extra space ++ * [mp, ALIGN_UP(mp, SIZE_2MB) which contains initialized data area and hole ++ * if ac == mp, the extra space only contains hole ++ * s1 s2 mp ac ++ * | | | | ++ * |--------|--------|--------| ++ * ++ * 4. [s, mp) covers the tail hugepage start, ac is in the behind hugepage, ++ * the extra space is [mp, ALIGN_UP(mp, SIZE_2MB) which only contains initialized data area ++ * s1 s2 mp ac ++ * | | | | ++ * |--------|--------|--------|--------|--------| ++ * ++ * 5. if mp is 2MB aligned, no matter [s, mp) covers the tail hugepage start or not, ++ * no extra area ++ * s1 s2 s3 mp ac ++ * | | | | | ++ * |--------|--------|--------|--------|--------| ++ * ++ * there are a few points to note: ++ * 1. the extra part shold not overlap with the next segment ++ * 2. PT_LOAD segment which contains relro section should update mapstart ++ */ ++static __always_inline size_t ++_extra_mmap(struct link_map *l, const struct loadcmd loadcmds[], size_t nloadcmds, ++ const struct loadcmd *c, ElfW(Addr) mapstart) ++{ ++ ElfW(Addr) mapend = mapstart + (c->mapend - c->mapstart); ++ ElfW(Addr) hugepage = ALIGN_DOWN(mapend, SIZE_2MB); ++ size_t relro_len = _get_relro_len(l, c); ++ mapstart += relro_len; ++ ++ /* ++ * 1. mapend is 2MB aligned ++ * 2. [mapstart, mapend) does not cover the tail hugepage start ++ */ ++ if (mapend == ALIGN_UP(mapend, SIZE_2MB) || mapstart > hugepage) ++ return 0; ++ ++ /* the initialized data area end in the tail hugepage */ ++ ElfW(Addr) end = (mapstart - relro_len) + ALIGN_UP(c->allocend - c->mapstart, GLRO(dl_pagesize)) >= ++ ALIGN_UP(mapend, SIZE_2MB) ? ALIGN_UP(mapend, SIZE_2MB) : ++ (mapstart - relro_len) + ALIGN_UP(c->allocend - c->mapstart, GLRO(dl_pagesize)); ++ ++ size_t extra_len = ALIGN_UP(mapend, SIZE_2MB) - mapend; ++ if ((end - hugepage < THRESHOLD * GLRO(dl_pagesize)) || ((c < loadcmds + (nloadcmds - 1)) && ++ (ALIGN_UP(mapend, SIZE_2MB) > (mapstart - relro_len) + c[1].mapstart - c->mapstart))) ++ { ++ extra_len = 0; ++ } ++ ++ return extra_len; ++} ++ ++/* ++ * PT_LOAD segment is described by p_filesz and p_memsz. ++ * The bytes from the file are mapped to the beginning of the memory segment. ++ * If the segment’s memory size (p_memsz) is larger than the file size (p_filesz), ++ * the extra bytes are defined to hold the value 0 and to follow the segment’s ++ * initialized area ++ */ ++static __always_inline const char * ++_mmap_segment(struct link_map *l, const struct loadcmd loadcmds[], size_t nloadcmds, ++ const struct loadcmd *c, ElfW(Addr) mapstart, int fd, size_t *mapseglen) ++{ ++ const char * errstring = NULL; ++ size_t extra_len = _extra_mmap(l, loadcmds, nloadcmds, c, mapstart); ++ size_t memsz_len = 0; ++ if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_FILES)) ++ _dl_debug_printf("\t%s(0x%lx): extra_len = 0x%lx\n\t{\n", __func__, ++ (unsigned long)c, extra_len); ++ ++ errstring = _mmap_segment_filesz(l, c, mapstart, extra_len, fd); ++ if (__glibc_unlikely (errstring != NULL)) ++ return errstring; ++ errstring = _mmap_segment_memsz(l, c, mapstart, extra_len, &memsz_len); ++ if (__glibc_unlikely (errstring != NULL)) ++ return errstring; ++ ++ *mapseglen = c->mapend - c->mapstart + extra_len + memsz_len; ++ if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_FILES)) ++ _dl_debug_printf("\t} => mapseglen = 0x%lx, memsz_len = 0x%lx\n", *mapseglen, memsz_len); ++ return NULL; ++} ++ ++static __always_inline void * ++_mmap_hole(const struct loadcmd *current, const struct loadcmd *next, ++ ElfW(Addr) mapstart, size_t mapseglen, int fd) ++{ ++ if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_FILES)) ++ _dl_debug_printf("\tmmap hole area:[%lx-%lx)\n", mapstart + mapseglen, ++ mapstart + (next->mapstart - current->mapstart)); ++ return __mmap((void *)(mapstart + mapseglen), ++ next->mapstart - (current->mapstart + mapseglen), ++ PROT_NONE, MAP_FILE|MAP_PRIVATE|MAP_FIXED, ++ fd, current->mapoff + mapseglen); ++} ++ ++static __always_inline const char * ++_dl_map_segments_largein (struct link_map *l, int fd, ++ const ElfW(Ehdr) *header, int type, ++ const struct loadcmd loadcmds[], size_t nloadcmds, ++ const size_t maplength, bool has_holes) ++{ ++ if (__glibc_unlikely (type != ET_DYN)) ++ return DL_MAP_SEGMENTS_ERROR_TYPE; ++ ++ const char *errstring = NULL; ++ const struct loadcmd *text = _find_exec_segment(loadcmds, nloadcmds); ++ if (__glibc_unlikely (text == NULL)) ++ return DL_FIND_EXEC_SEGMENT_ERROR; ++ ++ size_t maparealen; ++ void *map_area_start = __mmap_reserved_area(loadcmds, nloadcmds, &maparealen); ++ if (__glibc_unlikely (map_area_start == MAP_FAILED)) ++ return DL_MAP_RESERVED_HUGEPAGE_AREA_ERROR; ++ if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_FILES)) ++ _dl_debug_printf("reserved area:[%lx-%lx)\n", ++ (unsigned long)map_area_start, (unsigned long)map_area_start + maparealen); ++ ++ /* First to mmap text segment */ ++ const struct loadcmd * c = loadcmds; ++ ElfW(Addr) text_addr = ALIGN_UP((ElfW(Addr))map_area_start + (text->mapstart - c->mapstart), SIZE_2MB); ++ size_t mapseglen; ++ errstring = _mmap_segment(l, loadcmds, nloadcmds, text, text_addr, fd, &mapseglen); ++ if (__glibc_unlikely(errstring != NULL)) ++ goto unmap_reserved_area; ++ ++ const struct loadcmd *prev = text; ++ c = text + 1; ++ ElfW(Addr) map_addr = text_addr; ++ while (c < &loadcmds[nloadcmds]) ++ { ++ if (prev->mapstart + mapseglen > c->mapstart || c->mapstart < prev->mapstart) ++ { ++ errstring = DL_MAP_SEGMENTS_ERROR_ARRANGE; ++ goto unmap_reserved_area; ++ } ++ ++ if (prev->mapstart + mapseglen < c->mapstart && ++ _mmap_hole(prev, c, map_addr, mapseglen, fd) == MAP_FAILED) ++ { ++ errstring = DL_MAP_SEGMENTS_ERROR_MAP_HOLE_FILL; ++ goto unmap_reserved_area; ++ } ++ ++ map_addr += c->mapstart - prev->mapstart; ++ errstring = _mmap_segment(l, loadcmds, nloadcmds, c, map_addr, fd, &mapseglen); ++ if (__glibc_unlikely(errstring != NULL)) ++ goto unmap_reserved_area; ++ prev = c; ++ ++c; ++ } ++ ElfW(Addr) l_map_end = map_addr + mapseglen; ++ ++ /* search for the first segment */ ++ prev = text; ++ c = text - 1; ++ map_addr = text_addr; ++ while (c >= loadcmds) ++ { ++ if (prev->mapstart < c->mapstart) ++ { ++ errstring = DL_MAP_SEGMENTS_ERROR_ARRANGE; ++ goto unmap_reserved_area; ++ } ++ ++ map_addr -= prev->mapstart - c->mapstart; ++ errstring = _mmap_segment(l, loadcmds, nloadcmds, c, map_addr, fd, &mapseglen); ++ if (__glibc_unlikely(errstring != NULL)) ++ goto unmap_reserved_area; ++ ++ if (c->mapstart + mapseglen > prev->mapstart) ++ { ++ errstring = DL_MAP_SEGMENTS_ERROR_ARRANGE; ++ goto unmap_reserved_area; ++ } ++ ++ if (c->mapstart + mapseglen < prev->mapstart && ++ _mmap_hole(c, prev, map_addr, mapseglen, fd) == MAP_FAILED) ++ { ++ errstring = DL_MAP_SEGMENTS_ERROR_MAP_HOLE_FILL; ++ goto unmap_reserved_area; ++ } ++ prev = c; ++ --c; ++ } ++ ++ ++c; ++ l->l_map_start = map_addr; ++ l->l_map_end = l->l_map_start + maplength; ++ l->l_addr = l->l_map_start - c->mapstart; ++ l->l_contiguous = 1; ++ ++ c = loadcmds; ++ while (c < &loadcmds[nloadcmds]) ++ { ++ _dl_postprocess_loadcmd (l, header, c); ++ ++c; ++ } ++ ++ if (l->l_map_start > (ElfW(Addr))map_area_start) ++ { ++ if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_FILES)) ++ _dl_debug_printf("__munmap [%lx-%lx)\n", (ElfW(Addr))map_area_start, l->l_map_start); ++ __munmap(map_area_start, l->l_map_start - (ElfW(Addr))map_area_start); ++ } ++ ++ /* ++ * l->l_map_end is caculated by maplength, l_map_end may end with extra space ++ * use l->l_map_end may munmap extra space part ++ */ ++ if ((ElfW(Addr))map_area_start + maparealen > l_map_end) ++ { ++ if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_FILES)) ++ _dl_debug_printf("__munmap [%lx-%lx)\n", l_map_end, (ElfW(Addr))map_area_start + maparealen); ++ __munmap((void *)l_map_end, (ElfW(Addr))map_area_start + maparealen - l_map_end); ++ } ++ ++ return NULL; ++ ++unmap_reserved_area: ++ __munmap(map_area_start, maparealen); ++ ++ return errstring; ++} +diff --git a/elf/dl-support.c b/elf/dl-support.c +index d8c06ba7..f6ab2627 100644 +--- a/elf/dl-support.c ++++ b/elf/dl-support.c +@@ -166,6 +166,10 @@ size_t _dl_phnum; + uint64_t _dl_hwcap; + uint64_t _dl_hwcap2; + ++#ifdef HUGEPAGE_SHARED_LIB ++int _dl_hugepage_mask; ++#endif ++ + /* The value of the FPU control word the kernel will preset in hardware. */ + fpu_control_t _dl_fpu_control = _FPU_DEFAULT; + +diff --git a/elf/elf.h b/elf/elf.h +index 4738dfa2..c5315d1b 100644 +--- a/elf/elf.h ++++ b/elf/elf.h +@@ -730,6 +730,8 @@ typedef struct + + /* Legal values for p_flags (segment flags). */ + ++/* libhugetlbfs's hugeedit use 0x00100000, here use another */ ++#define PF_HUGEPAGE (0x01000000) + #define PF_X (1 << 0) /* Segment is executable */ + #define PF_W (1 << 1) /* Segment is writable */ + #define PF_R (1 << 2) /* Segment is readable */ +diff --git a/elf/hugepageedit.c b/elf/hugepageedit.c +new file mode 100644 +index 00000000..14a91a4b +--- /dev/null ++++ b/elf/hugepageedit.c +@@ -0,0 +1,169 @@ ++/* Mark ELF object ELF header hugepage flag Generic version. ++ Copyright (C) 2021-2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <http://www.gnu.org/licenses/>. */ ++#include <stdio.h> ++#include <stdlib.h> ++#include <fcntl.h> ++#include <string.h> ++#include <unistd.h> ++#include <elf.h> ++#include <link.h> ++#include <sys/stat.h> ++#include <sys/mman.h> ++#include <sys/types.h> ++ ++/* reference kernel load_elf_phdrs program header table size constraint */ ++#define ELF_MIN_ALIGN 4096 ++#define TOOL_NAME "hugepageedit" ++ ++int check_ptr(void *ptr, void *start, size_t len) ++{ ++ if (ptr < start || ptr > start + len) ++ return -1; ++ return 0; ++} ++ ++void print_usage(void) ++{ ++ fprintf(stderr, "%s [-x] [-d] <ELF file>\n" \ ++ "\tdefault mark all PT_LOAD segment PF_HUGEPAGE flag\n" \ ++ "\t-x option only mark executable PT_LOAD segment PF_HUGEPAGE flag\n" \ ++ "\t-d option delete all the PT_LOAD segment PF_HUGEPAGE flag\n", TOOL_NAME); ++} ++ ++int main(int argc, char *argv[]) ++{ ++ int exit_status = -1; ++ int i, opt, delete = 0, exec_only = 0; ++ while ((opt = getopt(argc, argv, "dx")) != -1) ++ { ++ switch (opt) ++ { ++ case 'd': ++ delete = 1; ++ break; ++ case 'x': ++ exec_only = 1; ++ break; ++ default: ++ print_usage(); ++ return 0; ++ } ++ } ++ ++ if (delete && exec_only) ++ { ++ fprintf(stderr, "can not specify -x and -d option at the same time\n"); ++ return -1; ++ } ++ ++ if (optind >= argc) ++ { ++ fprintf(stderr, "Expected argument after options\n"); ++ return -1; ++ } ++ ++ int fd = open(argv[optind], O_RDWR); ++ if (fd < 0) ++ { ++ perror("open"); ++ return -1; ++ } ++ ++ struct stat statbuf; ++ if (fstat(fd, &statbuf) != 0) ++ { ++ perror("fstat"); ++ goto close_fd; ++ } ++ ++ /* this ensures file is large enough to hold ELF header */ ++ if (statbuf.st_size < sizeof (ElfW(Ehdr))) ++ { ++ fprintf(stderr, "file is not large enough to hold ELF header\n"); ++ goto close_fd; ++ } ++ ++ void *ehdr = mmap(NULL, statbuf.st_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); ++ if (ehdr == MAP_FAILED) ++ { ++ perror("mmap"); ++ goto close_fd; ++ } ++ ++ if (memcmp(((ElfW(Ehdr) *) ehdr)->e_ident, ELFMAG, SELFMAG) != 0) ++ { ++ fprintf(stderr, "file is not ELF format\n"); ++ goto unmap; ++ } ++ ++ if (((ElfW(Ehdr) *)ehdr)->e_phentsize != sizeof(ElfW(Phdr))) ++ { ++ fprintf(stderr, "ELF header's e_phentsize mismatch ElfW(Phdr) size\n"); ++ goto unmap; ++ } ++ ++ unsigned int size = ((ElfW(Ehdr) *)ehdr)->e_phnum * sizeof(ElfW(Phdr)); ++ if (size == 0 || size > ELF_MIN_ALIGN) ++ { ++ fprintf(stderr, "The program header table size specified by ELF header is abnormal: %u\n", size); ++ goto unmap; ++ } ++ ++ void *ephdr_s = ehdr + ((ElfW(Ehdr) *)ehdr)->e_phoff; ++ void *ephdr_e = ehdr + ((ElfW(Ehdr) *)ehdr)->e_phoff + size; ++ ++ if (check_ptr(ephdr_s, ehdr, statbuf.st_size) || ++ check_ptr(ephdr_e, ehdr, statbuf.st_size)) ++ { ++ fprintf(stderr, "ELF porgram header table is not fully mmaped\n"); ++ goto unmap; ++ } ++ ++ ElfW(Phdr) *phdr = (ElfW(Phdr) *)ephdr_s; ++ /* ++ * Here, mark hugepage flag in ELF header e_ident padding bytes won't work. ++ * elf/dl-load.c open_verify will check if shared object ELF header e_ident ++ * padding bytes match expected[EI_NIDENT] byte array which padding bytes ++ * should be zero. If it mismatches, ld.so will exit abnormally ++ */ ++ for (i = 0; i < ((ElfW(Ehdr) *)ehdr)->e_phnum; i++) ++ { ++ if (phdr[i].p_type == PT_LOAD) ++ { ++ if (delete) ++ { ++ phdr[i].p_flags &= ~PF_HUGEPAGE; ++ } ++ else ++ { ++ if (exec_only && !(phdr[i].p_flags & PF_X)) ++ continue; ++ phdr[i].p_flags |= PF_HUGEPAGE; ++ } ++ } ++ } ++ exit_status = 0; ++ ++unmap: ++ munmap(ehdr, statbuf.st_size); ++ ++close_fd: ++ close(fd); ++ ++ return exit_status; ++} +diff --git a/elf/rtld.c b/elf/rtld.c +index a8ae8b31..85596ee4 100644 +--- a/elf/rtld.c ++++ b/elf/rtld.c +@@ -2543,6 +2543,40 @@ dl_main (const ElfW(Phdr) *phdr, + /* Once we return, _dl_sysdep_start will invoke + the DT_INIT functions and then *USER_ENTRY. */ + } ++ ++#ifdef HUGEPAGE_SHARED_LIB ++/* prase the hugepage use strategy of loading shared object */ ++static void ++process_dl_hugepage (const char *dl_hugepage) ++{ ++ static const struct ++ { ++ uint64_t option; ++ int flag; ++ } hpopts[] = ++ { ++ {DL_HUGEPAGE_LARGE_IN, DL_HUGEPAGE_LIB_LARGE_IN_FLAG}, ++ }; ++#define nhpopts (sizeof (hpopts) / sizeof (hpopts[0])) ++ ++ if (dl_hugepage == NULL) ++ return; ++ ++ char *endptr; ++ uint64_t val = _dl_strtoul (dl_hugepage, &endptr); ++ /* Invalid digit in input string */ ++ if (*endptr != '\0') ++ return; ++ ++ for (size_t cnt = 0; cnt < nhpopts; ++cnt) ++ if (val == hpopts[cnt].option) ++ { ++ GLRO(dl_hugepage_mask) |= hpopts[cnt].flag; ++ break; ++ } ++} ++#endif ++ + + /* This is a little helper function for resolving symbols while + tracing the binary. */ +@@ -2678,6 +2712,9 @@ process_envvars (struct dl_main_state *state) + char **runp = _environ; + char *envline; + char *debug_output = NULL; ++#ifdef HUGEPAGE_SHARED_LIB ++ bool hugepage_lib_env = false; ++#endif + + /* This is the default place for profiling data file. */ + GLRO(dl_profile_output) +@@ -2790,6 +2827,15 @@ process_envvars (struct dl_main_state *state) + if (!__libc_enable_secure + && memcmp (envline, "DYNAMIC_WEAK", 12) == 0) + GLRO(dl_dynamic_weak) = 1; ++ ++#ifdef HUGEPAGE_SHARED_LIB ++ if (memcmp (envline, "HUGEPAGE_LIB", 12) == 0 && envline[13] != '\0') ++ { ++ hugepage_lib_env = true; ++ process_dl_hugepage(&envline[13]); ++ } ++#endif ++ + break; + + case 13: +@@ -2812,6 +2858,13 @@ process_envvars (struct dl_main_state *state) + && memcmp (envline, "PROFILE_OUTPUT", 14) == 0 + && envline[15] != '\0') + GLRO(dl_profile_output) = &envline[15]; ++ ++#ifdef HUGEPAGE_SHARED_LIB ++ if (memcmp (envline, "HUGEPAGE_PROBE", 14) == 0 && ++ envline[15] != '\0') ++ GLRO(dl_hugepage_mask) |= DL_HUGEPAGE_PROBE_FLAG; ++#endif ++ + break; + + case 16: +@@ -2841,6 +2894,16 @@ process_envvars (struct dl_main_state *state) + } + } + ++#ifdef HUGEPAGE_SHARED_LIB ++ /* LIB_HUGEPAGE_LIB and HUGEPAGE_PROBE are both set. use LIB_HUGEPAGE_LIB */ ++ if ((GLRO(dl_hugepage_mask) & DL_HUGEPAGE_PROBE_FLAG) && hugepage_lib_env) ++ { ++ GLRO(dl_hugepage_mask) &= ~DL_HUGEPAGE_PROBE_FLAG; ++ } ++ /* unsetenv LD_HUGEPAGE_LIB, child process should not get this env */ ++ unsetenv("LD_HUGEPAGE_LIB"); ++#endif ++ + /* Extra security for SUID binaries. Remove all dangerous environment + variables. */ + if (__builtin_expect (__libc_enable_secure, 0)) +diff --git a/sysdeps/generic/ldsodefs.h b/sysdeps/generic/ldsodefs.h +index a1d70ce7..a9fffd66 100644 +--- a/sysdeps/generic/ldsodefs.h ++++ b/sysdeps/generic/ldsodefs.h +@@ -549,7 +549,11 @@ struct rtld_global_ro + /* These two are used only internally. */ + #define DL_DEBUG_HELP (1 << 10) + #define DL_DEBUG_PRELINK (1 << 11) +- ++#ifdef HUGEPAGE_SHARED_LIB ++#define DL_HUGEPAGE_PROBE_FLAG (1 << 31) ++#define DL_HUGEPAGE_LIB_LARGE_IN_FLAG (1 << 30) ++#define DL_HUGEPAGE_LARGE_IN 1 ++#endif + /* OS version. */ + EXTERN unsigned int _dl_osversion; + /* Platform name. */ +@@ -672,6 +676,11 @@ struct rtld_global_ro + platforms. */ + EXTERN uint64_t _dl_hwcap2; + ++#ifdef HUGEPAGE_SHARED_LIB ++ /* Mask for how to use hugepage */ ++ EXTERN int _dl_hugepage_mask; ++#endif ++ + #ifdef SHARED + /* We add a function table to _rtld_global which is then used to + call the function instead of going through the PLT. The result +-- +2.37.3 + |