diff options
Diffstat (limited to 'malloc-hugepage-0005-malloc-Add-Huge-Page-support-to-arenas.patch')
| -rw-r--r-- | malloc-hugepage-0005-malloc-Add-Huge-Page-support-to-arenas.patch | 338 |
1 files changed, 338 insertions, 0 deletions
diff --git a/malloc-hugepage-0005-malloc-Add-Huge-Page-support-to-arenas.patch b/malloc-hugepage-0005-malloc-Add-Huge-Page-support-to-arenas.patch new file mode 100644 index 0000000..f759588 --- /dev/null +++ b/malloc-hugepage-0005-malloc-Add-Huge-Page-support-to-arenas.patch @@ -0,0 +1,338 @@ +From c1beb51d08d3d7ec935b0a2419b4c6fad91d1969 Mon Sep 17 00:00:00 2001 +From: Adhemerval Zanella <adhemerval.zanella@linaro.org> +Date: Fri, 20 Aug 2021 13:22:35 -0300 +Subject: [PATCH 5/7] malloc: Add Huge Page support to arenas + +It is enabled as default for glibc.malloc.hugetlb set to 2 or higher. +It also uses a non configurable minimum value and maximum value, +currently set respectively to 1 and 4 selected huge page size. + +The arena allocation with huge pages does not use MAP_NORESERVE. As +indicate by kernel internal documentation [1], the flag might trigger +a SIGBUS on soft page faults if at memory access there is no left +pages in the pool. + +On systems without a reserved huge pages pool, is just stress the +mmap(MAP_HUGETLB) allocation failure. To improve test coverage it is +required to create a pool with some allocated pages. + +Checked on x86_64-linux-gnu with no reserved pages, 10 reserved pages +(which trigger mmap(MAP_HUGETBL) failures) and with 256 reserved pages +(which does not trigger mmap(MAP_HUGETLB) failures). + +[1] https://www.kernel.org/doc/html/v4.18/vm/hugetlbfs_reserv.html#resv-map-modifications + +Reviewed-by: DJ Delorie <dj@redhat.com> +--- + malloc/Makefile | 7 ++- + malloc/arena.c | 134 +++++++++++++++++++++++++++++++++--------------- + malloc/malloc.c | 2 +- + 3 files changed, 99 insertions(+), 44 deletions(-) + +diff --git a/malloc/Makefile b/malloc/Makefile +index e9a6666d22..451eb84612 100644 +--- a/malloc/Makefile ++++ b/malloc/Makefile +@@ -91,10 +91,15 @@ tests-exclude-hugetlb1 = \ + tst-malloc-usable \ + tst-malloc-usable-tunables \ + tst-mallocstate ++# The tst-free-errno relies on the used malloc page size to mmap an ++# overlapping region. ++tests-exclude-hugetlb2 = \ ++ $(tests-exclude-hugetlb1) \ ++ tst-free-errno + tests-malloc-hugetlb1 = \ + $(filter-out $(tests-exclude-hugetlb1), $(tests)) + tests-malloc-hugetlb2 = \ +- $(filter-out $(tests-exclude-hugetlb1), $(tests)) ++ $(filter-out $(tests-exclude-hugetlb2), $(tests)) + + # -lmcheck needs __malloc_initialize_hook, which was deprecated in 2.24. + ifeq ($(have-GLIBC_2.23)$(build-shared),yesyes) +diff --git a/malloc/arena.c b/malloc/arena.c +index 9a6e1af2bd..e1852f8597 100644 +--- a/malloc/arena.c ++++ b/malloc/arena.c +@@ -41,6 +41,29 @@ + mmap threshold, so that requests with a size just below that + threshold can be fulfilled without creating too many heaps. */ + ++/* When huge pages are used to create new arenas, the maximum and minumum ++ size are based on the runtime defined huge page size. */ ++ ++static inline size_t ++heap_min_size (void) ++{ ++#if HAVE_TUNABLES ++ return mp_.hp_pagesize == 0 ? HEAP_MIN_SIZE : mp_.hp_pagesize; ++#else ++ return HEAP_MIN_SIZE; ++#endif ++} ++ ++static inline size_t ++heap_max_size (void) ++{ ++#if HAVE_TUNABLES ++ return mp_.hp_pagesize == 0 ? HEAP_MAX_SIZE : mp_.hp_pagesize * 4; ++#else ++ return HEAP_MAX_SIZE; ++#endif ++} ++ + /***************************************************************************/ + + #define top(ar_ptr) ((ar_ptr)->top) +@@ -56,10 +79,11 @@ typedef struct _heap_info + size_t size; /* Current size in bytes. */ + size_t mprotect_size; /* Size in bytes that has been mprotected + PROT_READ|PROT_WRITE. */ ++ size_t pagesize; /* Page size used when allocating the arena. */ + /* Make sure the following data is properly aligned, particularly + that sizeof (heap_info) + 2 * SIZE_SZ is a multiple of + MALLOC_ALIGNMENT. */ +- char pad[-6 * SIZE_SZ & MALLOC_ALIGN_MASK]; ++ char pad[-3 * SIZE_SZ & MALLOC_ALIGN_MASK]; + } heap_info; + + /* Get a compile-time error if the heap_info padding is not correct +@@ -125,10 +149,18 @@ static bool __malloc_initialized = false; + + /* find the heap and corresponding arena for a given ptr */ + +-#define heap_for_ptr(ptr) \ +- ((heap_info *) ((unsigned long) (ptr) & ~(HEAP_MAX_SIZE - 1))) +-#define arena_for_chunk(ptr) \ +- (chunk_main_arena (ptr) ? &main_arena : heap_for_ptr (ptr)->ar_ptr) ++static inline heap_info * ++heap_for_ptr (void *ptr) ++{ ++ size_t max_size = heap_max_size (); ++ return PTR_ALIGN_DOWN (ptr, max_size); ++} ++ ++static inline struct malloc_state * ++arena_for_chunk (mchunkptr ptr) ++{ ++ return chunk_main_arena (ptr) ? &main_arena : heap_for_ptr (ptr)->ar_ptr; ++} + + + /**************************************************************************/ +@@ -443,71 +475,72 @@ static char *aligned_heap_area; + of the page size. */ + + static heap_info * +-new_heap (size_t size, size_t top_pad) ++alloc_new_heap (size_t size, size_t top_pad, size_t pagesize, ++ int mmap_flags) + { +- size_t pagesize = GLRO (dl_pagesize); + char *p1, *p2; + unsigned long ul; + heap_info *h; ++ size_t min_size = heap_min_size (); ++ size_t max_size = heap_max_size (); + +- if (size + top_pad < HEAP_MIN_SIZE) +- size = HEAP_MIN_SIZE; +- else if (size + top_pad <= HEAP_MAX_SIZE) ++ if (size + top_pad < min_size) ++ size = min_size; ++ else if (size + top_pad <= max_size) + size += top_pad; +- else if (size > HEAP_MAX_SIZE) ++ else if (size > max_size) + return 0; + else +- size = HEAP_MAX_SIZE; ++ size = max_size; + size = ALIGN_UP (size, pagesize); + +- /* A memory region aligned to a multiple of HEAP_MAX_SIZE is needed. ++ /* A memory region aligned to a multiple of max_size is needed. + No swap space needs to be reserved for the following large + mapping (on Linux, this is the case for all non-writable mappings + anyway). */ + p2 = MAP_FAILED; + if (aligned_heap_area) + { +- p2 = (char *) MMAP (aligned_heap_area, HEAP_MAX_SIZE, PROT_NONE, +- MAP_NORESERVE); ++ p2 = (char *) MMAP (aligned_heap_area, max_size, PROT_NONE, mmap_flags); + aligned_heap_area = NULL; +- if (p2 != MAP_FAILED && ((unsigned long) p2 & (HEAP_MAX_SIZE - 1))) ++ if (p2 != MAP_FAILED && ((unsigned long) p2 & (max_size - 1))) + { +- __munmap (p2, HEAP_MAX_SIZE); ++ __munmap (p2, max_size); + p2 = MAP_FAILED; + } + } + if (p2 == MAP_FAILED) + { +- p1 = (char *) MMAP (0, HEAP_MAX_SIZE << 1, PROT_NONE, MAP_NORESERVE); ++ p1 = (char *) MMAP (0, max_size << 1, PROT_NONE, mmap_flags); + if (p1 != MAP_FAILED) + { +- p2 = (char *) (((unsigned long) p1 + (HEAP_MAX_SIZE - 1)) +- & ~(HEAP_MAX_SIZE - 1)); ++ p2 = (char *) (((unsigned long) p1 + (max_size - 1)) ++ & ~(max_size - 1)); + ul = p2 - p1; + if (ul) + __munmap (p1, ul); + else +- aligned_heap_area = p2 + HEAP_MAX_SIZE; +- __munmap (p2 + HEAP_MAX_SIZE, HEAP_MAX_SIZE - ul); ++ aligned_heap_area = p2 + max_size; ++ __munmap (p2 + max_size, max_size - ul); + } + else + { +- /* Try to take the chance that an allocation of only HEAP_MAX_SIZE ++ /* Try to take the chance that an allocation of only max_size + is already aligned. */ +- p2 = (char *) MMAP (0, HEAP_MAX_SIZE, PROT_NONE, MAP_NORESERVE); ++ p2 = (char *) MMAP (0, max_size, PROT_NONE, mmap_flags); + if (p2 == MAP_FAILED) + return 0; + +- if ((unsigned long) p2 & (HEAP_MAX_SIZE - 1)) ++ if ((unsigned long) p2 & (max_size - 1)) + { +- __munmap (p2, HEAP_MAX_SIZE); ++ __munmap (p2, max_size); + return 0; + } + } + } + if (__mprotect (p2, size, mtag_mmap_flags | PROT_READ | PROT_WRITE) != 0) + { +- __munmap (p2, HEAP_MAX_SIZE); ++ __munmap (p2, max_size); + return 0; + } + +@@ -516,22 +549,42 @@ new_heap (size_t size, size_t top_pad) + h = (heap_info *) p2; + h->size = size; + h->mprotect_size = size; ++ h->pagesize = pagesize; + LIBC_PROBE (memory_heap_new, 2, h, h->size); + return h; + } + ++static heap_info * ++new_heap (size_t size, size_t top_pad) ++{ ++#if HAVE_TUNABLES ++ if (__glibc_unlikely (mp_.hp_pagesize != 0)) ++ { ++ /* MAP_NORESERVE is not used for huge pages because some kernel may ++ not reserve the mmap region and a subsequent access may trigger ++ a SIGBUS if there is no free pages in the pool. */ ++ heap_info *h = alloc_new_heap (size, top_pad, mp_.hp_pagesize, ++ mp_.hp_flags); ++ if (h != NULL) ++ return h; ++ } ++#endif ++ return alloc_new_heap (size, top_pad, GLRO (dl_pagesize), MAP_NORESERVE); ++} ++ + /* Grow a heap. size is automatically rounded up to a + multiple of the page size. */ + + static int + grow_heap (heap_info *h, long diff) + { +- size_t pagesize = GLRO (dl_pagesize); ++ size_t pagesize = h->pagesize; ++ size_t max_size = heap_max_size (); + long new_size; + + diff = ALIGN_UP (diff, pagesize); + new_size = (long) h->size + diff; +- if ((unsigned long) new_size > (unsigned long) HEAP_MAX_SIZE) ++ if ((unsigned long) new_size > (unsigned long) max_size) + return -1; + + if ((unsigned long) new_size > h->mprotect_size) +@@ -581,21 +634,14 @@ shrink_heap (heap_info *h, long diff) + + /* Delete a heap. */ + +-#define delete_heap(heap) \ +- do { \ +- if ((char *) (heap) + HEAP_MAX_SIZE == aligned_heap_area) \ +- aligned_heap_area = NULL; \ +- __munmap ((char *) (heap), HEAP_MAX_SIZE); \ +- } while (0) +- + static int + heap_trim (heap_info *heap, size_t pad) + { + mstate ar_ptr = heap->ar_ptr; +- unsigned long pagesz = GLRO (dl_pagesize); + mchunkptr top_chunk = top (ar_ptr), p; + heap_info *prev_heap; + long new_size, top_size, top_area, extra, prev_size, misalign; ++ size_t max_size = heap_max_size (); + + /* Can this heap go away completely? */ + while (top_chunk == chunk_at_offset (heap, sizeof (*heap))) +@@ -612,19 +658,23 @@ heap_trim (heap_info *heap, size_t pad) + assert (new_size > 0 && new_size < (long) (2 * MINSIZE)); + if (!prev_inuse (p)) + new_size += prev_size (p); +- assert (new_size > 0 && new_size < HEAP_MAX_SIZE); +- if (new_size + (HEAP_MAX_SIZE - prev_heap->size) < pad + MINSIZE + pagesz) ++ assert (new_size > 0 && new_size < max_size); ++ if (new_size + (max_size - prev_heap->size) < pad + MINSIZE ++ + heap->pagesize) + break; + ar_ptr->system_mem -= heap->size; + LIBC_PROBE (memory_heap_free, 2, heap, heap->size); +- delete_heap (heap); ++ if ((char *) heap + max_size == aligned_heap_area) ++ aligned_heap_area = NULL; ++ __munmap (heap, max_size); + heap = prev_heap; + if (!prev_inuse (p)) /* consolidate backward */ + { + p = prev_chunk (p); + unlink_chunk (ar_ptr, p); + } +- assert (((unsigned long) ((char *) p + new_size) & (pagesz - 1)) == 0); ++ assert (((unsigned long) ((char *) p + new_size) & (heap->pagesize - 1)) ++ == 0); + assert (((char *) p + new_size) == ((char *) heap + heap->size)); + top (ar_ptr) = top_chunk = p; + set_head (top_chunk, new_size | PREV_INUSE); +@@ -644,7 +694,7 @@ heap_trim (heap_info *heap, size_t pad) + return 0; + + /* Release in pagesize units and round down to the nearest page. */ +- extra = ALIGN_DOWN(top_area - pad, pagesz); ++ extra = ALIGN_DOWN(top_area - pad, heap->pagesize); + if (extra == 0) + return 0; + +diff --git a/malloc/malloc.c b/malloc/malloc.c +index 75efdc2ee7..1698d45d1e 100644 +--- a/malloc/malloc.c ++++ b/malloc/malloc.c +@@ -5302,7 +5302,7 @@ static __always_inline int + do_set_mmap_threshold (size_t value) + { + /* Forbid setting the threshold too high. */ +- if (value <= HEAP_MAX_SIZE / 2) ++ if (value <= heap_max_size () / 2) + { + LIBC_PROBE (memory_mallopt_mmap_threshold, 3, value, mp_.mmap_threshold, + mp_.no_dyn_threshold); +-- +2.33.0 + |