diff options
Diffstat (limited to 'LoongArch-Add-ifunc-support-for-memrchr-lsx-lasx.patch')
| -rw-r--r-- | LoongArch-Add-ifunc-support-for-memrchr-lsx-lasx.patch | 417 |
1 files changed, 417 insertions, 0 deletions
diff --git a/LoongArch-Add-ifunc-support-for-memrchr-lsx-lasx.patch b/LoongArch-Add-ifunc-support-for-memrchr-lsx-lasx.patch new file mode 100644 index 0000000..26a0f40 --- /dev/null +++ b/LoongArch-Add-ifunc-support-for-memrchr-lsx-lasx.patch @@ -0,0 +1,417 @@ +From c4c272fb8067364530a2a78df92c37403acc963f Mon Sep 17 00:00:00 2001 +From: dengjianbo <dengjianbo@loongson.cn> +Date: Mon, 28 Aug 2023 10:08:37 +0800 +Subject: [PATCH 16/29] LoongArch: Add ifunc support for memrchr{lsx, lasx} + +According to glibc memrchr microbenchmark, this implementation could reduce +the runtime as following: + +Name Percent of rutime reduced +memrchr-lasx 20%-83% +memrchr-lsx 20%-64% + +Signed-off-by: Peng Fan <fanpeng@loongson.cn> +Signed-off-by: ticat_fp <fanpeng@loongson.cn> +--- + sysdeps/loongarch/lp64/multiarch/Makefile | 3 + + .../lp64/multiarch/ifunc-impl-list.c | 8 ++ + .../loongarch/lp64/multiarch/ifunc-memrchr.h | 40 ++++++ + .../lp64/multiarch/memrchr-generic.c | 23 ++++ + .../loongarch/lp64/multiarch/memrchr-lasx.S | 123 ++++++++++++++++++ + .../loongarch/lp64/multiarch/memrchr-lsx.S | 105 +++++++++++++++ + sysdeps/loongarch/lp64/multiarch/memrchr.c | 33 +++++ + 7 files changed, 335 insertions(+) + create mode 100644 sysdeps/loongarch/lp64/multiarch/ifunc-memrchr.h + create mode 100644 sysdeps/loongarch/lp64/multiarch/memrchr-generic.c + create mode 100644 sysdeps/loongarch/lp64/multiarch/memrchr-lasx.S + create mode 100644 sysdeps/loongarch/lp64/multiarch/memrchr-lsx.S + create mode 100644 sysdeps/loongarch/lp64/multiarch/memrchr.c + +diff --git a/sysdeps/loongarch/lp64/multiarch/Makefile b/sysdeps/loongarch/lp64/multiarch/Makefile +index 2f4802cf..7b87bc90 100644 +--- a/sysdeps/loongarch/lp64/multiarch/Makefile ++++ b/sysdeps/loongarch/lp64/multiarch/Makefile +@@ -27,5 +27,8 @@ sysdep_routines += \ + memchr-aligned \ + memchr-lsx \ + memchr-lasx \ ++ memrchr-generic \ ++ memrchr-lsx \ ++ memrchr-lasx \ + # sysdep_routines + endif +diff --git a/sysdeps/loongarch/lp64/multiarch/ifunc-impl-list.c b/sysdeps/loongarch/lp64/multiarch/ifunc-impl-list.c +index a567b9cf..8bd5489e 100644 +--- a/sysdeps/loongarch/lp64/multiarch/ifunc-impl-list.c ++++ b/sysdeps/loongarch/lp64/multiarch/ifunc-impl-list.c +@@ -109,5 +109,13 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + #endif + IFUNC_IMPL_ADD (array, i, memchr, 1, __memchr_aligned) + ) ++ ++ IFUNC_IMPL (i, name, memrchr, ++#if !defined __loongarch_soft_float ++ IFUNC_IMPL_ADD (array, i, memrchr, SUPPORT_LASX, __memrchr_lasx) ++ IFUNC_IMPL_ADD (array, i, memrchr, SUPPORT_LSX, __memrchr_lsx) ++#endif ++ IFUNC_IMPL_ADD (array, i, memrchr, 1, __memrchr_generic) ++ ) + return i; + } +diff --git a/sysdeps/loongarch/lp64/multiarch/ifunc-memrchr.h b/sysdeps/loongarch/lp64/multiarch/ifunc-memrchr.h +new file mode 100644 +index 00000000..8215f9ad +--- /dev/null ++++ b/sysdeps/loongarch/lp64/multiarch/ifunc-memrchr.h +@@ -0,0 +1,40 @@ ++/* Common definition for memrchr implementation. ++ All versions must be listed in ifunc-impl-list.c. ++ Copyright (C) 2023 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <ldsodefs.h> ++#include <ifunc-init.h> ++ ++#if !defined __loongarch_soft_float ++extern __typeof (REDIRECT_NAME) OPTIMIZE (lasx) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (lsx) attribute_hidden; ++#endif ++extern __typeof (REDIRECT_NAME) OPTIMIZE (generic) attribute_hidden; ++ ++static inline void * ++IFUNC_SELECTOR (void) ++{ ++#if !defined __loongarch_soft_float ++ if (SUPPORT_LASX) ++ return OPTIMIZE (lasx); ++ else if (SUPPORT_LSX) ++ return OPTIMIZE (lsx); ++ else ++#endif ++ return OPTIMIZE (generic); ++} +diff --git a/sysdeps/loongarch/lp64/multiarch/memrchr-generic.c b/sysdeps/loongarch/lp64/multiarch/memrchr-generic.c +new file mode 100644 +index 00000000..ced61ebc +--- /dev/null ++++ b/sysdeps/loongarch/lp64/multiarch/memrchr-generic.c +@@ -0,0 +1,23 @@ ++/* Generic implementation of memrchr. ++ Copyright (C) 2023 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library. If not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#if IS_IN (libc) ++# define MEMRCHR __memrchr_generic ++#endif ++ ++#include <string/memrchr.c> +diff --git a/sysdeps/loongarch/lp64/multiarch/memrchr-lasx.S b/sysdeps/loongarch/lp64/multiarch/memrchr-lasx.S +new file mode 100644 +index 00000000..5f3e0d06 +--- /dev/null ++++ b/sysdeps/loongarch/lp64/multiarch/memrchr-lasx.S +@@ -0,0 +1,123 @@ ++/* Optimized memrchr implementation using LoongArch LASX instructions. ++ Copyright (C) 2023 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library. If not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <sysdep.h> ++#include <sys/regdef.h> ++#include <sys/asm.h> ++ ++#if IS_IN (libc) && !defined __loongarch_soft_float ++ ++#ifndef MEMRCHR ++# define MEMRCHR __memrchr_lasx ++#endif ++ ++LEAF(MEMRCHR, 6) ++ beqz a2, L(ret0) ++ addi.d a2, a2, -1 ++ add.d a3, a0, a2 ++ andi t1, a3, 0x3f ++ ++ bstrins.d a3, zero, 5, 0 ++ addi.d t1, t1, 1 ++ xvld xr0, a3, 0 ++ xvld xr1, a3, 32 ++ ++ sub.d t2, zero, t1 ++ li.d t3, -1 ++ xvreplgr2vr.b xr2, a1 ++ andi t4, a0, 0x3f ++ ++ srl.d t2, t3, t2 ++ xvseq.b xr0, xr0, xr2 ++ xvseq.b xr1, xr1, xr2 ++ xvmsknz.b xr0, xr0 ++ ++ ++ xvmsknz.b xr1, xr1 ++ xvpickve.w xr3, xr0, 4 ++ xvpickve.w xr4, xr1, 4 ++ vilvl.h vr0, vr3, vr0 ++ ++ vilvl.h vr1, vr4, vr1 ++ vilvl.w vr0, vr1, vr0 ++ movfr2gr.d t0, fa0 ++ and t0, t0, t2 ++ ++ bltu a2, t1, L(end) ++ bnez t0, L(found) ++ bstrins.d a0, zero, 5, 0 ++L(loop): ++ xvld xr0, a3, -64 ++ ++ xvld xr1, a3, -32 ++ addi.d a3, a3, -64 ++ xvseq.b xr0, xr0, xr2 ++ xvseq.b xr1, xr1, xr2 ++ ++ ++ beq a0, a3, L(out) ++ xvmax.bu xr3, xr0, xr1 ++ xvseteqz.v fcc0, xr3 ++ bcnez fcc0, L(loop) ++ ++ xvmsknz.b xr0, xr0 ++ xvmsknz.b xr1, xr1 ++ xvpickve.w xr3, xr0, 4 ++ xvpickve.w xr4, xr1, 4 ++ ++ vilvl.h vr0, vr3, vr0 ++ vilvl.h vr1, vr4, vr1 ++ vilvl.w vr0, vr1, vr0 ++ movfr2gr.d t0, fa0 ++ ++L(found): ++ addi.d a0, a3, 63 ++ clz.d t1, t0 ++ sub.d a0, a0, t1 ++ jr ra ++ ++ ++L(out): ++ xvmsknz.b xr0, xr0 ++ xvmsknz.b xr1, xr1 ++ xvpickve.w xr3, xr0, 4 ++ xvpickve.w xr4, xr1, 4 ++ ++ vilvl.h vr0, vr3, vr0 ++ vilvl.h vr1, vr4, vr1 ++ vilvl.w vr0, vr1, vr0 ++ movfr2gr.d t0, fa0 ++ ++L(end): ++ sll.d t2, t3, t4 ++ and t0, t0, t2 ++ addi.d a0, a3, 63 ++ clz.d t1, t0 ++ ++ sub.d a0, a0, t1 ++ maskeqz a0, a0, t0 ++ jr ra ++L(ret0): ++ move a0, zero ++ ++ ++ jr ra ++END(MEMRCHR) ++ ++libc_hidden_builtin_def (MEMRCHR) ++#endif +diff --git a/sysdeps/loongarch/lp64/multiarch/memrchr-lsx.S b/sysdeps/loongarch/lp64/multiarch/memrchr-lsx.S +new file mode 100644 +index 00000000..39a7c8b0 +--- /dev/null ++++ b/sysdeps/loongarch/lp64/multiarch/memrchr-lsx.S +@@ -0,0 +1,105 @@ ++/* Optimized memrchr implementation using LoongArch LSX instructions. ++ Copyright (C) 2023 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library. If not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <sysdep.h> ++#include <sys/regdef.h> ++#include <sys/asm.h> ++ ++#if IS_IN (libc) && !defined __loongarch_soft_float ++ ++# define MEMRCHR __memrchr_lsx ++ ++LEAF(MEMRCHR, 6) ++ beqz a2, L(ret0) ++ addi.d a2, a2, -1 ++ add.d a3, a0, a2 ++ andi t1, a3, 0x1f ++ ++ bstrins.d a3, zero, 4, 0 ++ addi.d t1, t1, 1 ++ vld vr0, a3, 0 ++ vld vr1, a3, 16 ++ ++ sub.d t2, zero, t1 ++ li.d t3, -1 ++ vreplgr2vr.b vr2, a1 ++ andi t4, a0, 0x1f ++ ++ srl.d t2, t3, t2 ++ vseq.b vr0, vr0, vr2 ++ vseq.b vr1, vr1, vr2 ++ vmsknz.b vr0, vr0 ++ ++ ++ vmsknz.b vr1, vr1 ++ vilvl.h vr0, vr1, vr0 ++ movfr2gr.s t0, fa0 ++ and t0, t0, t2 ++ ++ bltu a2, t1, L(end) ++ bnez t0, L(found) ++ bstrins.d a0, zero, 4, 0 ++L(loop): ++ vld vr0, a3, -32 ++ ++ vld vr1, a3, -16 ++ addi.d a3, a3, -32 ++ vseq.b vr0, vr0, vr2 ++ vseq.b vr1, vr1, vr2 ++ ++ beq a0, a3, L(out) ++ vmax.bu vr3, vr0, vr1 ++ vseteqz.v fcc0, vr3 ++ bcnez fcc0, L(loop) ++ ++ ++ vmsknz.b vr0, vr0 ++ vmsknz.b vr1, vr1 ++ vilvl.h vr0, vr1, vr0 ++ movfr2gr.s t0, fa0 ++ ++L(found): ++ addi.d a0, a3, 31 ++ clz.w t1, t0 ++ sub.d a0, a0, t1 ++ jr ra ++ ++L(out): ++ vmsknz.b vr0, vr0 ++ vmsknz.b vr1, vr1 ++ vilvl.h vr0, vr1, vr0 ++ movfr2gr.s t0, fa0 ++ ++L(end): ++ sll.d t2, t3, t4 ++ and t0, t0, t2 ++ addi.d a0, a3, 31 ++ clz.w t1, t0 ++ ++ ++ sub.d a0, a0, t1 ++ maskeqz a0, a0, t0 ++ jr ra ++L(ret0): ++ move a0, zero ++ ++ jr ra ++END(MEMRCHR) ++ ++libc_hidden_builtin_def (MEMRCHR) ++#endif +diff --git a/sysdeps/loongarch/lp64/multiarch/memrchr.c b/sysdeps/loongarch/lp64/multiarch/memrchr.c +new file mode 100644 +index 00000000..8baba9ab +--- /dev/null ++++ b/sysdeps/loongarch/lp64/multiarch/memrchr.c +@@ -0,0 +1,33 @@ ++/* Multiple versions of memrchr. ++ All versions must be listed in ifunc-impl-list.c. ++ Copyright (C) 2023 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++/* Define multiple versions only for the definition in libc. */ ++#if IS_IN (libc) ++# define memrchr __redirect_memrchr ++# include <string.h> ++# undef memrchr ++ ++# define SYMBOL_NAME memrchr ++# include "ifunc-memrchr.h" ++ ++libc_ifunc_redirected (__redirect_memrchr, __memrchr, IFUNC_SELECTOR ()); ++libc_hidden_def (__memrchr) ++weak_alias (__memrchr, memrchr) ++ ++#endif +-- +2.33.0 + |